diff --git a/.github/workflows/rust-boundaries.yml b/.github/workflows/rust-boundaries.yml new file mode 100644 index 000000000..8cdef66b0 --- /dev/null +++ b/.github/workflows/rust-boundaries.yml @@ -0,0 +1,268 @@ +--- +name: Rust boundary verification + +on: + pull_request: + paths: + - 'rust/**' + - 'cuprum/_streams_rs.py' + - 'cuprum/_pipeline_stream*.py' + - 'pyproject.toml' + - 'scripts/*boundary*' + - 'scripts/tests/test_boundary*' + - 'tools/**' + - 'Makefile' + - '.github/workflows/rust-boundaries.yml' + schedule: + - cron: '23 4 * * 1' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: rust-boundaries-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + native: + name: Native contracts (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + strategy: + fail-fast: false + max-parallel: 2 + matrix: + os: [ubuntu-latest, windows-2022, macos-latest] + defaults: + run: + shell: bash + working-directory: rust + env: + CARGO_BUILD_JOBS: '2' + RUSTFLAGS: '-D warnings' + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + with: + # The job builds and runs repository code, so the checkout token + # must not stay in .git/config where that code could reach it. + persist-credentials: false + - name: Compute shared dependency cache keys + uses: ./.github/actions/cache-keys + with: + python-version: '3.13' + - name: Restore the shared Cargo registry + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: ${{ env.CARGO_CACHE_KEY }} + restore-keys: ${{ env.CARGO_CACHE_PREFIX }} + - name: Install supported project toolchain + run: rustup toolchain install 1.85.0 --profile minimal --component clippy + - name: Restore the Linux boundary compiler cache + if: runner.os == 'Linux' + id: boundary-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/sccache + key: boundary-native-linux-${{ github.run_id }} + restore-keys: boundary-native-linux- + - name: Set up sccache + if: runner.os == 'Linux' + uses: ./.github/actions/setup-sccache + with: + cache-size: 1G + - name: Reset compiler-cache counters + if: runner.os == 'Linux' + run: sccache --zero-stats + - name: Check default feature configuration + run: cargo check -p cuprum-streams -p cuprum-native-io --all-targets --no-default-features + - name: Lint all feature and helper targets + run: cargo clippy -p cuprum-streams -p cuprum-native-io --all-targets --all-features -- -D warnings + - name: Exercise native effects and safe stream policy + run: | + set -o pipefail + mkdir -p target/boundary-verification + cargo test -p cuprum-streams -p cuprum-native-io --all-targets --all-features 2>&1 | + tee target/boundary-verification/native.log + - name: Record compiler-cache effectiveness + if: always() && runner.os == 'Linux' + run: sccache --show-stats --stats-format json + - name: Archive native evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: native-boundaries-${{ runner.os }} + path: rust/target/boundary-verification + retention-days: 14 + + - name: Save reviewed boundary cache generation + if: runner.os == 'Linux' && github.event_name == 'schedule' && github.ref == 'refs/heads/main' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/sccache + key: ${{ steps.boundary-cache.outputs.cache-primary-key }} + + verus: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + with: + # The job builds and runs repository code, so the checkout token + # must not stay in .git/config where that code could reach it. + persist-credentials: false + - name: Compute shared dependency cache keys + uses: ./.github/actions/cache-keys + with: + python-version: '3.13' + - name: Restore the shared Cargo registry + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: ${{ env.CARGO_CACHE_KEY }} + restore-keys: ${{ env.CARGO_CACHE_PREFIX }} + - uses: astral-sh/setup-uv@f98e06938123ccabd21905ea5d0069192241f9f1 # v8.3.1 + with: + python-version: '3.13' + - name: Restore pinned verifier, solver, and compiler objects + id: boundary-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.local/share/cuprum-verus-0.2026.09.06.8dea4a2 + ~/.rustup/toolchains/1.98.0-x86_64-unknown-linux-gnu + .cache/boundary-z3 + ~/.cache/sccache + key: boundary-verus-${{ hashFiles('tools/verus/*', 'scripts/install_boundary_z3.py') }}-${{ github.run_id }} + restore-keys: boundary-verus-${{ hashFiles('tools/verus/*', 'scripts/install_boundary_z3.py') }}- + - name: Set up sccache + uses: ./.github/actions/setup-sccache + with: + cache-size: 1G + - name: Reset compiler-cache counters + run: sccache --zero-stats + - name: Install prebuilt verifier and solver + run: | + make install-verus + rustup toolchain install 1.98.0 --profile minimal + - name: Test proof correspondence and binary installation contracts + run: uv run pytest scripts/tests/test_boundary_*.py + - name: Verify production kernels and compiler prohibition + run: | + set -o pipefail + mkdir -p rust/target/boundary-verification + make boundary-verus boundary-contract 2>&1 | tee rust/target/boundary-verification/verus.log + - name: Record compiler-cache effectiveness + if: always() + run: sccache --show-stats --stats-format json + - name: Archive proof and negative compiler evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: verus-and-compiler-contracts + path: rust/target/boundary-verification + retention-days: 14 + + - name: Save reviewed boundary cache generation + if: github.event_name == 'schedule' && github.ref == 'refs/heads/main' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.local/share/cuprum-verus-0.2026.09.06.8dea4a2 + ~/.rustup/toolchains/1.98.0-x86_64-unknown-linux-gnu + .cache/boundary-z3 + ~/.cache/sccache + key: ${{ steps.boundary-cache.outputs.cache-primary-key }} + + extended: + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false + max-parallel: 1 + matrix: + check: [kani, miri] + env: + CARGO_BUILD_JOBS: '2' + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + with: + # The job builds and runs repository code, so the checkout token + # must not stay in .git/config where that code could reach it. + persist-credentials: false + - name: Compute shared dependency cache keys + uses: ./.github/actions/cache-keys + with: + python-version: '3.13' + - name: Restore the shared Cargo registry + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: ${{ env.CARGO_CACHE_KEY }} + restore-keys: ${{ env.CARGO_CACHE_PREFIX }} + - uses: astral-sh/setup-uv@f98e06938123ccabd21905ea5d0069192241f9f1 # v8.3.1 + with: + python-version: '3.13' + - name: Restore proof tools and Miri sysroots + id: boundary-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.kani + ~/.cache/miri + ~/.rustup/toolchains/nightly-2025-11-21-x86_64-unknown-linux-gnu + ~/.rustup/toolchains/nightly-2026-08-07-x86_64-unknown-linux-gnu + .cache/boundary-kani + .cache/boundary-z3 + ~/.local/share/cuprum-verus-0.2026.09.06.8dea4a2 + ~/.rustup/toolchains/1.98.0-x86_64-unknown-linux-gnu + key: boundary-${{ matrix.check }}-${{ hashFiles('tools/kani/*', 'Makefile') }} + - name: Install prebuilt Kani frontend and bundle + if: matrix.check == 'kani' + run: | + make install-boundary-kani + echo "${GITHUB_WORKSPACE}/.cache/boundary-kani/bin" >> "$GITHUB_PATH" + - name: Install pinned Miri binary toolchain + if: matrix.check == 'miri' + run: rustup toolchain install nightly-2026-08-07 --profile minimal --component miri + - name: Execute extended verification + run: | + set -o pipefail + mkdir -p rust/target/boundary-verification + make boundary-${{ matrix.check }} 2>&1 | tee rust/target/boundary-verification/${{ matrix.check }}.log + - name: Exercise deliberate production faults + if: matrix.check == 'kani' + run: | + make install-verus + rustup toolchain install 1.98.0 --profile minimal + make boundary-faults + - name: Archive proof summaries and failures + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: extended-${{ matrix.check }} + path: rust/target/boundary-verification + retention-days: 14 + + - name: Save reviewed boundary cache generation + if: github.event_name == 'schedule' && github.ref == 'refs/heads/main' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.kani + ~/.cache/miri + ~/.rustup/toolchains/nightly-2025-11-21-x86_64-unknown-linux-gnu + ~/.rustup/toolchains/nightly-2026-08-07-x86_64-unknown-linux-gnu + .cache/boundary-kani + .cache/boundary-z3 + ~/.local/share/cuprum-verus-0.2026.09.06.8dea4a2 + ~/.rustup/toolchains/1.98.0-x86_64-unknown-linux-gnu + key: ${{ steps.boundary-cache.outputs.cache-primary-key }} diff --git a/Makefile b/Makefile index 01a9684c6..11bf6082d 100644 --- a/Makefile +++ b/Makefile @@ -60,6 +60,8 @@ TEST_CARGO_BUILD_JOBS ?= 1 PYTEST_WORKERS ?= 0 PYTEST_TARGETS ?= cuprum/unittests/test_*.py \ tests/test_ci_*.py \ + tests/test_native_sdist.py \ + scripts/tests/test_boundary_*.py \ tests/behaviour/test_[a-h]*.py \ tests/behaviour/test_[i-r]*.py \ tests/behaviour/test_[s-z]*.py @@ -336,3 +338,46 @@ benchmark-e2e: build uv ## Run hyperfine end-to-end throughput benchmark help: ## Show available targets @grep -E '^[a-zA-Z_-]+:.*?##' $(MAKEFILE_LIST) | \ awk 'BEGIN {FS=":"; printf "Available targets:\n"} {printf " %-20s %s\n", $$1, $$2}' + +# Boundary verifiers use their own pinned binary toolchains; normal gates keep +# rust/rust-toolchain.toml. Kani install is deliberately not source-built here. +PROVER_TOOLS_SOURCE ?= git+https://github.com/leynos/rust-prover-tools@98929b558253659a0a8ae03be7c49dafeef5f673 +PROVER_TOOLS = $(UV_RUN_ENV) uv tool run --python 3.14 --from $(PROVER_TOOLS_SOURCE) prover-tools +VERUS_INSTALL_DIR ?= $(HOME)/.local/share/cuprum-verus-0.2026.09.06.8dea4a2 +MIRI_TOOLCHAIN = nightly-2026-08-07 +KANI_VERSION = 0.67.0 +KANI_LIBRARY_PATH = $(HOME)/.kani/kani-$(KANI_VERSION)/toolchain/lib:$(HOME)/.kani/kani-$(KANI_VERSION)/lib + +.PHONY: install-verus boundary-verus boundary-kani boundary-miri boundary-test +install-verus: ## Install the checksum-verified prebuilt Verus release + $(PROVER_TOOLS) verus install --install-dir $(VERUS_INSTALL_DIR) + $(UV_RUN_ENV) uv run python scripts/install_boundary_z3.py + +boundary-verus: ## Verify actual production length/accounting kernels + $(UV_RUN_ENV) uv run python scripts/render_boundary_proofs.py + VERUS_Z3_PATH=$(CURDIR)/.cache/boundary-z3/z3 RUSTUP_TOOLCHAIN=1.98.0 $(VERUS_INSTALL_DIR)/verus/verus rust/target/boundary-verification/progress.rs --crate-type=lib + +boundary-kani: ## Check bounded native ownership and existing policy proofs + $(PROVER_TOOLS) kani check-version + cd $(RUST_DIR) && LD_LIBRARY_PATH="$(KANI_LIBRARY_PATH)" $(CARGO) kani --package cuprum-native-io + cd $(RUST_DIR) && LD_LIBRARY_PATH="$(KANI_LIBRARY_PATH)" $(CARGO) kani --package cuprum-streams + +boundary-miri: ## Interpret isolated native resource and memory paths + cd $(RUST_DIR) && $(CARGO) +$(MIRI_TOOLCHAIN) miri test --package cuprum-native-io --lib + +boundary-test: ## Run isolated native integration and verification-tool contracts + cd $(RUST_DIR) && $(CARGO) test --package cuprum-native-io --lib + $(UV_RUN_ENV) uv run pytest scripts/tests/test_boundary_*.py + +.PHONY: install-boundary-kani +install-boundary-kani: ## Install checksum-verified Kani binaries without a source build + $(UV_RUN_ENV) uv run python scripts/install_boundary_kani.py + PATH="$(CURDIR)/.cache/boundary-kani/bin:$(PATH)" $(CARGO) kani setup --use-local-bundle $(CURDIR)/.cache/boundary-kani/kani-$(KANI_VERSION)-x86_64-unknown-linux-gnu.tar.gz + +.PHONY: boundary-contract +boundary-contract: ## Confirm actual safe targets reject all forms of unsafe Rust + $(UV_RUN_ENV) uv run python scripts/check_boundary_contract.py + +.PHONY: boundary-faults +boundary-faults: ## Require proof and real-unwind regressions to detect deliberate faults + $(UV_RUN_ENV) uv run python -m scripts.check_boundary_faults diff --git a/cuprum/_pipeline_stream_fds.py b/cuprum/_pipeline_stream_fds.py index a6129f7c0..1a2a02619 100644 --- a/cuprum/_pipeline_stream_fds.py +++ b/cuprum/_pipeline_stream_fds.py @@ -87,17 +87,24 @@ class _ReaderPause: A transport with no ``pause_reading`` hook has no callbacks to race. One with a pause hook but no ``resume_reading`` hook is unsafe to hand off: it cannot be paused and later returned to asyncio, so it must fall back. + + ``closing_transport`` marks the one decline its caller may overrule. A + closing transport must not have its descriptor duplicated, but a caller + that supplies its own descriptors is never exposed to that hazard. """ may_hand_off: bool resume: cabc.Callable[[], None] | None decline_reason: RustPumpDeclineReason | None + closing_transport: bool def __init__( self, may_hand_off: object = None, resume: cabc.Callable[[], None] | None = None, decline_reason: RustPumpDeclineReason | None = None, + *, + closing_transport: bool = False, ) -> None: """Record a pause outcome, deriving its verdict from a decline reason.""" object.__setattr__( @@ -107,13 +114,33 @@ def __init__( ) object.__setattr__(self, "resume", resume) object.__setattr__(self, "decline_reason", decline_reason) + object.__setattr__(self, "closing_transport", closing_transport) def _pause_reader_transport( reader: asyncio.StreamReader, ) -> _ReaderPause: - """Pause reader callbacks while a duplicate-backed native worker pumps.""" + """Pause reader callbacks while a duplicate-backed native worker pumps. + + A closing transport is reported as ``closing_transport`` rather than + declined outright: only a caller that derives the worker descriptor from + this transport is exposed to the hazard, and only that caller may overrule + the decline. + + Returns + ------- + _ReaderPause + The hand-off verdict, and the resume hook when a pause was applied. + """ transport = _stream_transport(reader) + is_closing = getattr(transport, "is_closing", None) + if callable(is_closing) and is_closing(): + # asyncio silently ignores pause on a closing transport. Its queued + # connection_lost callback can still close the FD during hand-off. + return _ReaderPause( + decline_reason=RustPumpDeclineReason.READER_PAUSE_FAILED, + closing_transport=True, + ) pause_reading = getattr(transport, "pause_reading", None) resume_reading = getattr(transport, "resume_reading", None) if not callable(pause_reading): diff --git a/cuprum/_pipeline_streams.py b/cuprum/_pipeline_streams.py index c92e70d31..f7aee550f 100644 --- a/cuprum/_pipeline_streams.py +++ b/cuprum/_pipeline_streams.py @@ -35,6 +35,7 @@ from cuprum._pipeline_stream_fds import ( _extract_stream_fd, _pause_reader_transport, + _ReaderPause, _resume_reader_transport, _suppressed_teardown_failure, ) @@ -153,6 +154,28 @@ async def _run_rust_pump( return True +def _permit_test_owned_descriptor_handoff(reader_pause: _ReaderPause) -> _ReaderPause: + """Overrule a closing-reader decline when the test seam supplies descriptors. + + The dispatch seam lends descriptors it owns, so a reader whose transport is + already closing no longer invalidates this hand-off. A real hand-off derives + the worker descriptor from the transport's descriptor and keeps the decline. + + Returns + ------- + _ReaderPause + A permitted verdict for a closing reader under the seam, or + ``reader_pause`` unchanged. + """ + if ( + reader_pause.closing_transport + and _PUMP_STREAM_DISPATCH_TEST_HOOKS.raw_fd_extractor is not None + ): + # Nothing was paused, so the permitted verdict carries no resume. + return _ReaderPause(may_hand_off=True) + return reader_pause + + async def _pump_over_raw_fds( *, reader: asyncio.StreamReader, @@ -161,6 +184,7 @@ async def _pump_over_raw_fds( ) -> bool: """Transfer a hop after acquiring the reader and descriptor hand-off.""" reader_pause = _pause_reader_transport(reader) + reader_pause = _permit_test_owned_descriptor_handoff(reader_pause) if not reader_pause.may_hand_off: _log_rust_pump_declined( reader_pause.decline_reason or RustPumpDeclineReason.READER_PAUSE_FAILED, diff --git a/cuprum/_streams_rs.py b/cuprum/_streams_rs.py index 809cbde4b..d898bbc30 100644 --- a/cuprum/_streams_rs.py +++ b/cuprum/_streams_rs.py @@ -127,6 +127,26 @@ def _validate_buffer_size_before_writer_transfer(buffer_size: int) -> int: return size +def _prepare_native_reader(reader_fd: int) -> int: + """Validate ABI representation while Python still owns the writer.""" + # A negative value is rejected before the platform conversion: on Windows + # the conversion calls ``msvcrt.get_osfhandle``, which raises ``OSError`` + # for a negative CRT descriptor rather than the documented ``ValueError``. + raw_reader = operator.index(reader_fd) + if raw_reader < 0: + resource = "file handle" if os.name == "nt" else "file descriptor" + msg = f"{resource} must be non-negative" + raise ValueError(msg) + reader = operator.index(_convert_fd_for_platform(raw_reader)) + if not _I64_MIN <= reader <= _I64_MAX: + msg = "Python int too large to convert to C long" + raise OverflowError(msg) + if os.name != "nt" and not -(1 << 31) <= reader < (1 << 31): + msg = "file descriptor out of range" + raise ValueError(msg) + return reader + + def _prepare_rust_pump_call( *, reader_fd: int, @@ -141,7 +161,15 @@ def _prepare_rust_pump_call( _emit_rust_pump_handoff_outcome(RustPumpHandoffOutcome.NATIVE_LOAD_FAILED) raise try: - reader = _convert_fd_for_platform(reader_fd) + validated_buffer_size = _validate_buffer_size_before_writer_transfer( + buffer_size + ) + except BaseException: + _close_writer_after_pre_native_failure(writer_fd) + _emit_rust_pump_handoff_outcome(RustPumpHandoffOutcome.BUFFER_VALIDATION_FAILED) + raise + try: + reader = _prepare_native_reader(reader_fd) except BaseException as error: _pump_obs._log_native_pump_handoff_failed( _LOGGER, @@ -153,14 +181,6 @@ def _prepare_rust_pump_call( RustPumpHandoffOutcome.READER_PREPARATION_FAILED ) raise - try: - validated_buffer_size = _validate_buffer_size_before_writer_transfer( - buffer_size - ) - except BaseException: - _close_writer_after_pre_native_failure(writer_fd) - _emit_rust_pump_handoff_outcome(RustPumpHandoffOutcome.BUFFER_VALIDATION_FAILED) - raise return _PreparedRustPumpCall( native_pump=native_pump, reader=reader, @@ -279,15 +299,19 @@ def rust_pump_stream( ------ OSError If the Rust pump encounters an I/O error. + ValueError + If the wrapper rejects an argument before any native hand-off: + ``buffer_size`` that is not positive or exceeds 1 GiB, or a negative + ``reader_fd``. Notes ----- - The wrapper validates ``buffer_size`` before transferring a Windows writer - resource, preserving the native entry point's errors without leaking a - duplicated handle. Other failures propagate unchanged from the Rust - extension: ``ImportError`` if the native module cannot be imported and - ``OSError`` if an I/O error occurs while pumping bytes. - """ + The wrapper validates ``buffer_size`` and ``reader_fd`` before transferring + a Windows writer resource, preserving the native entry point's errors + without leaking a duplicated handle. Other failures propagate unchanged + from the Rust extension: ``ImportError`` if the native module cannot be + imported and ``OSError`` if an I/O error occurs while pumping bytes. + """ # ruff: ignore[docstring-extraneous-exception] - ValueError propagates from _prepare_rust_pump_call. prepared = _prepare_rust_pump_call( reader_fd=reader_fd, writer_fd=writer_fd, diff --git a/cuprum/unittests/test_concurrent_fail_fast.py b/cuprum/unittests/test_concurrent_fail_fast.py index c6c442252..5a2af9ebd 100644 --- a/cuprum/unittests/test_concurrent_fail_fast.py +++ b/cuprum/unittests/test_concurrent_fail_fast.py @@ -152,25 +152,18 @@ def test_fail_fast_mode_cancels_pending() -> None: catalogue, python_program = python_catalogue() python = sh.make(python_program, catalogue=catalogue) - # First command fails immediately, second sleeps + # Leave the second command pending until cancellation reaches it. cmd1 = python("-c", "import sys; sys.exit(42)") - cmd2 = python("-c", "import time; time.sleep(1); print('should not complete')") + cmd2 = python("-c", "import time; time.sleep(10); print('should not complete')") with scoped(ScopeConfig(allowlist=frozenset([python_program]))): - start = time.perf_counter() result = run_concurrent_sync( cmd1, cmd2, config=ConcurrentConfig(fail_fast=True) ) - elapsed = time.perf_counter() - start assert result.ok is False, "the failing command makes the run unsuccessful" - # The slow command should be cancelled, so elapsed time should be short - assert elapsed < 1.0, f"Expected < 1.0s with fail-fast, got {elapsed:.3f}s" - - # Verify shape of results and failures - # At minimum cmd1 completed (failed); cmd2 may or may not be in results - assert len(result.results) >= 1, ( - "At least the failed command should be in results" + assert result.submission_indices == (0,), ( + "the pending command must be cancelled before producing a result" ) assert result.failures == (0,), "First result should be the failure" assert result.first_failure is not None, ( diff --git a/cuprum/unittests/test_pipeline_streams_closing_reader.py b/cuprum/unittests/test_pipeline_streams_closing_reader.py new file mode 100644 index 000000000..c4a5e3d9b --- /dev/null +++ b/cuprum/unittests/test_pipeline_streams_closing_reader.py @@ -0,0 +1,81 @@ +"""A queued asyncio pipe close must prevent native descriptor borrowing.""" + +from __future__ import annotations + +import asyncio +import os +import typing as typ + +import pytest + +from cuprum._pipeline_stream_fds import _pause_reader_transport, _ReaderPause +from cuprum.pump_events import RustPumpDeclineReason + +if typ.TYPE_CHECKING: + import collections.abc as cabc + + +@pytest.fixture +def pipe_at_eof() -> cabc.Iterator[typ.BinaryIO]: + """Yield a real read pipe containing a prefix followed by kernel EOF.""" + reader_fd, writer_fd = os.pipe() + with ( + os.fdopen(reader_fd, "rb", buffering=0) as reader, + os.fdopen(writer_fd, "wb", buffering=0) as writer, + ): + writer.write(b"buffered prefix") + writer.close() + yield reader + + +class _ClosingReaderProtocol(asyncio.StreamReaderProtocol): + """Observe hand-off during real EOF notification, before queued close.""" + + def __init__( + self, + reader: asyncio.StreamReader, + outcome: asyncio.Future[_ReaderPause], + ) -> None: + """Retain the reader and a completion signal for the EOF observation.""" + super().__init__(reader) + self.reader = reader + self.outcome = outcome + + def eof_received(self) -> bool | None: + """Attempt the real pause while the transport is already closing.""" + self.outcome.set_result(_pause_reader_transport(self.reader)) + return super().eof_received() + + +@pytest.mark.skipif(os.name == "nt", reason="Unix read-pipe EOF callback contract") +def test_closing_reader_declines_native_borrow(pipe_at_eof: typ.BinaryIO) -> None: + """Queued close invalidates the FD, but Python retains the buffered prefix.""" + + async def exercise() -> None: + """Observe real EOF and complete the Python fallback without native I/O.""" + reader = asyncio.StreamReader() + outcome: asyncio.Future[_ReaderPause] = ( + asyncio.get_running_loop().create_future() + ) + protocol = _ClosingReaderProtocol(reader, outcome) + transport, _ = await asyncio.get_running_loop().connect_read_pipe( + lambda: protocol, pipe_at_eof + ) + try: + pause = await outcome + assert transport.is_closing(), "the actual transport must be closing" + assert pipe_at_eof.closed, "queued close invalidates the retained FD" + assert not pause.may_hand_off, "a closing descriptor cannot be borrowed" + assert pause.decline_reason is RustPumpDeclineReason.READER_PAUSE_FAILED, ( + "an ignored pause must use the existing pause-failure decline" + ) + assert pause.closing_transport, ( + "the pause must record the transport close that denied hand-off" + ) + assert await reader.read() == b"buffered prefix", ( + "the Python fallback must preserve bytes already read by asyncio" + ) + finally: + transport.close() + + asyncio.run(exercise()) diff --git a/cuprum/unittests/test_pump_stream_dispatch_selection.py b/cuprum/unittests/test_pump_stream_dispatch_selection.py index e4154854b..93d954f9b 100644 --- a/cuprum/unittests/test_pump_stream_dispatch_selection.py +++ b/cuprum/unittests/test_pump_stream_dispatch_selection.py @@ -7,11 +7,13 @@ from __future__ import annotations import asyncio +import os import typing as typ import pytest from cuprum import _pipeline_streams +from cuprum._pipeline_stream_fds import _pause_reader_transport, _ReaderPause from cuprum._testing import ( configure_pump_stream_dispatch_for_testing, set_rust_availability_for_testing, @@ -244,3 +246,93 @@ async def fake_pump( "the dispatch seam must retain the explicit benchmark read size, " f"got {observed}" ) + + +class TestTestOwnedDescriptorHandoff: + """A closing-reader decline yields only to the test-owned descriptor seam.""" + + @staticmethod + def _closing_pause() -> _ReaderPause: + """Return the decline a closing reader transport produces.""" + return _ReaderPause( + decline_reason=RustPumpDeclineReason.READER_PAUSE_FAILED, + closing_transport=True, + ) + + def test_closing_pause_stays_declined_without_a_test_extractor(self) -> None: + """A real hand-off keeps refusing a reader whose transport is closing.""" + assert ( + _pipeline_streams._PUMP_STREAM_DISPATCH_TEST_HOOKS.raw_fd_extractor is None + ), "this case describes the unstubbed descriptor supply" + closing = self._closing_pause() + + permitted = _pipeline_streams._permit_test_owned_descriptor_handoff(closing) + + assert permitted is closing, ( + "without a stubbed supply the decline must pass through untouched" + ) + assert not permitted.may_hand_off, "a closing reader cannot lend its FD" + assert permitted.decline_reason is RustPumpDeclineReason.READER_PAUSE_FAILED, ( + "the unstubbed decline must keep its pause-failure reason" + ) + + def test_closing_pause_is_permitted_for_a_test_owned_extractor(self) -> None: + """The seam lends its own descriptors, so the closing veto cannot apply.""" + configure_pump_stream_dispatch_for_testing( + raw_fd_extractor=lambda _stream: 7, + ) + + permitted = _pipeline_streams._permit_test_owned_descriptor_handoff( + self._closing_pause() + ) + + assert permitted.may_hand_off, "a stubbed supply must permit the hand-off" + assert permitted.decline_reason is None, ( + "a permitted hand-off must drop the decline reason" + ) + assert permitted.resume is None, ( + "nothing was paused, so the permitted verdict carries no resume" + ) + + @pytest.mark.skipif(os.name == "nt", reason="Unix read-pipe close contract") + def test_real_closing_transport_is_permitted_for_a_test_extractor(self) -> None: + """The real pause producer, not just a hand-built one, reaches the seam. + + The cases above construct ``_ReaderPause`` themselves, so a change to + the transport inspection that produces it would leave them green. This + case closes an actual transport and gives its pause to the seam. + """ + configure_pump_stream_dispatch_for_testing( + raw_fd_extractor=lambda _stream: 7, + ) + + async def exercise() -> _ReaderPause: + """Close a live read-pipe transport and take the pause it yields.""" + reader_fd, writer_fd = os.pipe() + try: + with os.fdopen(reader_fd, "rb", buffering=0) as read_pipe: + reader = asyncio.StreamReader() + protocol = asyncio.StreamReaderProtocol(reader) + transport, _ = await asyncio.get_running_loop().connect_read_pipe( + lambda: protocol, read_pipe + ) + transport.close() + return _pause_reader_transport(reader) + finally: + os.close(writer_fd) + + pause = asyncio.run(exercise()) + + assert pause.closing_transport, ( + "the real producer must mark the close the seam may overrule" + ) + assert not pause.may_hand_off, "a closing transport must first decline" + + permitted = _pipeline_streams._permit_test_owned_descriptor_handoff(pause) + + assert permitted.may_hand_off, ( + "the stubbed descriptor supply must overrule the closing decline" + ) + assert permitted.decline_reason is None, ( + "a permitted hand-off must drop the decline reason" + ) diff --git a/cuprum/unittests/test_stream_parity.py b/cuprum/unittests/test_stream_parity.py index f07967ede..956767fd5 100644 --- a/cuprum/unittests/test_stream_parity.py +++ b/cuprum/unittests/test_stream_parity.py @@ -217,7 +217,13 @@ def test_backpressure_large_payload( pipeline, allowlist = _build_pipeline(script, stages=stages) result = run_parity_pipeline(pipeline, allowlist) - assert result.stdout == "x" * size, "payload should survive pipeline intact" + # Report byte counts without constructing a huge repetitive-string diff. + matches = result.stdout == "x" * size + assert matches, ( + f"payload should survive pipeline intact: expected={size}, " + f"actual={len(result.stdout or '')}, " + f"stage_exits={[stage.exit_code for stage in result.stages or ()]}" + ) assert result.ok is True, "pipeline should succeed" assert len(result.stages) == stages, ( f"pipeline should have exactly {stages} stages" diff --git a/cuprum/unittests/test_streams_rs_reader_validation.py b/cuprum/unittests/test_streams_rs_reader_validation.py new file mode 100644 index 000000000..05cad473d --- /dev/null +++ b/cuprum/unittests/test_streams_rs_reader_validation.py @@ -0,0 +1,81 @@ +"""Invalid native reader arguments must not strand an owned writer duplicate.""" + +from __future__ import annotations + +import contextlib +import os +from types import SimpleNamespace +from unittest import mock + +import pytest + +from cuprum import _streams_rs + + +def _close_duplicate(fd: int) -> None: + """Close a duplicate the native pump may already have consumed.""" + with contextlib.suppress(OSError): + os.close(fd) + + +@pytest.mark.parametrize( + ("reader", "error"), + [ + (-1, ValueError), + (1 << 64, OverflowError), + pytest.param( + 1 << 31, + ValueError, + marks=pytest.mark.skipif( + os.name == "nt", reason="Windows handles are pointer-sized" + ), + ), + ], +) +def test_invalid_reader_closes_duplicate_before_native_call( + monkeypatch: pytest.MonkeyPatch, + reader: int, + error: type[Exception], +) -> None: + """ABI validation fails before transferring the only writer owner.""" + with contextlib.ExitStack() as stack: + source, sink = os.pipe() + stack.callback(os.close, source) + stack.callback(os.close, sink) + duplicate = os.dup(sink) + stack.callback(_close_duplicate, duplicate) + native = mock.Mock(return_value=0) + monkeypatch.setattr( + _streams_rs, + "_load_native", + lambda: SimpleNamespace(rust_pump_stream=native), + ) + monkeypatch.setattr(_streams_rs, "_convert_fd_for_platform", lambda fd: fd) + with pytest.raises(error): + _streams_rs.rust_pump_stream(reader, duplicate) + native.assert_not_called() + with pytest.raises(OSError, match="Bad file descriptor"): + os.fstat(duplicate) + os.fstat(sink) + + +def test_windows_negative_reader_is_rejected_before_conversion( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A negative CRT descriptor must not reach the platform conversion.""" + converted: list[int] = [] + + def convert_reader(fd: int) -> int: + """Record the descriptor and model the real conversion's refusal.""" + converted.append(fd) + if fd < 0: + msg = "get_osfhandle refuses a negative descriptor" + raise OSError(msg) + return fd + + monkeypatch.setattr(_streams_rs.os, "name", "nt") + monkeypatch.setattr(_streams_rs, "_convert_fd_for_platform", convert_reader) + with pytest.raises(ValueError, match="file handle must be non-negative"): + _streams_rs._prepare_native_reader(-1) + + assert not converted, "conversion must not run for a negative descriptor" diff --git a/docs/adr-011-audited-rust-boundaries.md b/docs/adr-011-audited-rust-boundaries.md new file mode 100644 index 000000000..c008de89e --- /dev/null +++ b/docs/adr-011-audited-rust-boundaries.md @@ -0,0 +1,191 @@ +# Architectural decision record (ADR) 011: Audited Rust safety boundaries + +## Status + +Accepted on 2026-09-08. Cuprum confines native descriptor, handle, and syscall +obligations to the audited `cuprum-native-io` and `cuprum-rust` boundaries and +keeps `cuprum-streams` free of unsafe Rust. + +## Date + +2026-09-08. + +## Context and problem statement + +The original `cuprum-rust` crate combined abstract stream policy with raw +descriptor construction, native reads and writes, Linux `splice`, and an +unchecked UTF-8 conversion. That made the requested `unsafe_code = "forbid"` +policy incompatible with the whole crate, even though the stream state machine +did not require unsafe Rust. Integer range checks at the PyO3 boundary also +could not establish that a descriptor or handle was live, uniquely owned, or +protected from concurrent close and reuse. + +The Python pipeline has a corresponding cross-language contract. It pauses the +reader transport, changes both descriptors to blocking mode, duplicates the +writer, submits native work, and waits for native cleanup after cancellation. +That sequence establishes the validity and lifetime assumptions for the native +worker and must remain observable in Python regression tests. + +The hand-off also rejects a transport already closing before borrowing its raw +reader descriptor. Asyncio can silently accept `pause_reading()` in that state +while a queued `connection_lost` callback closes the descriptor, so the hop +declines with `READER_PAUSE_FAILED` and the Python fall-back retains its +buffered prefix. The H9 real-EOF regression covers this contract; it does not +establish the cause of the historical native payload mismatch. + +## Decision + +Split the Rust workspace by responsibility: + +- `cuprum-streams` contains safe orchestration, pump state transitions, error + classification, Linux fallback policy, and checked UTF-8 replacement. Its + crate root and compile-contract tests forbid unsafe Rust. +- `cuprum-native-io` contains lifetime-bound `AsStream` borrows, unique owned + stream adoption, single read/write/splice calls, checked progress kernels, + and Unix/Windows native test fixtures. `borrow` is safe and returns a + lifetime-bound standard-library borrow. `adopt_writer` and `borrow_reader` + are the only raw-resource constructors and are unsafe APIs with documented + caller obligations. The crate denies unsafe operations inside unsafe + functions. +- `cuprum-rust` stays the thin `cdylib` and PyO3/maturin integration crate. It + validates Python arguments, releases the Global Interpreter Lock (GIL), + converts errors, and is the only caller of the unsafe native constructors. + PyO3-generated ABI code remains an audited integration trust boundary. + +The writer contract is implemented through the generic +`cuprum_native_io::with_owned_writer`: the reader is borrowed, the transferred +writer is owned by the scope, and Rust drop elaboration closes it on normal, +error, and real unwind paths. Its callback receives an immutable reference, so +callback code cannot replace or move the owned writer out of the scope. The +normal path explicitly drops the writer after the callback; unwind relies on +automatic RAII. The safe policy crate never receives raw integers. The +avoidable unchecked UTF-8 operation was replaced by checked conversion. + +The approved unsafe-bearing crates are limited to `cuprum-native-io` and the +raw PyO3 integration in `cuprum-rust`. Workspace membership is checked by +`scripts/check_boundary_contract.py`; adding another member requires an +explicit inventory decision. The same script compiles deliberate unsafe probes +against a copy of `cuprum-streams` library and compile-test targets, proving +the safe crate's rejection without modifying source files. + +## Verification decision + +Verify pure, typed progress contracts with Verus first. The proof input is +generated from the exact executable bodies in +`rust/cuprum-native-io/src/progress.rs` by `scripts/render_boundary_proofs.py`; +the correspondence test detects drift. Use Kani for production ownership/drop +and policy obligations where Verus does not practically support +operating-system handles, user-defined destructors, raw pointers, or unwind. +Keep native tests authoritative for kernel resource effects and real panic +unwind. Use Miri on applicable isolated native targets and memory paths, +excluding PyO3 and unsupported operating-system operations with an explicit +record. + +The progress Verus run used Verus `0.2026.09.06.8dea4a2`, Rust `1.98.0`, and +prebuilt Z3 `4.16.0`; it verified two functions with zero errors. A direct +assessment of the unchanged production `adopt_writer` body failed because +`OwnedFd` and `FromRawFd::from_raw_fd` are unsupported representations in this +Verus version (`/tmp/issue379-verus-resource-assessment.log`). A separate +generic memory assessment lacks callback preconditions; that incompleteness is +not the sole justification for the Kani fallback. The Verus archive omits Z3, +so the separate prebuilt solver installation is required. The +`leynos/rust-prover-tools` installation succeeds, but its wrapper's version +parser rejects the annotation +`(overridden by environment variable RUSTUP_TOOLCHAIN)`; the local command +therefore invokes the pinned Verus binary directly. + +Kani `0.67.0` with compiler `rustc 1.93.0-nightly (53732d5e0 2025-11-20)` and +CBMC `6.8.0` passed 7 native harnesses with 0 failures in the final native run; +see `/tmp/issue379-round27-kani-native.log`. The 11 safe-policy/decoder +harnesses remain current and passed in the complete run recorded at +`/tmp/issue379-round24-kani.log`. The native ownership bounds enumerate +`Normal` and `Error` exits, and the repeated-borrow harness limits borrows to +three with `#[kani::unwind(4)]`; the remaining symbolic and fixed domains are +recorded in the verification matrix. These are bounded results, not an +unbounded proof. Miri `nightly-2026-08-07` uses +`rustc 1.99.0-nightly (84b36a78a 2026-08-06)` and passed all 13 isolated native +tests with zero ignored in `/tmp/issue379-round27-miri.log`. PyO3, unshimmed +`libc::splice`, and unsupported operating-system representations remain +excluded. Round 27's fault run passed its controls and detected all four +mutations in `/tmp/issue379-round27-faults.log`; the real native unwind test +remains authoritative for OS effects. + +Round 7's focused Python ABI regression first failed before reader +representability validation moved ahead of writer transfer, then passed all 14 +focused tests (`/tmp/issue379-reader-validation-before.log` and +`/tmp/issue379-round7-focused-tests.log`). The validation proves only integer +representability and non-negativity; it does not establish resource validity, +lifetime, or exclusive ownership. + +Nine safe-crate unsafe-rejection probes, including a future target without a +crate-level attribute, are covered by the contract gate. The normal callback +compile regression passed with the expected `&W` versus `&mut W` diagnostic in +`/tmp/issue379-round25-trybuild-normal.log`; the final fault run passed its +controls and detected all four mutations in `/tmp/issue379-round27-faults.log`. +Hosted Windows and macOS runtime execution passed on 2026-09-15: the "Rust +boundary verification" run 35004943625 succeeded with the +`Native contracts (windows-2022)`, `Native contracts (macos-latest)`, and +`Native contracts (ubuntu-latest)` jobs alongside `verus`; the "CI" run +35004943782 succeeded with the +`Extension-gated tests (Windows Python/Rust boundary)` and +`Extension-gated tests (Python/Rust boundary)` jobs. The `extended` Kani/Miri +job is skipped on pull requests by design, so these hosted runs add no verifier +coverage beyond the bounded local rounds recorded above; the scheduled Loom +interleaving harness remains a separate pending deliverable. Packaging coverage +found that `uv build --sdist` omitted the Rust workspace and that +`uv run maturin sdist --manifest-path rust/cuprum-rust/Cargo.toml` omitted +`rust/rust-toolchain.toml`; the red evidence is in +`/tmp/issue379-native-sdist-before2.log`. Explicit archive entries and the +parametrized `tests/test_native_sdist.py` regression now cover the native +workspace and reject `target` output. Round 24 ran both cases successfully in +26.25 seconds (`2 passed`), and the extracted maturin source archive built a +CPython 3.13 native wheel with all three crates compiling; see +`/tmp/issue379-round24-extracted-wheel.log`. This is a source-archive +completeness correction; the optional pure-wheel backend is unchanged. + +## Consequences + +- Safe stream functionality can enforce `unsafe_code = "forbid"` without + weakening native support or hiding unsafe code in an internal module. +- Raw-resource obligations have small, named APIs and one audited caller. +- Existing Python behaviour, optional backend selection, maturin packaging, + and supported platform paths remain unchanged. +- Native integration tests remain necessary for descriptor/handle lifetime, + EOF delivery, short and interrupted I/O, broken pipes, and cleanup after + failure or cancellation. +- Verus, Kani, and Miri have different scopes. Tool limitations, trusted + syscall and interpreter assumptions, bounds, and excluded targets must stay + visible in + [Rust boundary verification and unsafe inventory](rust-boundary-verification.md). + +## Supersession and related decisions + +This decision refines [ADR-001](adr-001-rust-extension.md) and +[ADR-002](adr-002-additional-rust-components.md) by preserving the optional +PyO3 extension while separating its safe policy and native resource boundaries. +The historical `fd_ownership_model.rs` remains useful provenance for the +earlier bounded model, but it is not the current ownership implementation and +does not prove actual operating-system close or real panic unwind. Existing +verification context in issues [#80], [#81], [#84], [#89], [#125], and [#233] +remains in force. Scheduled Loom interleaving work is a separate deliverable. + +[#80]: https://github.com/leynos/cuprum/issues/80 +[#81]: https://github.com/leynos/cuprum/issues/81 +[#84]: https://github.com/leynos/cuprum/issues/84 +[#89]: https://github.com/leynos/cuprum/issues/89 +[#125]: https://github.com/leynos/cuprum/issues/125 +[#233]: https://github.com/leynos/cuprum/issues/233 + +The safe crate's descriptor fixtures use `cap_std::fs::File` constructed from +owned descriptors. The Windows native adapter uses the same capability file +wrapper for borrowed-handle I/O, with the production retention kernel +suppressing its owner drop. This avoids carrying the former extension-wide +`std::fs` lint exclusion into either extracted crate; `cap-std` and its handle +adapters are trusted dependencies, not verifier-proved implementations. + +The safe crate also declares `unsafe_code = "forbid"` in Cargo's lint table, so +future automatically discovered test, example, and benchmark targets inherit +the prohibition. Cargo cannot override an individual workspace-inherited lint; +the safe manifest therefore carries the workspace lint tables plus this one +addition. The boundary-contract gate requires exact policy parity, preventing +that required duplication from weakening other lints or drifting silently. diff --git a/docs/ci-cache-ownership.md b/docs/ci-cache-ownership.md index 38b0d034b..eec55491a 100644 --- a/docs/ci-cache-ownership.md +++ b/docs/ci-cache-ownership.md @@ -27,6 +27,29 @@ Every key also carries a generation, the operating system, and the architecture. `target` archive would preserve, and a `target` archive is invalidated by every source change while the objects inside it are not. +### Boundary verification cache family + +The Rust boundary workflow is a scoped member of the scheduled cache family. It +restores the shared Cargo registry through `.github/actions/cache-keys`; it +does not create a second registry owner. Linux native and Verus jobs use the +existing `setup-sccache` action with a 1 GiB cap and record compiler-cache +statistics. Verus compiler objects share that boundary-private sccache cache; +the Verus, Kani, Z3, and Miri binaries or sysroots are cached separately. No +job archives a Cargo `target` tree. + +The native matrix uses the pinned `windows-2022` image alongside the supported +Linux and macOS images. The Kani and Miri jobs are scheduled or manually +dispatched rather than added to pull requests, run serially, and use no +ordinary target cache. Their proof, failure, and fault logs are uploaded as +artefacts with 14-day retention. Only scheduled runs on `refs/heads/main` save +boundary-private generations; pull requests and manual dispatches are read-only. + +`tests/test_ci_rust_boundaries.py` checks the hosted matrix, time and fan-out +budgets, cache paths, and the absence of target archives. The runner manifest +registers `verus` and `extended` in `GITHUB_HOSTED_JOBS`, and the native matrix +has an explicit shared-registry test entry. These checks preserve the runner +placement and cache ownership contract when the workflow changes. + ## Why the lane is in every key `runner.environment` renders to `self-hosted` on Ubicloud and `github-hosted` diff --git a/docs/contents.md b/docs/contents.md index 65e7c4239..8c6e8e7a6 100644 --- a/docs/contents.md +++ b/docs/contents.md @@ -52,6 +52,8 @@ open each document. govern identifiers and source prose with the repository spelling gate. - [ADR-010: Rust-pump executor-hop spans][adr-010] - accepted decision to add opt-in executor-hop tracing without extending the pump event channel. +- [ADR-011: Audited Rust safety boundaries][adr-011] - accepted decision to + isolate native resource operations from safe stream policy. ## Planning and validation references @@ -67,6 +69,9 @@ open each document. - [Tee hot-path read-size sweep (2026-08-29)][tee-read-size-sweep] - the interleaved measurement that selected the Python stream read size for roadmap item 5.1.1. +- [Rust boundary verification and unsafe inventory][rust-boundary-verification] + - unsafe inventory, crate contracts, verifier evidence, and trusted + assumptions for the native stream boundaries. [adr-003]: adr-003-two-tier-python-linting.md [adr-004]: adr-004-interrogate-docstring-gate.md @@ -76,7 +81,9 @@ open each document. [adr-008]: adr-008-rust-pump-observation-channel.md [adr-009]: adr-009-enforce-oxford-spelling-in-source.md [adr-010]: adr-010-rust-pump-hop-span.md +[adr-011]: adr-011-audited-rust-boundaries.md [local-validation]: local-validation-of-github-actions-with-act-and-pytest.md [migration-020]: migration-0.2.0.md +[rust-boundary-verification]: rust-boundary-verification.md [tee-baseline]: tee-hotpath-profiling-baseline-2026-06-12.md [tee-read-size-sweep]: tee-hotpath-read-size-sweep-2026-08-29.md diff --git a/docs/cuprum-design.md b/docs/cuprum-design.md index f6e2e52b9..108ed0697 100644 --- a/docs/cuprum-design.md +++ b/docs/cuprum-design.md @@ -2152,7 +2152,10 @@ over cleverness—especially in the type‑system and configuration layers. [ADR-009](adr-009-enforce-oxford-spelling-in-source.md) governs en-GB-oxendict spelling for identifiers and source prose; the repository spelling gate -enforces that policy. +enforces that policy. [ADR-011](adr-011-audited-rust-boundaries.md) governs the +separation between safe stream policy and the audited native resource boundary. +The inventory and verification status are maintained in +[Rust boundary verification](rust-boundary-verification.md). ______________________________________________________________________ @@ -2271,6 +2274,16 @@ and performs any platform-specific file descriptor conversion (for example, translating Windows file descriptors into OS handles). This keeps the native module name stable while allowing Python-only adaptations. +The Rust implementation is split internally by safety responsibility: +`cuprum-streams` contains safe orchestration, state-machine policy, fallback +selection, and checked UTF-8 decoding; `cuprum-native-io` contains lifetime +bound operating-system borrows, unique writer adoption, single-call native I/O, +and platform fixtures; and `cuprum-rust` remains the thin PyO3/maturin +integration boundary. Only the latter two approved boundaries contain unsafe +Rust, and only `cuprum-rust` reconstructs resources from Python raw values. +This preserves the public module and maturin path while allowing +`cuprum-streams` to forbid unsafe code. + The availability probe is separate from the stream shim. The Python module `cuprum._rust_backend` exposes the raw `is_available()` probe, which imports `cuprum._rust_backend_native`, returns `False` when that native module is @@ -2910,7 +2923,7 @@ contract. The non-splice read/write loop keeps its control flow — how each read and write outcome moves the running byte total and the latched writer state — in a pure, -`io::Error`-free state machine in `rust/cuprum-rust/src/pump_machine.rs`, +`io::Error`-free state machine in `rust/cuprum-streams/src/pump_machine.rs`, separate from the descriptor I/O that feeds it. `advance` is the whole public surface. It takes a `PumpState`, the length just diff --git a/docs/debugging/debugging-plan-20260908-issue379-parity.md b/docs/debugging/debugging-plan-20260908-issue379-parity.md new file mode 100644 index 000000000..7e8ea5945 --- /dev/null +++ b/docs/debugging/debugging-plan-20260908-issue379-parity.md @@ -0,0 +1,357 @@ +# Debugging plan: issue 379 validation failures + +**Generated:** 2026-09-08 **Issue:** #379 **Severity:** high (possible +captured-output loss) **Falsification sub-agent:** `alchemist` + +The planning agent prepared this document. The alchemist executes the bounded +experiment; it must not run full repository gates or edit production code. + +## Current state + +Round 35 integrated local checks are green: Python recorded 1,568 passes and +one skip in `/tmp/issue379-round35-test.log`, Rust recorded 115/115 tests with +no skips, and the extension-required suite recorded 80 passes and one skip. The +nine compiler probes, Windows cross-Clippy, and development build also passed. +The cached Kani setup had already passed in Round 33. The formal and native +evidence remains 2 Verus functions, 7 native plus 11 safe Kani harnesses, 13 +Miri tests, and 4 detected fault mutations. The Round 33 checkpoint remains +retained in the history. Hosted Windows/macOS runtime checks and the second +CodeRabbit review remain pending; the first review passed with 0 findings. + +Round 35's `fmt`, `check-fmt`, `lint`, `typecheck`, `test`, `markdownlint`, and +`nixie` gates all passed. + +The historical native payload mismatch remains unexplained and is not claimed +fixed. H7 reproduced an empty `HELLO` result on the archived baseline, while H9 +identified and fixed a separate closing-transport reader-lease gap. Baseline +comparison and the bounded investigations below retain their original scope. + +## Problem and context + +An earlier full native-enabled Python suite reported 1,565 passes, one skip, +and two failures in `/tmp/issue379-round28-test.log`. The Rust two-stage +backpressure case returned something other than its expected 1 MiB ASCII +payload; the three-stage and Python cases passed. A separate fail-fast test +took 1.089 seconds against a one-second assertion. This shared host was heavily +loaded. The exact captured byte count and EOF-grace observations are not in the +failure report. Cuprum's Rust boundary extraction and Python pre-transfer +validation were uncommitted at the time; the starting revision is +`34a59eac4cd13de82c3730d4b569eeaefbb5ee61`. + +## H1: capture EOF-grace expiry truncates the parity output + +The Python capture consumer can miss its fixed EOF-grace deadline under load, +causing cancellation before all already-produced output is captured. This is +independent of a native descriptor-transfer error. + +Prediction: a reproduced truncation coincides with a capture EOF-grace expiry +observation. The same mechanism can occur with the starting revision's backend. +A truncated run with no expiry observation falsifies this proposed explanation +for that run. No reproduction is inconclusive, not a pass or falsification. + +### Minimal experiment + +1. Use a scratch runner under `.cache/issue379-parity-experiment`; do not edit + tracked files. Import the existing parity pipeline builder and run helper. + Force the Rust backend in a fresh interpreter and record the imported Python + package and native extension paths. Record their file hashes. +2. Run only the existing two-stage, 1 MiB ASCII pipeline, at most 20 times. + Capture the existing EOF-grace reporting hook/log events without changing + its scheduling, deadline, or return value. Report each captured byte count, + equality to the expected payload, stage exit codes, and expiry observations. + Bound each pipeline to 30 seconds. Stop on the first reproduced truncation. +3. If truncation has no expiry, stop: H1 is falsified for that execution. + If no truncation occurs, stop and report inconclusive. +4. Only if truncation coincides with expiry, build the starting revision's + native wheel from a `git archive` source copy under `.cache`, with shared + Cargo registry and build output under the original `rust/target`. Do not + install it into the working environment. Extract it into a scratch import + root, then run the identical bounded experiment in a fresh interpreter. + Record provenance and whether the baseline reproduces the same mechanism. + +Save stdout, stderr, and observations to `/tmp/issue379-parity-h1.log`. Return +falsified, not-falsified, or inconclusive, with exact evidence. Do not add +load, change timeouts in production, broaden the experiment, or classify an +intermittent non-reproduction as proof of correctness. + +## H2: the fail-fast assertion measures host scheduling + +The one-second wall-clock assertion includes child startup and scheduling, +although the property under test is cancellation of a pending command. This +hypothesis is not authorized for execution in the H1 packet. A subsequent +bounded experiment must distinguish successful cancellation from waiting for +the slow command, using completion evidence rather than relaxing the timer. + +## Termination and next action + +Stop after the H1 packet and report to the planning agent. The planner must +revise the hypothesis or authorize another specific experiment before further +investigation. Production fixes require regression evidence and all commit +gates. The first failed full run must remain recorded. + +## H1 result + +The alchemist's 20 fresh-interpreter trials all returned 1,048,576 matching +bytes, stage exits `[0, 0]`, and no EOF-grace expiry. Evidence: +`/tmp/issue379-parity-h1.log`. H1 is inconclusive; this does not explain the +full-suite mismatch. The imported native extension hash was +`e04a682ef69e8aa17bab98d4f2b6fb9dc7a29e31afa4d4be58ca2dd274d70850`. + +## H2 authorized experiment + +The same alchemist may now execute exactly two real concurrent runs in a +scratch script, using the existing catalogue and `run_concurrent_sync` API. The +first child waits 1.1 seconds before exiting 42; the second waits 10 seconds +before printing a completion marker. The delay deliberately represents latency +before the first failure becomes observable; it is not a claim about the cause +of the historical timing measurement. + +Run once with fail-fast enabled, then once disabled. Bound each run to 20 +seconds. Record elapsed time, submission indices, failure indices, exit codes, +and completion-marker presence. Prediction: enabled cancellation excludes the +slow result even though total elapsed time exceeds one second; disabled mode +includes the slow completion. If the enabled run includes the slow completion, +this counterexample to the timer's validity is not established. If the two +modes differ as predicted, the one-second assertion is not a valid standalone +cancellation criterion. Save `/tmp/issue379-fail-fast-h2.log` and stop. Do not +edit tracked code, change production deadlines, or run other tests. + +## H2 result + +Fail-fast returned only submission zero after 1.390 seconds, with exit 42 and +no slow completion marker. Collect-all returned both submissions after 10.270 +seconds, including the slow command's successful marker. Evidence: +`/tmp/issue379-fail-fast-h2.log`. The hypothesis is not falsified. The +regression now checks exclusion of the pending submission directly, with a +ten-second child delay; it no longer equates startup latency with failed +cancellation. This does not attribute the original run's delay to a particular +cause. + +## H3: the pre-buffer hand-off loses bytes before native pumping + +The reader's buffered prefix can change across the awaited writer drain, or the +writer can retain pending bytes when native pumping begins. This could produce +a mismatch even when native reads and writes conserve their own bytes. + +Prediction: a failing run has a prefix/native/capture accounting discrepancy at +this seam. Equal accounting on a reproduced mismatch falsifies this +explanation. No reproduction remains inconclusive. + +The alchemist may run at most 40 trials in one interpreter, alternating the +existing two-stage and three-stage 1 MiB pipeline with the Rust backend +explicitly selected. Use scratch wrappers around the actual +`_drain_reader_buffer` and `rust_pump_stream`: record reader buffer lengths +before and after the original drain, writer transport pending bytes after +drain, native returned byte counts, captured length and stage exits. Do not +alter return values, add scheduling delays, or replace the actual operations. +Confirm the native wrapper was called; an unused imported extension is not +evidence of native execution. Bound each pipeline to 30 seconds, stop at the +first mismatch, and save `/tmp/issue379-parity-h3.log`. Record file provenance. +Do not run full gates or edit tracked code. Stop and report the verdict. + +## H3 result and H4 authorization + +All 40 trials matched, with 60 confirmed native pump calls. Every pre-buffer +and pending writer buffer was empty. Evidence: `/tmp/issue379-parity-h3.log`. +H3 is inconclusive and did not exercise its proposed buffered-prefix condition. + +H4 narrows the hypothesis: a non-empty asyncio prefix is mishandled while the +native pump independently transfers the remainder. The alchemist may use the +same scratch instrumentation for at most eight two-stage trials, delaying the +second actual subprocess creation by 0.1 seconds with an awaited sleep. This is +an explicit scheduling intervention to allow the first process's reader buffer +to fill before hand-off, not artificial host load. All subprocess and native +operations remain real. Use the existing 1 MiB payload and 30-second per-trial +limit. Confirm a non-zero prefix; otherwise report inconclusive. Record prefix +length, remaining prefix, writer pending bytes, native count, captured length +and exit statuses. Stop on first mismatch. A reproduced mismatch with lost +prefix bytes supports H4; preserved non-zero prefixes falsify the proposed loss +for the exercised schedules only. Save `/tmp/issue379-parity-h4.log`; no +tracked edits or full gates. + +## H4 result and gate resumption + +All eight delayed-start trials preserved the full payload. Seven prefixes +contained 196,608 bytes and one contained 139,264 bytes. Each prefix plus its +native return count equalled 1,048,576, with zero pending writer bytes and +stage exits `[0, 0]`. Evidence: `/tmp/issue379-parity-h4.log`. Prefix loss is +falsified for these exercised schedules; the historical mismatch remains +unexplained. No native implementation change follows from these experiments. + +The full gate sequence may resume with the improved parity failure diagnostic +and direct fail-fast cancellation assertion. A repeated mismatch must be +investigated using its new byte counts and stage exits; the earlier failure +must not be erased by a later passing run. + +## Reproduction in round 29 and H5 authorization + +Round 29 passed all 1,567 unit cases (one skip), including all four large +payload parity cases. Its behavioural stream parity case then failed with a +payload mismatch and no pipeline failure index. That scenario has three stages. +The behavioural assertion now also records lengths and stage exits. The +discrepancy is recurring and gates are paused again. + +H5 tests whether pytest's surrounding scenario sequence exposes the missing +schedule: a mismatch should reproduce under the real behavioural module with +passive hand-off accounting, even though scratch pipelines passed. The +alchemist may run only `tests/behaviour/test_stream_parity_behaviour.py`, at +most ten fresh pytest invocations, stopping on the first mismatch. Use an +ignored scratch pytest plugin to wrap the original drain and native pump and +record the same prefix/pending/native counts as H3, associated with each node +ID. Preserve fixtures, pytest capture and return values. Each invocation has a +60-second external bound; do not add delays or host load. Record native module +provenance and actual calls. Save `/tmp/issue379-parity-h5.log` and stop; no +tracked edits or full gates. Non-reproduction remains inconclusive. + +## H5 result and H6 authorization + +Nine instrumented module runs passed all eight cases each. Every three-stage +native transfer accounted for 1 MiB per hop, including one 8 KiB prefix. One +earlier scratch plugin registration error consumed the remaining invocation +budget without collecting tests. Evidence: `/tmp/issue379-parity-h5.log`. H5 is +inconclusive. + +H6 tests surrounding-module state or instrumentation sensitivity: the exact +26-case `tests/behaviour/test_[s-z]*.py` group from the failed Makefile batch +may reproduce where the instrumented parity module does not. The alchemist may +run this group without its scratch plugin at most ten times in fresh pytest +interpreters, using the Makefile's serial `-v -n 0` flags. Each run has a +60-second external bound. Stop on the first mismatch, retaining the new +length/status assertion and full captured failure output in +`/tmp/issue379-parity-h6.log`. Do not invoke the full repository gate, add +load, change environment beyond the Makefile's existing test settings, or edit +tracked files. A reproduction supports this narrower context dependence; +non-reproduction is inconclusive and does not establish correctness. + +## H6 result and diagnostic rerun + +The fifth exact batch reproduced a Rust UTF-8 mismatch, after four passing +batches. Pytest then exceeded its 30-second timeout inside `difflib` while +rendering the repetitive-string comparison, obscuring the length diagnostic. +Evidence: `/tmp/issue379-parity-h6.log`. H6 is not falsified, but this result +does not identify a cause or the first differing byte. + +The existing assertions now compare into a boolean before asserting, retaining +exact equality while preventing automatic large-string diff rendering. The +alchemist may repeat the H6 packet for at most ten fresh batches, stopping at +the first failure, with the same 60-second bound per batch and no plugin. Save +`/tmp/issue379-parity-h6-diagnostic.log`. This is a diagnostic refinement of +H6, not a new production hypothesis or a relaxation of the payload contract. + +## Diagnostic rerun result and bounded sampling extension + +All ten diagnostic batches passed (260 cases), leaving the failure intermittent +and unexplained. Evidence: `/tmp/issue379-parity-h6-diagnostic.log`. Because +the earlier reproduction lacked usable data, the alchemist may extend this same +unchanged diagnostic experiment by at most 30 fresh batches, stopping at the +first failure. Retain the same per-batch bound and settings, no plugin, no +artificial load, and no tracked edits. Save +`/tmp/issue379-parity-h6-extended.log`. This extension seeks failure evidence; +passing repetitions cannot be used as evidence that the defect was fixed. + +## Extended result and H7 baseline control + +All 30 additional batches passed (780 cases). Evidence: +`/tmp/issue379-parity-h6-extended.log`. The current tree still has no proposed +production fix for the observed mismatch. + +H7 tests whether the same mismatch predates extraction. The alchemist may +export starting revision `34a59eac4cd13de82c3730d4b569eeaefbb5ee61` into an +ignored `.cache/issue379-parity-baseline` source directory, build its native +extension with the normal pinned toolchain and shared Cargo registry, and place +build output under the original `rust/target/parity-baseline`. Do not install +into or mutate the active virtual environment. Run the baseline Python package +and native extension together from that source directory with the existing +interpreter. Confirm imported paths and native hash. Add only the current +boolean/length diagnostic to the scratch behavioural assertion. + +Run the same 26-case behavioural group at most ten times, 60 seconds each, +stopping on first mismatch. Save build and trial logs under +`/tmp/issue379-parity-h7-*`. A matching baseline failure establishes that the +extraction is not necessary for this symptom, but does not discharge the +underlying contract. No reproduction remains inconclusive. No verifier build, +tracked edits, environment installation, or full repository gate is authorized. + +## H7 result and H8 syscall localization + +The archived baseline reproduced an empty result from the two-stage +`echo -n hello | python uppercase` pipeline, with both stages exiting zero. The +baseline extension SHA-256 was +`bd7fcfbc971460bc9b1d510bf0b5164b1a6818ed5199b73a2b107e896ec927e2`. Evidence: +`/tmp/issue379-parity-h7-build.log` and `/tmp/issue379-parity-h7-trials.log`. +This establishes a pre-extraction native output-loss symptom, not that every +mismatch has the same cause. + +A documentation inspection accidentally invoked an additional `make test` +through shell substitution during this period. Its own process group was +terminated after discovery; no incomplete output counts as gate evidence. The +H7 workload therefore was not isolated from other local test activity. That +limits scheduling comparisons, but does not change its archived-code provenance +or the observed empty output from two successful processes. + +H8 tests whether asyncio reads upstream bytes that never reach the downstream +pipe during hand-off. Prediction: a failing small-payload trace contains an +upstream `read("hello")` without a corresponding downstream write/splice of +those bytes. If downstream reads `hello` and writes `HELLO`, the proposed +inter-stage loss is falsified for that trace and capture is the next boundary. + +The alchemist may run at most 200 current-tree two-stage uppercase pipelines in +one Rust-selected interpreter under `strace -f -yy`, tracing only +`read,write,splice,close,dup,dup2,dup3,fcntl,pipe,pipe2`. Use a scratch runner, +record each trial's stage PIDs, exits, captured value, and native provenance, +and stop at first mismatch. A 300-second outer timeout and 30-second +per-pipeline limit bound this experiment. Do not add delays or replace stream +operations. Syscall tracing is the only instrumentation; no Python hand-off +wrappers. Save `/tmp/issue379-parity-h8.log` and +`/tmp/issue379-parity-h8-syscalls.log`. No tracked edits or full gates. + +## H9: a closing transport cannot establish the reader lease + +Live CPython 3.13.13 source shows that `_UnixReadPipeTransport.pause_reading` +returns silently when `_closing` is true. EOF handling sets that flag and queues +`_call_connection_lost`, which later closes the pipe even if the +`StreamReader` remains reachable. Cuprum currently interprets that silent +return as successful hand-off. This is a specific lifetime-contract gap; its +relationship to the observed output mismatch is not established. + +After H8 stops, the alchemist may execute one Unix-only real-transport case: +create a real pipe and attach its reader via `loop.connect_read_pipe` and +`StreamReaderProtocol`; close the writer; synchronously invoke the transport's +actual `_read_ready` once to observe real EOF and queue connection loss. Before +yielding, record `is_closing`, descriptor validity, and Cuprum's actual +`_pause_reader_transport` result. Yield once for queued callbacks, retain the +reader object, then record descriptor validity again. Do not call native Rust +with a released descriptor. Close only owned test resources, guarding against +double close. Bound the experiment to ten seconds and save +`/tmp/issue379-reader-lease-h9.log`. Prediction: hand-off is accepted while the +descriptor is then closed despite object retention. A rejected hand-off or a +descriptor that remains open falsifies that prediction for this case. No +tracked edits, substitute transport implementation, or full gates. + +## H9 result and regression authorization + +H9 observed an open, closing transport with `may_hand_off=True`; after one +yield its descriptor was closed despite retaining the reader object. Evidence: +`/tmp/issue379-reader-lease-h9.log`. The hypothesis is not falsified. The +planner added `test_pipeline_streams_closing_reader.py`, which uses the real +EOF protocol callback rather than manually invoking private transport methods. +The alchemist may run only this test once before the fix, capturing +`/tmp/issue379-reader-lease-before.log`. The expected failure is acceptance of +the closing descriptor. Stop for the planner's fix; no full gates or edits. + +The regression failed as predicted: a real closed pipe was accompanied by +`may_hand_off=True`. Evidence: `/tmp/issue379-reader-lease-before.log`. The +pause helper now rejects `is_closing()` transports with the existing +`READER_PAUSE_FAILED` decline, leaving buffered bytes for the Python fallback. +The alchemist may rerun that one test once after the fix, recording +`/tmp/issue379-reader-lease-after.log`; no broader testing is authorized here. + +The focused regression passed after the guard: one test passed, preserving the +buffered prefix and refusing the invalid native borrow. Full gates may resume. +This fixes the demonstrated lifetime gap; the historical payload mismatches +have not been traced to that gap and must not be described as proven fixed. + +H8 completed 200 traced small pipelines without a mismatch. The trace confirms +valid transfer on its passing examples only; it did not localize a failing +execution. Its first scratch launch had a runner indexing error, repaired +without tracked changes before the bounded traced run. diff --git a/docs/developers-guide.md b/docs/developers-guide.md index 36116e782..c835c689b 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -13,6 +13,13 @@ of truth for day-to-day contributor expectations. For the system design, see the - [ADR-006: Split cuprum/context.py into a context package](adr-006-context-package-split.md) - [ADR-007: Subprocess execution module boundaries](adr-007-subprocess-execution-module-boundaries.md) - [ADR-009: Enforce Oxford spelling in source](adr-009-enforce-oxford-spelling-in-source.md) +- [ADR-010: Rust-pump executor-hop spans](adr-010-rust-pump-hop-span.md) +- [ADR-011: Audited Rust safety boundaries](adr-011-audited-rust-boundaries.md) + +The +[Rust boundary verification and unsafe inventory](rust-boundary-verification.md) +is the source of truth for native safety contracts, verifier bounds, trusted +assumptions, and current proof status. ## GitHub Actions runners @@ -809,7 +816,13 @@ partial-failure paths in one place rather than inlined in the pump: caller falls back to the Python pump rather than racing it. A transport exposing no pause hooks answers `True`, since there are no callbacks to suspend. A transport with `pause_reading()` but no `resume_reading()` answers - `False`: pausing it could not be undone. + `False`: pausing it could not be undone. A transport that is already closing + is never paused — `_pause_reader_transport` asks `is_closing()` before + pausing — because asyncio silently ignores a pause there and its queued + `connection_lost` callback can still close the descriptor during hand-off. + The outcome is an instance of `_ReaderPause`, the frozen slots dataclass that + carries `may_hand_off`, `resume`, `decline_reason`, and the + `closing_transport` marker, reporting a `reader_pause_failed` decline. Before native I/O begins, the reader transport is paused and callbacks queued before the pause are allowed to settle. If `StreamReader._buffer` contains @@ -820,7 +833,17 @@ the remaining raw bytes. Rust borrows the reader duplicate and consumes the writer duplicate. After the worker settles, Python closes the remaining reader duplicate; it closes the writer duplicate itself only when preparation or executor submission fails. If any preparation step cannot be completed safely, -dispatch falls back to the Python pump. +dispatch falls back to the Python pump. That fall-back has one exception, and +it is test-only: `_permit_test_owned_descriptor_handoff` in +`cuprum/_pipeline_streams.py` overrules the closing-reader decline — and only +that one — when the dispatch test seam has installed its own descriptor supply +(`_PUMP_STREAM_DISPATCH_TEST_HOOKS.raw_fd_extractor` is not `None`). A real +hand-off derives its worker descriptor from the transport and keeps the +decline. The permitted verdict carries no resume hook, because nothing was +paused. That permission depends on the configured test hooks: the stub must be +cleared with `reset_pump_stream_dispatch_for_testing()`, which the +`clear_backend_caches` fixture applies around each test in the pump-stream +dispatch test modules, leaving nothing for a later test to inherit. Cancellation is handled explicitly. `run_in_executor` cannot interrupt the worker thread running the Rust pump. `_run_rust_pump_with_blocking_fds` shields @@ -860,7 +883,7 @@ Table 1: `cuprum_reason` values and the seam each one reports | `cuprum_reason` | Seam that declined | | --------------------------- | ------------------------------------------------------------------ | | `raw_fd_unavailable` | `_extract_stream_fd` found no descriptor on at least one transport | -| `reader_pause_failed` | `pause_reading()` raised, so asyncio may still be consuming | +| `reader_pause_failed` | `pause_reading()` raised, or the transport was already closing | | `blocking_mode_unavailable` | `_BlockingModeGuard.engage` could not switch both descriptors | These are logged at `DEBUG`, deliberately. A fall-back is a per-hop routing @@ -1361,6 +1384,37 @@ guessed from PID: a `start` without an `exec_id` creates no span, and `stdout`/ `pipeline_fail_fast`/`exit` without one are dropped. Every event Cuprum itself emits carries an `exec_id`, so this only affects hand-built event streams. +## Rust-pump executor-hop span boundary + +[ADR-010](adr-010-rust-pump-hop-span.md) defines the separate, opt-in tracing +surface for Rust-pump executor hops. Keep it separate from the `PumpEvent` +channel: pump events describe routing and cleanup observations, while hop spans +cover the lifetime of one scheduled native transfer. + +The implementation is split by responsibility: + +- `cuprum/pump_span_events.py` owns the stable span name, bounded attribute + names, and the closed `PumpHopOutcome` vocabulary. +- `cuprum/pump_span_observation.py` owns the context-local tracer registry, + registration handles, observer-failure policy, and opening or closing the + spans held by one hop. +- `cuprum/_pipeline_streams.py` owns Rust-pump dispatch. It opens spans only + after the fast path accepts the hop and attaches the carrier to the native + executor future. +- `cuprum/_pipeline_rust_pump_completion.py` owns the completion callback. It + determines the terminal outcome, closes the hop spans, restores asyncio + stream state, and signals that cleanup is complete. +- `cuprum/_pipeline_stream_cleanup_observation.py` owns the cancellation drain + that waits for native worker settlement and emits the existing cleanup + observations. + +The completion callback is the lifetime boundary: it must settle the native +worker before restoration is reported complete, and the cancellation drain must +remain covered by the hop span. Do not pass trace context through the PyO3 +boundary; the Rust-internal span remains parentless. Keep hop attributes +bounded and avoid adding command payloads, descriptor values, exception text, +or identifiers to this surface. + ## Canonical `_TokenRegistration` handle base All `ContextVar`-backed scope-registration handles — `AllowRegistration`, @@ -2278,23 +2332,28 @@ cache and tool directories on both platforms. ## Rust error taxonomy (`PumpError`) -The `cuprum-rust` crate reports stream pump and consume failures through one -semantic error enum, `PumpError` (`rust/cuprum-rust/src/errors.rs`), derived +The `cuprum-streams` crate reports stream pump and consume failures through one +semantic error enum, `PumpError` (`rust/cuprum-streams/src/errors.rs`), derived with `thiserror`: - `LengthOverflow` — an integer length conversion overflowed its target type ("impossible" on supported platforms, kept observable rather than silently truncating). - `BufferRangeExceeded` — a computed range exceeded the backing buffer. +- `BufferAllocationFailed` — the scratch buffer could not be allocated. The + stream loops reserve fallibly because `vec![0_u8; len]` aborts the process on + allocation failure instead of unwinding, which would take an embedding + interpreter down with it. - `Io(io::Error)` — an operating-system I/O failure (transparent wrapper). -Conversion to a Python exception happens in exactly one place -(`From for PyErr`). The overflow variants surface as plain -`OSError`. The non-fatal write classification (broken pipe / connection reset) -lives on the enum as `PumpError::is_nonfatal_write`, replacing the free -function the splice and read/write paths previously shared. New failure -conditions get a variant here rather than a stringly-typed -`io::Error::other(...)`. +Conversion to a Python exception happens in exactly one place, +`pump_error_to_py_err` in `rust/cuprum-rust/src/errors.rs` (called from +`stream_pyfunctions.rs`). Every non-`Io` variant surfaces as a plain `OSError` +carrying the stable message from `PumpError::py_os_error_message`. The +non-fatal write classification (broken pipe / connection reset) lives on the +enum as `PumpError::is_nonfatal_write`, replacing the free function the splice +and read/write paths previously shared. New failure conditions get a variant +here rather than a stringly-typed `io::Error::other(...)`. ### Preserving the operating-system error code @@ -2399,17 +2458,18 @@ sees the `#[cfg(unix)]` arm and never the Windows one. ## Rust FD-borrow ownership contract -The pump and consume entry points in `rust/cuprum-rust/src/lib.rs` sort every -descriptor they touch into a *borrowed* or a *consumed* role. `pump_stream` -borrows its reader and consumes its writer; `consume_stream` borrows its reader -and takes no writer at all. The borrow half is centralized in one helper, -`with_borrowed_reader`. The helper rebuilds a `StreamHandle` from the -caller-owned raw descriptor, wraps it in `ManuallyDrop`, and runs the caller's -closure against it. `ManuallyDrop` suppresses the close on *every* exit path — -a normal return and unwinding from a panicking operation alike — so a -descriptor the Python side still owns is never closed by Rust. `pump_stream` and -`consume_stream` both route their reader through the helper, keeping the -"borrow this FD without owning it" rule in a single place. +The PyO3 pump and consume entry points in +`rust/cuprum-rust/src/stream_pyfunctions.rs:42,80` sort every descriptor they +touch into a *borrowed* or a *consumed* role. `pump_stream` borrows its reader +and consumes its writer; `consume_stream` borrows its reader and takes no +writer at all. The unsafe raw-reader constructor +`cuprum_native_io::borrow_reader` is called only at this PyO3 integration +boundary. The safe `cuprum_native_io::borrow` API returns a lifetime-bound OS +borrow; `cuprum-streams` receives typed `AsStream` values and cannot +reconstruct a resource from an integer. The native crate's +`with_retained_owner` kernel is used by the Windows `cap_std::fs::File` adapter +to keep a temporary reconstructed owner from closing a caller-owned handle +across normal return, error, or real unwind. The private `stream_pyfunctions::run_stream_operation` helper is limited to the two PyO3 stream exports. It owns their shared buffer validation, reader @@ -2423,53 +2483,154 @@ This supersedes an earlier pattern that reconstructed the handle and called `std::mem::forget` after the inner operation returned. Because a panic unwinds past the trailing `forget`, that pattern dropped — and therefore closed — the caller-owned descriptor on the unwind path, exposing the Python transport to a -double close of the same FD (the `#125` panic-unwind hazard). `ManuallyDrop` -holds regardless of how the scope exits, so no drop guard or `forget` call is -required. +double close of the same FD (the `#125` panic-unwind hazard). The borrowed +reader path now constructs a lifetime-bound `BorrowedStream` directly. The +Windows adapter uses `ManuallyDrop` through `with_retained_owner`; the real +`catch_unwind` tests cover that retention path. There is deliberately no borrowed *writer* variant. The writer FD handed to `pump_stream` is consumed: it must close on drop — including during unwinding — -so downstream readers observe EOF. Reconstruct the writer with -`stream_from_raw` (which yields an owning handle) and let it drop; reserve -`with_borrowed_reader` for descriptors whose ownership stays with the caller. -Python callers must therefore fully relinquish the writer descriptor supplied to -`pump_stream`/`rust_pump_stream`. The pipeline caller passes worker-owned -duplicates of both asyncio transport descriptors: asyncio keeps and closes the -originals, Rust borrows the reader duplicate without closing it, and Rust -closes the received writer duplicate on drop to signal EOF. The Python hand-off -owner closes the reader duplicate after the worker settles. The two descriptor -numbers in each pair must never be shared between those owners. The helper's -safety contract obliges the caller to guarantee each `fd` is a valid open -descriptor (or Windows handle) for the duration of the call and that ownership -remains with the caller; in return the helper guarantees it never closes the -borrowed reader `fd`. +so downstream readers observe EOF. The PyO3 boundary calls the unsafe +`cuprum_native_io::adopt_writer` only after Python has transferred a unique +resource. `cuprum_native_io::with_owned_writer` then owns the drop scope and +gives its callback only an immutable reference to the writer. This prevents +callback code from replacing or moving the actual owned resource out of the +scope. The normal path explicitly drops the writer after the callback; unwind +relies on automatic RAII. Reserve `cuprum_native_io::borrow` for resources +whose ownership stays with the caller. Python callers must therefore fully +relinquish the writer descriptor supplied to `pump_stream`/`rust_pump_stream`. +The pipeline caller passes worker-owned duplicates of both asyncio transport +descriptors: asyncio keeps and closes the originals, Rust borrows the reader +duplicate without closing it, and Rust closes the received writer duplicate on +drop to signal EOF. The Python hand-off owner closes the reader duplicate after +the worker settles. The two descriptor numbers in each pair must never be +shared between those owners. The helper's safety contract obliges the caller to +guarantee each `fd` is a valid open descriptor (or Windows handle) for the +duration of the call and that ownership remains with the caller; in return the +helper guarantees it never closes the borrowed reader `fd`. + +The Windows-handle wording above describes direct Rust extension calls. The +pipeline dispatcher declines Windows asyncio subprocess-pipe handles with +`platform_unsupported`, because ProactorEventLoop's overlapped handles cannot +be used safely by the pump's synchronous Rust I/O. Native Windows wheels and +the direct extension API remain supported independently of this pipeline +restriction. The contract is checked at two levels, which are deliberately not -interchangeable. `rust/cuprum-rust/src/lib_tests.rs` holds the +interchangeable. `rust/cuprum-native-io/src/ownership_tests.rs` holds the integration-level regression tests: they open *real* descriptors, run the helper both normally and through a panicking operation, and assert the borrowed FD is still open afterwards. That is the only place actual `close(2)` behaviour is exercised, and it stays the authority on it. Alongside them, -`rust/cuprum-rust/src/fd_ownership_kani_proofs.rs` carries a bounded Kani proof -of the ownership invariant itself: a borrowed reader FD is never closed by Rust -on a normal or an unwinding exit, and a `pump_stream` writer is always consumed -and closes exactly once to signal EOF. - -Kani does not interpret I/O, so the proof cannot use real descriptors. It runs -against the pure model in `fd_ownership_model.rs` — gated -`#[cfg(any(test, kani))]`, so `make test` exercises it as ordinary unit tests -too — where `ModelFd` records a close on drop instead of issuing one. The -`ManuallyDrop` wrapper, the closure call, and the early-exit edge are all real -Rust, so Rust's own drop elaboration decides the outcome rather than any -handwritten accounting. Because Kani compiles panics as aborts, the unwind path -is modelled with `?`: a `?` early return and a real unwind both leave the frame -without running the statements that follow the operation, so reintroducing the -superseded trailing-`mem::forget` makes the proof fail (that mutation was run -to confirm the proof is not vacuous). Being a bounded model checker, Kani -establishes this over an explicitly bounded state space — the two exit modes, -and at most three repeated borrows — rather than for all executions. Active -verification tracking, including whether Verus adds anything beyond the Kani -model once that model is complete, lives in issue `#89`. +`rust/cuprum-native-io/src/fd_ownership_kani_proofs.rs` carries the historical +bounded close-count model proof. The current production drop and progress +proofs are in `rust/cuprum-native-io/src/kani_proofs.rs`: a borrowed reader is +retained, an owned writer is dropped once, and checked counts reject invalid +external progress. + +Kani does not interpret I/O, so the proof cannot use real descriptors. The +historical model in `rust/cuprum-native-io/src/fd_ownership_model.rs` remains +provenance for the earlier close-count proof, while the current production +ownership proof exercises `with_owned_writer` and the shared +`memory::with_retained_owner` kernel. Neither model observes an OS close. The +historical model is gated `#[cfg(any(test, kani))]`, so `make test` exercises +it as ordinary unit tests too — where `ModelFd` records a close on drop instead +of issuing one. The `ManuallyDrop` wrapper, the closure call, and the +early-exit edge are all real Rust, so Rust's own drop elaboration decides the +outcome rather than any handwritten accounting. The model covers only its +explicit `Normal` and `Error` outcomes; it does not simulate panic unwind. Real +`catch_unwind` native tests remain authoritative for that path and for OS close +effects. Being a bounded model checker, Kani establishes its claims over an +explicitly bounded state space rather than for all executions. The final fault +run passed its controls and detected the historical trailing-`mem::forget`, +invalid-prefix-bound, incorrect-accounting, and leaked-writer mutations; the +archived results and exact command are recorded in the boundary verification +document. Active verification tracking and the distinction between model +evidence and real OS effects lives in +[Rust boundary verification](rust-boundary-verification.md) and issue `#89`. + +### Native boundary verification + +The boundary checks have dedicated Makefile targets so the normal Rust +toolchain and the verifier toolchains remain separate: + +```bash +make boundary-test +make boundary-verus +make boundary-kani +make boundary-miri +make boundary-contract +make boundary-faults +``` + +`make boundary-contract` confirms the safe targets reject unsafe Rust, and +`make boundary-faults` requires the verification harnesses to detect deliberate +faults. + +Install the checksum-verified prebuilt tools first with `make install-verus` and +`make install-boundary-kani`. The pinned `rust-prover-tools` installer selects +Python `3.14` explicitly, including when CI exports `UV_PYTHON=3.13`; this +tool-only runtime does not change the project Python test matrix. Verus uses +Rust `1.98.0`, Verus `0.2026.09.06.8dea4a2`, and a separately installed +prebuilt Z3 `4.16.0`. Kani uses `0.67.0` with compiler +`rustc 1.93.0-nightly (53732d5e0 2025-11-20)` and CBMC `6.8.0`; Miri uses +`nightly-2026-08-07` with `rustc 1.99.0-nightly (84b36a78a 2026-08-06)`. The +Verus input is regenerated from the production progress kernels on every run. +The final-source progress proof verifies two functions with zero errors, +recorded in `/tmp/issue379-round24-verus.log`, while direct assessment of the +unchanged production `adopt_writer` fails on unsupported `OwnedFd` and +`FromRawFd::from_raw_fd` representations; see +`/tmp/issue379-verus-resource-assessment.log`. The installer succeeds in +fetching and validating the pinned Verus and Kani binary caches. Current proof +status, bounds, excluded targets, and trusted assumptions are recorded in +[Rust boundary verification](rust-boundary-verification.md); a tool that is not +available or compatible is a blocker rather than a passing skip. + +The final native Kani boundary run passed 7 harnesses with 0 failures in +`/tmp/issue379-round27-kani-native.log`; the safe-policy/decoder run has 11 +harnesses passed, covered by the current result in +`/tmp/issue379-round24-kani.log`. The native ownership proofs enumerate two +exits; the repeated-borrow harness restricts borrows to three and uses +`#[kani::unwind(4)]`. Production progress proofs use symbolic `usize` +count/capacity and `u64` total/remaining/written values. Safe pump proofs use +symbolic `PumpState`, `usize` read lengths, and `u64` write counts; the +three-step proof uses `#[kani::unwind(4)]`. UTF-8 proofs use fixed three- and +four-byte arrays, with unwind bounds 5 for the first four harnesses and 4 for +the final prefix harness. These are bounded results, not an unbounded proof. +Round 27 Miri passed all 13 isolated native tests with zero ignored; see +`/tmp/issue379-round27-miri.log`. The Miri run excludes PyO3, unshimmed +`libc::splice`, and unsupported operating-system representations. Round 33's +prior local checkpoint passed, and Round 35's integrated local run passed the +native and repository gates. Windows and macOS runtime tests are no longer +pending hosted execution. On 2026-09-15, `Rust boundary verification` run +35004943625 succeeded with its `Native contracts (windows-2022)`, +`Native contracts (macos-latest)`, and `Native contracts (ubuntu-latest)` jobs, +and `CI` run 35004943782 succeeded with its +`Extension-gated tests (Windows Python/Rust boundary)` and +`Extension-gated tests (Python/Rust boundary)` jobs. The boundary workflow's +`extended` job is skipped on pull requests by design. + +The boundary installer and fault harnesses have narrow reuse policies. +`scripts/install_boundary_kani.py` owns `checked_download` for the Kani and Z3 +installers only: HTTPS-only bounded redirects, digest validation, and no source +builds. `scripts/check_boundary_faults.py` scopes its runner to the four +verification commands and a disposable source copy; it is not a general process +abstraction. Its progress and ownership checks have separate private runners; +their scope is the existing four mutations, with the same controls and failure +diagnostics. The download helpers separate bounded redirect traversal, one +connection's lifetime, and response classification. They remain private to the +pinned binary installer. The compiler-contract checker uses a named predicate +requiring both failure and an unsafe-forbid diagnostic, so unrelated compiler +failures cannot pass the probe. + +Windows borrowed I/O lives beside its handle adapter in +`rust/cuprum-native-io/src/windows.rs`. Its private `with_file` helper retains +the borrowed handle and converts the byte count once for both read and write; +Unix adapters retain their direct syscall and buffer contracts. + +`make boundary-faults` archives controls and deliberate fault failures for the +scheduled `extended` job. The final run passed its controls and detected all +four mutations; see `/tmp/issue379-round27-faults.log`. ### Python-side native pump descriptor lifetime @@ -2501,16 +2662,34 @@ Blocking-mode failure selects the Python fallback and emits `blocking_setup_failed`. The outcome events are counted by `cuprum_rust_pump_handoff_total` as described above. +`_prepare_rust_pump_call` completes every Python-side check before the writer +descriptor reaches native code: it loads the native module, validates the +buffer size (`_validate_buffer_size_before_writer_transfer`), then prepares the +reader. Buffer-size validation calls `operator.index` and raises +`OverflowError` for values outside the `i64` range and `ValueError` for a +non-positive size or one above the 1 GiB cap (`_MAX_BUFFER_SIZE`). Reader +preparation (`_prepare_native_reader`) requires a non-negative descriptor +(`ValueError`, "file descriptor must be non-negative", or "file handle must be +non-negative" on Windows) before conversion, converts it for the platform, then +requires an `i64`-representable value (`OverflowError`) and a 32-bit signed +range on non-Windows platforms (`ValueError`, "file descriptor out of range"). +Each failure closes the writer Python still owns +(`_close_writer_after_pre_native_failure`) and emits the matching hand-off +outcome (`native_load_failed`, `buffer_validation_failed`, or +`reader_preparation_failed`); a reader-preparation failure additionally logs the +`reader_preparation` phase at `DEBUG` on the `cuprum._streams_rs` logger. + ## Rust splice-loop and drain contract -The Linux zero-copy path in `rust/cuprum-rust/src/splice.rs` follows one -canonical loop. `try_splice_pump` performs the first `splice_once` solely to -detect support: `EINVAL` on that first call signals unsupported descriptor -types and the read/write fallback. Every outcome thereafter — including the -first call's, which is fed into the loop — is handled by the same arms: `Ok(0)` -ends the transfer, `Ok(n)` accumulates, a non-fatal write error (broken pipe / -connection reset) drains the reader and reports the bytes transferred so far, -and anything else propagates. +The Linux zero-copy policy in `rust/cuprum-streams/src/splice/mod.rs` follows +one canonical loop. Its native syscall adapter is +`rust/cuprum-native-io/src/lib.rs`; `try_splice_pump` performs the first +`splice_once` solely to detect support: `EINVAL` on that first call signals +unsupported descriptor types and the read/write fallback. Every outcome +thereafter — including the first call's, which is fed into the loop — is +handled by the same arms: `Ok(0)` ends the transfer, `Ok(n)` accumulates, a +non-fatal write error (broken pipe / connection reset) drains the reader and +reports the bytes transferred so far, and anything else propagates. `splice_once` retries at the syscall level: an interrupted splice (`EINTR`) is re-issued rather than surfaced, matching the Unix read and write policies, so a @@ -2598,7 +2777,7 @@ looked like it was guarding against. That encapsulation is what the proptests and Kani proofs assume rather than establish — neither can observe a transition it cannot spell — so it is pinned separately by the compile-fail case -`rust/cuprum-rust/tests/ui/fail/pump_transition_unreachable.rs`. That case +`rust/cuprum-streams/tests/ui/fail/pump_transition_unreachable.rs`. That case includes `src/pump_machine.rs` as a child module, reproducing the relationship `lib.rs` has with it, and asserts the compiler's refusal of an attempt to build a `Transition::Wrote` and pass it to `step` from the parent. Widening either @@ -2671,18 +2850,25 @@ The span is created at `error` level so the `warn`/`error` events keep their operation context even under a `warn`/`error`-only production filter, where an `info` span would be disabled; it emits no log line of its own. -Unix Rust tests share pipe creation, duplicated-file wrapping, result helpers, -and descriptor-state checks through `test_support`. Re-use that module for +Safe stream tests share pipe creation, typed descriptor duplication, result +helpers, and descriptor-state checks through +`rust/cuprum-streams/src/test_support.rs`. Re-use that module for descriptor-backed test setup; keep production code independent of test helpers. -The splice behavioural tests expose that shared `make_pipe` as an rstest `pipe` -fixture, so scenarios needing several independent pipes inject it once per -`#[from(pipe)]` parameter. +Its `OwnedFd`-typed cloning API rejects a raw integer by construction, while +the duplicate-owner lifetime test confirms that dropping the clone leaves the +original usable. Keep the wrong-direction `EBADF` regressions for attempting to +write through a pipe reader and read through a pipe writer. The splice +behavioural tests expose the shared `make_pipe` as an rstest `pipe` fixture, so +scenarios needing several independent pipes inject it once per `#[from(pipe)]` +parameter. ## Rust property testing and verification -Rust-level tests for `cuprum-rust` live with the crate under -`rust/cuprum-rust/src/`. Use them for pure decoder, parsing, state-machine, and -adapter logic where Python integration tests would only cover a few examples. +Rust-level tests for safe stream policy live with `cuprum-streams` under +`rust/cuprum-streams/src/`; native ownership and syscall tests live with +`cuprum-native-io` under `rust/cuprum-native-io/src/`. Use them for pure +decoder, parsing, state-machine, and adapter logic where Python integration +tests would only cover a few examples. Property tests use [proptest](https://docs.rs/proptest/latest/proptest/) as a development dependency. Prefer generated payloads and small helper functions @@ -2936,18 +3122,19 @@ make lint ``` Run Kani separately because it is a bounded model checker rather than a normal -unit-test runner. The Kani installer places the verifier under `~/.kani`; the -dynamic library path is required when invoking the crate harnesses. Resolve the -toolchain library directory from the installed version rather than hard-coding -it: +unit-test runner. Install the checksum-verified prebuilt pinned Kani binaries +without a source build, then run the boundary harnesses from the repository +root: ```bash -KANI_VERSION=$(cargo kani --version | awk '{print $2}') -cd rust && \ - LD_LIBRARY_PATH="$HOME/.kani/kani-${KANI_VERSION}/toolchain/lib" \ - cargo kani --package cuprum-rust +make install-boundary-kani +make boundary-kani ``` +`make boundary-kani` confirms the pinned Kani `0.67.0` installation and +verifies both `cuprum-native-io` and `cuprum-streams`. The target supplies the +dynamic library path itself, so no manual `KANI_VERSION` resolution is needed. + When adding new Kani proofs, keep the bounds explicit with attributes such as `#[kani::unwind(N)]`, include `kani::cover!` statements for the intended boundary cases, and avoid broad symbolic comparisons that force Kani through @@ -3601,6 +3788,47 @@ probe. Reuse the existing helper rather than re-deriving the `sysconfig` scan; extend `maturin_script_locatable()` in place if maturin changes how it locates its binary. +## Native Rust source-distribution contract + +A source distribution must be able to build the optional native extension, so +the Rust workspace travels inside the sdist. `tests/test_native_sdist.py` holds +that contract: it builds a real archive with both packaging frontends and +checks the members that result. + +`[tool.uv].source-include` in `pyproject.toml` lists the workspace files the +`uv build --sdist` archive must retain: + +- the workspace manifests and pins — `rust/Cargo.toml`, `rust/Cargo.lock`, + `rust/rust-toolchain.toml`, and `rust/dylint.toml`; +- each crate manifest — `rust/cuprum-rust/Cargo.toml`, + `rust/cuprum-streams/Cargo.toml`, and `rust/cuprum-native-io/Cargo.toml`; +- all three crate source trees, including each crate's `src/` and the + integration `tests/` of `cuprum-rust` and `cuprum-streams`. + +Cargo build output is excluded: `target` never appears in an archive, so an +sdist carries sources rather than a multi-gigabyte build cache. + +Maturin builds its archive from the same workspace, and the +`[tool.maturin].include` entries carry the two files it would not otherwise +copy — `rust/rust-toolchain.toml` and `rust/dylint.toml`, both restricted with +`format = "sdist"` so they never enter a wheel. + +`tests/test_native_sdist.py` verifies both frontends, and it is deliberately +not a unit test: each case builds a genuine archive with `uv build --sdist` or +`uv run maturin sdist`, so the check fails if either frontend stops honouring +the configuration. Its parametrization covers `uv` and `maturin` separately +because the two read different manifest keys and can drift independently. + +**When the workspace changes.** Adding, renaming, or removing a crate — or +adding a file a build needs at the workspace root — requires a matching update +in three places: `[tool.uv].source-include` in `pyproject.toml`, the +`[tool.maturin].include` list when the file is a root-level pin, and the +expected-member set in `tests/test_native_sdist.py`. Run the contract with: + +```bash +uv run pytest tests/test_native_sdist.py -q +``` + ## Mutation-testing harness The mutmut mutation-testing workflow runs the selected test suite from its @@ -3625,16 +3853,17 @@ uv run pytest cuprum/unittests/test_mutmut_config_contract.py -q ## Rust stream buffer-size validation `rust/cuprum-rust/src/lib.rs` validates the `buffer_size` argument to -`rust_pump_stream` / `rust_consume_stream` at the PyO3 boundary through a pure -`checked_buffer_size(i64) -> Result` helper, wrapped by -`validate_buffer_size` (which maps the message to `PyValueError`). The contract -is: reject non-positive values, values that overflow `usize` on the target -platform, and values above `MAX_BUFFER_SIZE` (1 GiB, `1 << 30`) — the cap -guards against absurd allocations while comfortably exceeding any realistic -transfer buffer (the default is 64 KiB). `checked_buffer_size` is kept pure so -its boundaries are property tested directly in -`rust/cuprum-rust/src/buffer_size_tests.rs`; the Python-side error mapping is -exercised in `cuprum/unittests/test_rust_streams_boundary_property.py`. Keep the +`rust_pump_stream` / `rust_consume_stream` at the PyO3 boundary through +`validate_buffer_size(i64) -> PyResult`, which maps the message to +`PyValueError`. `BufferSize::new(size)` in `rust/cuprum-streams/src/lib.rs` +owns the validation and returns the validated size used by the stream loops. +The contract is: reject non-positive values, values that overflow `usize` on +the target platform, and values above `MAX_BUFFER_SIZE` (1 GiB, `1 << 30`) — +the cap guards against absurd allocations while comfortably exceeding any +realistic transfer buffer (the default is 64 KiB). The Rust boundary cases are +property tested in `rust/cuprum-streams/src/buffer_size_tests.rs`; the +Python-side error mapping is exercised in +`cuprum/unittests/test_rust_streams_boundary_property.py`. Keep the `_streams_rs.py` wrapper docstrings, `docs/cuprum-design.md`, and the users' guide aligned with this contract when the cap changes. @@ -3890,15 +4119,20 @@ with the command above (or as part of a full mutmut pass). The test validates: ## Compile-time UI tests (trybuild) -The Rust crate at `rust/cuprum-rust/` uses -[trybuild](https://github.com/dtolnay/trybuild) to validate contracts that hold -at compile time and so cannot be observed by a runtime test: PyO3 macro -behaviour, and encapsulation boundaries such as the pump machine's private -`Transition`. Tests live under `rust/cuprum-rust/tests/ui/`: +The Rust crates use [trybuild](https://github.com/dtolnay/trybuild) to validate +contracts that hold at compile time and so cannot be observed by a runtime +test. PyO3 macro behaviour remains in `rust/cuprum-rust/tests/ui/`; the safe +stream crate keeps encapsulation and writer-lifetime probes, including the pump +machine's private `Transition`, in `rust/cuprum-streams/tests/ui/`. Each +crate's UI directory contains: - `tests/ui/pass/` — Rust files that **must compile** without error. - `tests/ui/fail/` — Rust files that **must fail** compilation with diagnostics - matching the corresponding `.stderr` file. +matching the corresponding `.stderr` file. + +The PyO3 pass fixtures include +`rust/cuprum-rust/tests/ui/pass/const_availability_export.rs`; the safe stream +fixtures include the `Transition` and writer-lifetime probes. Run compile-time UI tests with: diff --git a/docs/repository-layout.md b/docs/repository-layout.md index fea13fcbf..4b77c83ce 100644 --- a/docs/repository-layout.md +++ b/docs/repository-layout.md @@ -26,31 +26,38 @@ _Figure 1: Simplified repository tree for contributor orientation._ ## Path responsibilities -| Path | Responsibility | -| ------------------- | ----------------------------------------------------------------------------------------------------------------------------- | -| `.github/` | GitHub Actions workflows, Dependabot configuration, and reusable workflow actions. | -| `.rules/` | Python coding rules referenced by `AGENTS.md` and maintainer guidance. | -| `benchmarks/` | Benchmark drivers, profiling helpers, deterministic fixtures, and benchmark validation code. | -| `cuprum/` | Python package source for the command catalogue, safe command builders, execution runtime, streams, and Rust backend adapter. | -| `docs/` | User, maintainer, design, decision, roadmap, and reference documentation. | -| `docs/execplans/` | Durable execution plans for non-trivial implementation work. | -| `rust/` | Cargo workspace for Rust extension code and Rust-specific build/test targets. | -| `rust/cuprum-rust/` | PyO3-backed Rust crate that provides accelerated Cuprum components. | -| `test-wheelhouse/` | Local wheel artefacts used by validation workflows and compatibility tests. | -| `tests/` | Behavioural, integration, and regression tests for the Python package and user-visible workflows. | -| `AGENTS.md` | Repository-specific assistant and contributor instructions. | -| `Makefile` | Canonical entry point for build, format, lint, typecheck, test, documentation lint, and diagram validation gates. | -| `pyproject.toml` | Python project metadata, dependency declarations, and tool configuration. | -| `uv.lock` | Locked Python dependency graph for reproducible `uv` environments. | +| Path | Responsibility | +| ------------------------ | ----------------------------------------------------------------------------------------------------------------------------- | +| `.github/` | GitHub Actions workflows, Dependabot configuration, and reusable workflow actions. | +| `.rules/` | Python coding rules referenced by `AGENTS.md` and maintainer guidance. | +| `benchmarks/` | Benchmark drivers, profiling helpers, deterministic fixtures, and benchmark validation code. | +| `cuprum/` | Python package source for the command catalogue, safe command builders, execution runtime, streams, and Rust backend adapter. | +| `docs/` | User, maintainer, design, decision, roadmap, and reference documentation. | +| `docs/execplans/` | Durable execution plans for non-trivial implementation work. | +| `rust/` | Cargo workspace for Rust extension code and Rust-specific build/test targets. | +| `rust/cuprum-rust/` | Thin PyO3/maturin integration crate for the optional native backend; raw Python resource hand-off lives here. | +| `rust/cuprum-streams/` | Safe stream orchestration, pump policy, state machine, errors, and checked UTF-8 decoding; unsafe Rust is forbidden. | +| `rust/cuprum-native-io/` | Audited native descriptor/handle ownership, single-call I/O, progress kernels, and platform test fixtures. | +| `test-wheelhouse/` | Local wheel artefacts used by validation workflows and compatibility tests. | +| `tests/` | Behavioural, integration, and regression tests for the Python package and user-visible workflows. | +| `AGENTS.md` | Repository-specific assistant and contributor instructions. | +| `Makefile` | Canonical entry point for build, format, lint, typecheck, test, documentation lint, and diagram validation gates. | +| `pyproject.toml` | Python project metadata, dependency declarations, and tool configuration. | +| `uv.lock` | Locked Python dependency graph for reproducible `uv` environments. | _Table 1: Major repository paths and their ownership boundaries._ ## Conventions -Source code for the Python package belongs under `cuprum/`; source code for the -Rust extension belongs under `rust/cuprum-rust/`. Cross-language behaviour -should be documented in [Cuprum design](cuprum-design.md) and surfaced in the -[users' guide](users-guide.md) when it affects public behaviour. +Source code for the Python package belongs under `cuprum/`. The Rust workspace +keeps PyO3 integration in `rust/cuprum-rust/`, safe stream policy in +`rust/cuprum-streams/`, and native resource operations in +`rust/cuprum-native-io/`. Cross-language behaviour should be documented in +[Cuprum design](cuprum-design.md) and surfaced in the +[users' guide](users-guide.md) when it affects public behaviour. See +[ADR-011](adr-011-audited-rust-boundaries.md) and +[Rust boundary verification](rust-boundary-verification.md) for the safety and +verification contracts. Long-lived documentation belongs under `docs/`. New architectural decisions should be recorded as ADRs in `docs/` and linked from diff --git a/docs/roadmap.md b/docs/roadmap.md index 6a2d55051..7391a2225 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -518,6 +518,34 @@ and 5). - Success: the decision and its rationale (likely continued deferral) are recorded in adr-002-additional-rust-components.md. +### 8.4. Isolate unsafe Rust in audited boundary crates + +This step resolves issue #379 by keeping abstract stream policy safe while +placing native descriptor, handle, and syscall obligations behind small +workspace boundaries. See [ADR-011](adr-011-audited-rust-boundaries.md) and +[Rust boundary verification](rust-boundary-verification.md). + +- [x] 8.4.1. Keep `cuprum-streams` free of unsafe Rust, including test and + helper targets, and keep `cuprum-rust` limited to the PyO3/maturin hand-off. + - Success: the safe-crate contract rejects deliberate unsafe probes, the + workspace allowlist names only approved boundary crates, and Python API, + optional backend, packaging, and supported-platform behaviour retain + regression coverage. +- [x] 8.4.2. Maintain native ownership and I/O contracts for borrowed readers, + uniquely transferred writers, initialized buffers, short or interrupted + calls, byte accounting, and real cleanup on normal, error, and unwind paths. + - Success: native Unix/Linux and Windows tests observe descriptor/handle + effects, EOF, fallback, broken pipes, and setup or cancellation cleanup; + fault sensitivity detects leaks, double closes, invalid bounds, and bad + accounting. +- [x] 8.4.3. Run and archive the applicable Verus, Kani, and Miri evidence with + exact pinned toolchains, bounds, coverage witnesses, exclusions, and trusted + assumptions. + - Success: Verus verifies exact production kernels where feasible; Kani is + used for unsuitable representations with bounded results labelled as such; + Miri runs applicable isolated targets; CI has affordable pull-request + checks and documented scheduled or manual heavier validation. + ## 9. Idle heartbeat for quiet children (issue `#359`) Idea: a run that emits nothing for minutes is indistinguishable in a CI log diff --git a/docs/rust-boundary-verification.md b/docs/rust-boundary-verification.md new file mode 100644 index 000000000..89e3c5f3f --- /dev/null +++ b/docs/rust-boundary-verification.md @@ -0,0 +1,374 @@ +# Rust boundary verification and unsafe inventory + +## Status and scope + +This document records the initial unsafe-code audit for issue [issue 379]. The +inventory is anchored to baseline commit +[`34a59eac4cd13de82c3730d4b569eeaefbb5ee61`](https://github.com/leynos/cuprum/tree/34a59eac4cd13de82c3730d4b569eeaefbb5ee61) +and covers Unix, Linux, and Windows conditional code, test targets, and +PyO3-generated foreign-function interface (FFI) paths. It concerns Rust +unsafety only; the separately named `cuprum.unsafe` command namespace is out of +scope. + +The implemented architecture has two Rust libraries behind the original +extension crate. `cuprum-streams` owns safe stream policy, pump state +transitions, error classification, and checked UTF-8 decoding, and enforces +`unsafe_code = "forbid"` for library and test targets. `cuprum-native-io` owns +the smallest necessary descriptor/handle and syscall surface. The existing +`cuprum-rust` crate remains the thin PyO3/maturin integration crate and is the +only caller of the raw-resource constructors. The approved unsafe-bearing crate +set is therefore `cuprum-native-io` and the PyO3 integration boundary in +`cuprum-rust`; `cuprum-streams` is the safe crate. + +No row below is proof that the target architecture or its verification is +complete. “Existing coverage” describes baseline tests or models; “gap” is work +required by issue #379. + +The baseline scan found no `unsafe fn`, `unsafe impl`, or unsafe trait +implementation. Every explicit unsafe block is represented in Table 1; the +generated PyO3 operations are recorded separately because their expansion is +not visible in the source scan. + +## Baseline inventory + +Table 1. Rust unsafe operations and their proposed boundary. + +| Baseline location | Operation and obligations | Caller or entry point | Proposed destination | Existing coverage and gap | +| ---------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `rust/cuprum-rust/src/lib.rs:224-228` | `OwnedFd::from_raw_fd` reconstructs an owning Unix descriptor. The integer must identify a live descriptor, remain valid during the operation, and not be closed or reused concurrently. The borrow path must suppress the reconstructed owner's drop on normal, error, and real unwind paths. | `rust_pump_stream` and `rust_consume_stream`, ultimately called by Python with a transport descriptor. | `cuprum-native-io` borrowed-reader constructor with a lifetime-bearing safe API; raw validity stays at the audited integration boundary. | `lib_tests.rs` checks borrowed-reader success and panic paths and checks the descriptor remains open. The Kani model checks drop counts only; it does not execute `close(2)` or a real panic unwind. | +| `rust/cuprum-rust/src/lib.rs:246-250` | `ManuallyDrop` is the ownership guard around the reconstructed reader. Any replacement must preserve caller ownership without a trailing `mem::forget` that is skipped during unwind. The consumed writer must still drop exactly once. | `pump_stream` and `consume_stream`. | `cuprum-native-io` typed borrowed/owned resource handles, with safe stream callers unable to confuse the two. | Real descriptor regression tests exist. The historical trailing-`mem::forget` unwind error must remain a deliberate fault in verification or test harnesses. | +| `rust/cuprum-rust/src/lib.rs:253-260` | `File::from_raw_handle` reconstructs an owning Windows handle. The handle must be a valid, uniquely transferred, pointer-sized native handle and must not be closed or reused concurrently; drop must close it once. | Python's Windows handoff, after CRT descriptor conversion and duplication. | `cuprum-native-io` Windows ownership boundary. | Baseline source has no equivalent real Windows lifetime/unwind evidence in this crate. Add native handle tests and document Win32/CRT assumptions. | +| `rust/cuprum-rust/src/io_utils/mod.rs:222-228` | `libc::read` receives a live descriptor and a mutable, initialized buffer valid for `buffer.len()` writes. A non-negative result must fit the returned type; `EINTR`, EOF, and errors must preserve valid state. | `read_stream` and the pump/consume loops. | `cuprum-native-io` checked read primitive returning only initialized, in-bounds bytes. | Baseline pipe, EOF, unreadable-descriptor, and injected-`EINTR` tests exist. Symbolic bounds, pointer/initialization proof, and native short/error tests remain pending. | +| `rust/cuprum-rust/src/io_utils/mod.rs:265-277` | `libc::write` receives a live descriptor and an immutable, initialized slice. Partial writes, zero progress, `EINTR`, broken pipes, and byte conversion must not invalidate the slice or overflow accounting. | `classify_write` and the read/write pump fallback. | `cuprum-native-io` checked write primitive; safe `cuprum-streams` retains policy classification. | Baseline injected partial-write, zero-progress, `EINTR`, broken-pipe, fatal-error, and overflow-adjacent tests exist. Native short writes and verifier witnesses for all accounting branches remain pending. | +| `rust/cuprum-rust/src/splice/mod.rs:69-89` | `libc::splice` uses two live descriptors, a valid length, and null offsets for pipe endpoints. `EINTR`, unsupported `EINVAL`, partial transfer, broken-pipe drain, and checked byte accumulation must preserve state. | Linux pump path, selected before read/write fallback. | `cuprum-native-io` Linux-only syscall adapter with a safe result contract; policy and fallback remain in `cuprum-streams`. | Baseline Linux pipe, regular-file fallback, broken-pipe drain, `EINTR`, generated sequence, and overflow tests exist. Verus cannot model the syscall directly; Kani/seam proof, Miri seam coverage, and native kernel evidence remain pending. | +| `rust/cuprum-rust/src/utf8.rs:78-89` | `str::from_utf8_unchecked` requires that the `valid_up_to` prefix is both in bounds and valid UTF-8. An invalid prefix would create undefined behaviour. | Incremental replacement decoder. | Remove this avoidable unsafe operation; use checked conversion in safe `cuprum-streams`. | Baseline decoder, chunk-boundary, invalid-byte, and Kani model tests exist. The unchecked operation must not be carried into a boundary crate; update or remove its proof target after checked conversion. | +| `rust/cuprum-rust/src/test_support.rs:8-15` | `libc::pipe` fills a two-element buffer; after success both descriptors are fresh and exclusively owned before `OwnedFd::from_raw_fd`. Failure must not leak a partial result. | Unix unit and integration fixtures. | Native boundary test support or a safe test seam in `cuprum-native-io`; safe crates must not import this helper. | Real descriptor tests depend on it. Move the unsafe setup with its failure contract and keep the tests as external-effect regressions. | +| `rust/cuprum-rust/src/test_support.rs:18-29` | `libc::dup` must return a valid new descriptor before `File::from_raw_fd` takes unique ownership; failure and drop must not leak or double-close. | Read/write test helpers. | Native boundary test support. | Existing tests exercise duplicated reads and writes but do not constitute a verifier proof of all setup-failure paths. Add deliberate duplicate/setup faults. | +| `rust/cuprum-rust/src/test_support.rs:56-68` | `fcntl(F_GETFD)` reads descriptor status without dereferencing memory; `EINTR` must retry and any other failure means closed. | `fd_is_open` assertions in Unix tests. | Native boundary test support: test-only, crate-private `fd_is_open` helper. | Useful real-close oracle exists. It must not create an unsafe loophole in `cuprum-streams`; retain it beside native descriptor tests. | +| `rust/cuprum-rust/src/lib_tests.rs:217-220` | Test-only `File::from_raw_fd` wraps a borrowed descriptor in `ManuallyDrop`; the test must not accidentally close the caller-owned descriptor. | Borrowed-reader panic/success regression. | Native boundary test support with the same borrowed contract. | Existing real descriptor test is required evidence and must move with the boundary, rather than being deleted or made safe-looking through an unverified wrapper. | + +The baseline workspace had only `cuprum-rust` as a Cargo member and declared no +`unsafe_code = "forbid"` lint in `rust/Cargo.toml`. That was the inventory +conflict: abstract stream policy and native operations shared one crate, so +forbidding unsafe code there would reject valid native work. The current split +keeps the lint on `cuprum-streams`, including its test and helper targets, and +confines raw-resource construction to the two approved boundaries. + +## Current split + +Table 2. Current crate responsibilities and safe/unsafe surfaces. + +| Crate | Current responsibility | Unsafe policy and boundary | +| ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `rust/cuprum-streams` | Safe stream orchestration, read/write and Linux `splice` policy, pump state machine, errors, and checked UTF-8 replacement. | `#![forbid(unsafe_code)]` at the crate root and in compile-contract tests. It receives `AsStream` borrows and `OwnedStream` writers from the native boundary. | +| `rust/cuprum-native-io` | Lifetime-bound OS borrows, unique writer adoption, single read/write/splice calls, checked native progress, Unix and Windows pipe fixtures, and close observation for native regressions. | Approved unsafe-bearing crate. `borrow` is safe and lifetime-bound; `adopt_writer` and `borrow_reader` are unsafe integration constructors with caller obligations. Every unsafe block has a local safety comment, and the crate denies unsafe operations inside unsafe functions. | +| `rust/cuprum-rust` | Original `cdylib`/PyO3 module, argument validation, GIL release, error conversion, and the raw Python hand-off. | Approved integration boundary. Only this crate calls `adopt_writer` and `borrow_reader`; its generated PyO3 ABI code and explicit call-site safety comments remain subject to the FFI contract. | + +The safe writer path is `cuprum_native_io::with_owned_writer`: generic drop +elaboration owns the transferred writer on normal return, error, and real +unwind, while the reader is borrowed for the operation. After a normal callback +return, the scope explicitly drops the writer; unwind relies on automatic RAII. +`fd_ownership_model.rs` now lives under `cuprum-native-io` as historical model +code; it is not the current implementation and does not observe OS close +effects. The production retention kernel used by the Windows adapter and its +real-unwind regression is `memory::with_retained_owner`. + +## Current-operation inventory + +Table 2a. Remaining unsafe operations and their current callers. + +| Responsibility and source | Raw operation and local obligation | Caller and validation | +| -------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `cuprum-native-io::adopt_writer` (`src/lib.rs:57`) | Unix `OwnedFd::from_raw_fd` (`:61`) and Windows `OwnedHandle::from_raw_handle` (`:67`) require a valid, open, uniquely transferred resource that is never concurrently closed or reused. Drop closes once on normal, error, and real-unwind paths. | PyO3 call at `rust/cuprum-rust/src/stream_pyfunctions.rs:53`; native tests `ownership_tests::{transferred_writer_closes_and_delivers_eof,owned_descriptor_reads_and_closes}` exercise adoption. Kani checks typed drop counts; native tests check close and EOF. | +| `cuprum-native-io::borrow_reader` (`src/lib.rs:78`) | Unix and Windows `BorrowedStream::borrow_raw` (`:82`, `:88`) require the owner to keep the same resource live for the returned lifetime and prevent close/reuse during GIL release, cancellation, and the callback. | PyO3 calls at `rust/cuprum-rust/src/stream_pyfunctions.rs:57,89`; native regression `ownership_tests::borrowed_reader_survives` exercises the raw borrow contract. Safe `borrow` uses an existing `AsStream` lifetime. | +| Unix single-call I/O (`src/lib.rs:97,110`) | `libc::read` (`:101`) requires live descriptors and writable initialized storage; `libc::write` (`:114`) requires live descriptors and initialized readable storage. Checked conversion rejects out-of-bounds counts. | `cuprum-streams` owns retry, short-I/O, classification, and accounting policy. Syscall and OS guarantees remain trusted. | +| Linux `splice_once` (`src/lib.rs:136`) | `libc::splice` (`:143-152`) requires live pipe endpoints, valid length, and null offsets; partial results, `EINTR`, `EINVAL`, and broken-pipe drain must preserve state. | `cuprum-streams` selects fallback and drives the loop. Miri excludes unshimmed `libc::splice`; Linux native tests cover kernel effects. | +| Unix fixtures and observation (`src/lib.rs:160,184`) | `libc::pipe` (`:163`) fills two endpoint slots before `from_raw_fd` (`:169-173`) adopts them. `fcntl(F_GETFD)` (`:188`) observes close state and retries `EINTR` without granting ownership. | Native fixtures only. The fallible `descriptor_guard` `LockResult` serializes close observations in `ownership_tests.rs:16-20`. | +| Windows fixtures (`src/windows.rs:18,43`) | `CreatePipe` (`:23`) fills handle slots before `from_raw_handle` (`:28-32`) adopts them. `GetHandleInformation` (`:47`) observes state without ownership. | Windows tests are written; cross-Clippy passed, and hosted Windows runtime execution passed in CI runs. | +| Windows borrowed I/O (`src/windows.rs:60`) | `cap_std::fs::File::from_raw_handle` (`:67`) temporarily reconstructs a file; `with_retained_owner` prevents closing the borrowed handle on return or unwind. | Private adapter used only by `read_once`/`write_once`. Locked dependency is `cap-std 4.0.3`; `rust/dylint.toml` has no filesystem exclusion. | +| PyO3 generated ABI (`rust/cuprum-rust/src/stream_pyfunctions.rs:40-41,78-79`; module `src/lib.rs:78-82`) | `#[pyfunction]`, `#[pyo3(signature = ...)]`, `#[pymodule]`, and `wrap_pyfunction!` generate/register wrappers. `Python::detach` (`stream_pyfunctions.rs:26`) requires `Send`, no Python-state access, and valid resource lifetimes through completion. | Explicit raw calls are `adopt_writer` (`stream_pyfunctions.rs:53`) and `borrow_reader` (`:57,89`). PyO3/CPython/asyncio/GIL behaviour remains trusted. | + +There are no unsafe `Send`/`Sync` implementations or unsafe trait +implementations. Auto-trait results for typed handles and private `FnOnce` +callbacks, callback re-entrancy, panic/drop behaviour, and GIL release remain +explicit trust assumptions. + +## Generated FFI and cross-language handoff + +The following operations contain no explicit `unsafe` token in the baseline, +but are part of the trusted boundary and must be audited with the explicit +operations above: + +- `#[pyfunction]` and `#[pyo3(signature = ...)]` on + `rust/cuprum-rust/src/stream_pyfunctions.rs:40-41` and `78-79` generate + Python ABI wrappers. `#[pymodule]` at `rust/cuprum-rust/src/lib.rs:78-82` and + `wrap_pyfunction!` register those wrappers. The compile-pass and compile-fail + UI tests in `rust/cuprum-rust/tests/ui/` cover macro shape and return-type + contracts; they do not prove descriptor validity, ownership, or + generated-wrapper unwind behaviour. +- `Python::detach` at `rust/cuprum-rust/src/stream_pyfunctions.rs:26` releases + the Global Interpreter Lock (GIL) while native I/O runs. The detached + operation must be `Send`, must not access Python state, and must keep every + borrowed resource valid until completion. PyO3, CPython, and the + operating-system ABI remain trusted dependencies; Miri should exclude the + interpreter path and test the Rust seam separately. +- `cuprum/_pipeline_stream_fds.py` extracts transport descriptors, pauses the + reader, and restores it after native completion (`:28-79`, `:111-142`). It + changes both descriptors to blocking mode and records their previous modes + (`:145-208`). These steps establish the conditions for a blocking native + worker and must be undone even when setup or the worker fails. +- `cuprum/_pipeline_streams.py` duplicates the writer before executor + submission (`:209-252`), passes the reader and duplicate to Rust, and waits + for cleanup after cancellation (`:176-206`, `:277-312`). A duplicate that + fails setup or submission must be closed exactly once. Once submitted, Rust + owns the duplicate and the completion callback must wait for Rust's drop + rather than close a possibly released or reused descriptor. +- `cuprum/_streams_rs.py` validates buffer sizes before transfer + (`:114-167`), converts platform values, and on Windows duplicates a native + handle before closing the CRT descriptor (`:192-250`). A Python integer's + range is not proof of validity, lifetime, or exclusivity. The caller must + establish that the extracted reader stays open and caller-owned, that the + writer duplicate is unique, and that no concurrent transport close or FD + reuse occurs during the detached operation. + + `_prepare_native_reader` now checks the platform ABI representation before + writer ownership is transferred. Round 7 first recorded a red focused + regression run in `/tmp/issue379-reader-validation-before.log`; the corrected + hand-off passed 14 focused tests, including the boundary-contract script, in + `/tmp/issue379-round7-focused-tests.log`. This check establishes only that + the integer can be represented and is non-negative. It does not establish + that the resource is valid, remains alive, or is exclusively owned. Round + 22's extension-required suite then passed 80 tests with 1 skip after buffer + validation was restored before reader preparation; the round 21 property test + caught that precedence regression. It preserves POSIX invalid-buffer error + precedence and closes the duplicate on validation failure. + +These Python functions are not Rust unsafe sites, but their tests are part of +the Rust boundary contract. In particular, preserve pause/block/duplicate/ +submit/await-cleanup cancellation regressions, native-load and setup-failure +rollback tests, and Windows duplicate-handle cleanup tests. + +### Pinned PyO3 trust inventory + +`rust/Cargo.lock` pins `pyo3`, `pyo3-ffi`, `pyo3-macros`, and +`pyo3-macros-backend` at 0.29.2. The following is an audit of the pinned local +sources and declarations. `cargo-expand` is unavailable, so actual macro +expansion was not executed; the macro rows use the pinned code-generation +sources. Cuprum UI tests and Python ABI tests exercise the wrapper surface, but +do not prove PyO3 internals or the CPython ABI. + +| Trusted operation (pinned source) | Obligation, caller, and current evidence | +| ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Raw argument arrays and extraction: `pyo3/src/impl_/extract_argument.rs::{extract_arguments_fastcall,extract_arguments_tuple_dict,cast_function_argument,unwrap_required_argument}`; `pyo3-ffi/src/modsupport.rs::{PyArg_ParseTuple,PyArg_UnpackTuple}`; `core::hint::unreachable_unchecked` at the required-argument paths. | PyO3 macros must establish non-null, valid argument arrays and required-argument presence before the unsafe casts/unreachable branch. Cuprum's `rust/cuprum-rust/tests/ui/` compile tests cover signatures and error shapes; pointer validity and the macro invariant remain assumed. | +| Borrowed and owned `PyObject` pointers: `pyo3/src/instance.rs::{Bound::from_borrowed_ptr,Bound::from_owned_ptr,Py::clone_ref,Py::drop_ref}` and `pyo3/src/internal/state.rs::{register_decref,ReferencePool}`. | The GIL-attached caller must supply a live borrowed pointer or exactly one owned reference; each transfer must balance INCREF/DECREF, including deferred decref and finalization. Cuprum import/error tests exercise the integration, not PyO3's refcount implementation. | +| Method/module C ABI registration and static definitions: `pyo3/src/pyclass/create_type_object.rs::PyClassTypeBuilder::finalize_methods_and_properties`; `pyo3/src/impl_/pymodule.rs::{ModuleDef::init_multi_phase,__pyo3_pymodexport,__pyo3_pyinit}`; `pyo3-ffi/src/modsupport.rs::{PyModule_AddFunctions,PyModule_Create2}`. | Generated `PyMethodDef`/slot arrays, callback signatures, NUL-terminated names, and `PyInit_*`/Python 3.15 `PyModExport_*` symbols must match the selected CPython ABI and remain valid for module lifetime. Cuprum UI/import/build tests cover generated registration; ABI and loader guarantees are dependency assumptions. | +| Return, error, and null sentinels: `pyo3/src/impl_/callback.rs::{PyCallbackOutput,IntoPyCallbackOutput::convert}` and `pyo3/src/err/mod.rs::{PyErr::occurred,PyErr::take,PyErr::fetch}`. | A callback must return the sentinel required by its C signature exactly when the Python error indicator is set, and must never dereference a null result. Cuprum error-mapping tests cover observed wrapper behaviour; the C ABI's sentinel contract is assumed. | +| Panic trampoline and attach guard: `pyo3/src/impl_/trampoline.rs::{trampoline,trampoline_unraisable}`, `pyo3/src/impl_/panic.rs::PanicTrap`, and `pyo3/src/internal/state.rs::{AttachGuard::attach,AttachGuard::drop}`. | No Rust unwind may cross C; panic conversion, double-panic abort, attachment counts, and re-entrancy/finalization rules must hold. Cuprum real-unwind tests cover native resource cleanup only; PyO3's trampoline remains outside Cuprum proofs. | +| Detach and finalization: `pyo3/src/marker.rs::{Ungil,Python::detach}`, `pyo3/src/internal/state.rs::SuspendAttach`, `pyo3-ffi/src/ceval.rs::{PyEval_SaveThread,PyEval_RestoreThread}`, and `pyo3-ffi/src/pylifecycle.rs::Py_IsFinalizing`. | The detached closure and result must satisfy `Ungil`, use no Python state, restore the same thread state, and avoid attach/decref after finalization. `rust/cuprum-rust/src/stream_pyfunctions.rs:26` plus Python cancellation tests cover Cuprum's hand-off; thread-state, GIL, and interpreter-lifecycle guarantees are trusted PyO3/CPython assumptions. | + +## Verification matrix + +Table 3. Verification evidence by boundary. + +Round 35 integrated local checks are green: the Python suite recorded 1,568 +passes and one skip in `/tmp/issue379-round35-test.log`, the Rust suite +recorded 115/115 tests with no skips, and the extension-required suite recorded +80 passes and one skip. The nine unsafe-code compiler probes, Windows +cross-Clippy, and development build also passed. The cached Kani setup check +had already passed in Round 33, which remains retained in the preceding +reports. Hosted Windows/macOS runtime execution passed on 2026-09-15: the +`Rust boundary verification` run 35004943625 succeeded with its +`Native contracts (windows-2022)`, `Native contracts (macos-latest)`, and +`Native contracts (ubuntu-latest)` jobs, and the `CI` run 35004943782 succeeded +with its `Extension-gated tests (Windows Python/Rust boundary)` and +`Extension-gated tests (Python/Rust boundary)` jobs. The boundary workflow's +`extended` Kani/Miri job is skipped on pull requests by design, so these hosted +runs add no Kani/Miri coverage. The second CodeRabbit review remains pending; +the first review passed with 0 findings. + +| Boundary and production surface | Property | Selected tool and executed subject | Assumptions and bounds | Native or integration evidence | Status and limitation | +| ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `cuprum-native-io` borrowed and owned Unix/Windows handles | Borrowed readers remain caller-owned; transferred writers close once on normal, error, and real unwind; no released/reused descriptor is touched. | Kani exercises the production `with_owned_writer` drop counter where Verus cannot represent OS handles or destructors. The direct Verus resource assessment is recorded below. | Valid-live-resource, unique-transfer, and no-concurrent-close obligations remain caller contracts. Kani's symbolic exit is bounded and has coverage witnesses. The writer callback receives an immutable reference, so it cannot replace or move the owned resource out of the scope. | Round 35 Rust checks covered 115/115 tests with no skips. Hosted Windows/macOS runtime execution was green in CI. | Round 27 Kani passed 7 native harnesses; Miri passed 13 tests with 0 ignored; the fault harness detected all 4 mutations. | +| `cuprum-native-io` read/write kernels | Buffers expose only initialized in-bounds bytes; short reads/writes, `EINTR`, zero progress, and checked byte accounting preserve valid state. | Verus verifies the exact production `progress.rs` bodies rendered by `scripts/render_boundary_proofs.py`; Kani checks the production progress kernels and ownership seam. | `checked_progress` rejects overflow and oversized lengths; `record_write_progress` commits tail/count only after both checks. Aggregate pump/consume tracing counts retain saturating-add semantics, so they are not exact beyond `u64::MAX`. | Round 35 Rust checks covered 115/115 tests with no skips. Hosted Windows/macOS runtime execution was green in CI. | Verus passed 2 functions with 0 errors; Kani/Miri and native tests remain scoped as recorded above. Neither verifier proves syscall effects. | +| Linux `splice` adapter and fallback policy | Unsupported descriptors fall back; partial/interrupted transfers and broken-pipe drain preserve bytes and termination. | Kani/Verus cover the typed decision, progress, and accumulation seams; the kernel call itself is external. Miri covers applicable native crate paths but excludes unshimmed `libc::splice`. | Syscall return/error guarantees, symbolic chunk and trace bounds, and non-vacuity witnesses are explicit assumptions or bounds. | Round 35 local Rust and native checks passed; hosted Windows/macOS runtime execution was green in CI. | The syscall is not directly verified or Miri-interpreted because `libc::splice` has no supported shim. Round 27 Miri passed 13 tests with 0 ignored. | +| `cuprum-streams` safe policy and decoder | Safe crate and all test/helper targets reject unsafe Rust; state machine never writes after closure, stops on EOF, and checked UTF-8 replacement preserves valid prefixes. | Compiler contract checks the copied safe crate and compile tests. Kani covers pure pump policy; checked UTF-8 no longer contains `from_utf8_unchecked`. | No raw resource integers or pointers enter the safe crate; lifetime-bound typed native handles do enter through `AsStream`. Aggregate tracing counts retain saturating-add semantics; per-chunk checked progress is verified separately. | Round 35 Rust checks covered 115/115 tests with no skips; nine compiler probes passed. Hosted Windows/macOS runtime execution was green in CI. | Safe-policy/decoder Kani passed 11 harnesses. | +| `cuprum-rust` PyO3 integration | Generated wrappers expose stable signatures, map errors, release the GIL safely, and preserve the native resource contract. | PyO3 compile UI tests and Python integration tests. Verus/Miri exclude interpreter-generated ABI code; typed Rust seams are verified separately. | PyO3/CPython ABI, `Send`, interpreter lifecycle, and Python transport hand-off are trusted assumptions. | Round 35 Python checks recorded 1,568 passes and one skip; the extension-required suite recorded 80 passes and one skip. | Local integration checks are green; Python lifetime establishment remains outside this evidence. Hosted Windows/macOS runs pass. The first CodeRabbit review passed with 0 findings; the second review remains pending. | + +Round 22 also found a packaging boundary omission. A real `uv build --sdist` +archive initially omitted the Rust workspace, and +`uv run maturin sdist --manifest-path rust/cuprum-rust/Cargo.toml` initially +omitted `rust/rust-toolchain.toml`; the red evidence is in +`/tmp/issue379-native-sdist-before2.log`. The source include entries and the +parametrized `tests/test_native_sdist.py` regression now cover all three +workspace manifests, source trees, lock/toolchain metadata, and the absence of +`target` output. Round 24 ran both parametrized cases successfully in 26.25 +seconds (`2 passed`), and the extracted maturin source archive built a CPython +3.13 native wheel with all three crates compiling; see +`/tmp/issue379-round24-extracted-wheel.log`. The optional pure-Python wheel +backend is unchanged. + +The writer callback contract was tightened after an API audit found that an +`&mut` callback could use `mem::replace` to move the actual owned writer out of +its scope. The callback now takes `&OwnedStream`, and the normal path +explicitly drops the writer after the callback while unwind relies on automatic +RAII. The deliberate `replace_scoped_writer.rs` fixture reproduced the old +defect before the fix in `/tmp/issue379-writer-replace-before.log`; the +corrected compile regression passed with the expected `&W` versus `&mut W` +diagnostic in `/tmp/issue379-round25-trybuild-normal.log`. + +The tool records must include exact Rust, Verus, Kani, and Miri versions and +reproducible commands. A missing compatible binary is a dependency blocker, not +a passing skip. Miri runs should cover applicable isolated native targets and +in-memory seams; they must name every excluded PyO3, syscall, platform, or +unsupported-representation test and must not disable undefined-behaviour checks. + +The correspondence test in `scripts/tests/test_boundary_proof_render.py` checks +that the rendered proof bodies still match the production kernels. The current +Verus run used Verus `0.2026.09.06.8dea4a2`, Rust `1.98.0`, and external +prebuilt Z3 `4.16.0`. `make boundary-verus` first regenerates the input from +the exact production `progress.rs` bodies, then runs the pinned Verus binary. +The default `make test` suite includes `tests/test_native_sdist.py` and the +`scripts/tests/test_boundary_*.py` contract tests. `make boundary-test` runs +all four boundary script contracts together with the native tests. The current +final-source result is `2 verified, 0 errors` in +`/tmp/issue379-round24-verus.log`. The Verus archive does not contain the +solver, so `scripts/install_boundary_z3.py` installs the required prebuilt Z3 +separately. The `leynos/rust-prover-tools` installer works for Verus, but its +wrapper's version parser rejects the annotation +`(overridden by environment variable RUSTUP_TOOLCHAIN)`; the Makefile uses the +pinned binary directly rather than silently changing toolchains. + +The pinned Z3 asset targets x86-64 Linux with glibc 2.39. Hosted execution +assumes that GitHub's `ubuntu-latest` image currently resolves to Ubuntu 24.04 +([runner-images README](https://github.com/actions/runner-images/blob/main/README.md)). +The installer and workflow must fail closed when the architecture, libc, or +image is incompatible; an unavailable compatible binary is a dependency +blocker, never a passing skip. + +The direct Verus assessment of the unchanged production `adopt_writer` body +failed because `OwnedFd` and `FromRawFd::from_raw_fd` are unsupported +representations in this Verus version. The exact diagnostic is recorded in +`/tmp/issue379-verus-resource-assessment.log`. This is a representation blocker +for direct resource-boundary verification, not a proof of the unsafe +constructor. A separate generic memory assessment also lacks callback +preconditions; that incompleteness is not the sole reason for the Kani +fallback. Kani covers the typed Rust-owned drop seam, while native tests cover +OS effects and real unwind. + +The reproducible resource assessment is: + +```bash +uv run python scripts/render_boundary_proofs.py +VERUS_Z3_PATH="$PWD/.cache/boundary-z3/z3" \ + RUSTUP_TOOLCHAIN=1.98.0 \ + "$HOME/.local/share/cuprum-verus-0.2026.09.06.8dea4a2/verus/verus" \ + rust/target/boundary-verification/resource-assessment.rs --crate-type=lib +``` + +The renderer creates `resource-assessment.rs` from the unchanged production +`adopt_writer` body. Its expected unsupported `OwnedFd`/`FromRawFd` diagnostic +is an assessment limitation, neither a proof nor a successful skip. + +Kani is pinned to `0.67.0` with compiler +`rustc 1.93.0-nightly (53732d5e0 2025-11-20)` and CBMC `6.8.0`. The final +native boundary run passed 7 harnesses with 0 failures in +`/tmp/issue379-round27-kani-native.log`; the 11 safe-policy/decoder harnesses +remain current and passed in the complete run recorded at +`/tmp/issue379-round24-kani.log`. The native ownership proofs enumerate +`Normal` and `Error` exits; only `repeated_borrows_never_close_the_reader` +loops, with `borrows <= 3` and `#[kani::unwind(4)]`. Production progress proofs +use symbolic `usize` `count`/`capacity` and `u64` `total`/`remaining`/ +`written`, and cover EOF, short progress, invalid lengths, and overflow. Safe +pump proofs use symbolic `PumpState`, `usize` read lengths, and `u64` write +counts; their three-step proof has `#[kani::unwind(4)]`. UTF-8 proofs use fixed +three- and four-byte arrays, with unwind bounds 5 for the first four harnesses +and 4 for the final prefix harness. These are bounded results, not an unbounded +proof. Verus is pinned to `0.2026.09.06.8dea4a2` with Rust `1.98.0` and +prebuilt Z3 `4.16.0`; the production progress proof passed 2 functions with 0 +errors in `/tmp/issue379-round24-verus.log`. Miri is pinned to +`nightly-2026-08-07` with `rustc 1.99.0-nightly (84b36a78a 2026-08-06)`; the +final native run passed 13 tests with 0 ignored in +`/tmp/issue379-round27-miri.log`. PyO3, unshimmed `libc::splice`, unsupported +syscall representations, and other excluded targets remain outside that Miri +run. + +Round 28 unit and round 29 behavioural runs still report unexplained native +payload mismatches. The isolated diagnostics pass, but no production fix is +claimed while the baseline comparison runs; see the +[issue 379 parity debugging plan](debugging/debugging-plan-20260908-issue379-parity.md). +An archived H7 baseline also produced empty `HELLO` output with a two-stage +exit status of zero, so the symptom predates the boundary extraction. +Separately, the H9 regression showed that an asyncio transport can silently +accept `pause_reading()` while closing; `_pause_reader_transport` now declines +with `READER_PAUSE_FAILED` before raw borrowing, preserving the buffered prefix +in the Python fall-back. This closes the transport race but does not prove that +it caused the historical payload mismatch; the real-EOF regression is recorded +in `/tmp/issue379-reader-lease-before.log` and `after.log`. + +## Trusted assumptions and fault sensitivity + +The trusted-code inventory currently includes the Rust standard-library drop +implementations for `OwnedFd` and Windows `File`, libc and OS syscall +contracts, the Windows Kernel32 handle API, PyO3's generated ABI glue, +CPython's GIL/thread contract, and Python asyncio transport behaviour. Native +tests can observe some of these effects; Verus and Kani must not present those +assumptions as proofs. The historical Kani ownership model is valuable evidence +for the Rust drop structure and now calls the production +`memory::with_retained_owner` kernel, but it covers only `Normal` and `Error` +outcomes. It makes no simulated panic-unwind claim, and its close log is not an +OS close. The real `catch_unwind` native tests remain authoritative for panic +unwind and resource effects. + +The fault harness exercises invalid bounds, incorrect accounting, leaked +writers, and the historical trailing-`mem::forget` unwind defect. The final run +detected all four mutations while its controls passed; see +`/tmp/issue379-round27-faults.log`. `make boundary-faults` regenerates a +disposable copy, runs controls before the mutations, and archives the results +for the scheduled Kani job. Real-descriptor tests remain authoritative for +descriptor state, close effects, EOF delivery, and actual panic unwind; models +and proof seams must identify exactly which production kernel they exercise. + +Kani's decoder `Vec`/`String` harnesses also rely on its allocator and +intrinsic models: the logs abstract allocation failure through `assume(false)` +in `alloc::raw_vec::handle_error`/`reserve` and `intrinsics::ToISize`, while +the successful bounded allocations are the exercised domain. They do not cover +out-of-memory behaviour or process abort. + +Existing verification provenance must be retained: + +- [issue 80] +- [issue 81] +- [issue 84] +- [issue 89] +- [issue 125] +- [issue 233] + +The scheduled Loom interleaving harness remains a separate deliverable. + +[issue 379]: https://github.com/leynos/cuprum/issues/379 +[issue 80]: https://github.com/leynos/cuprum/issues/80 +[issue 81]: https://github.com/leynos/cuprum/issues/81 +[issue 84]: https://github.com/leynos/cuprum/issues/84 +[issue 89]: https://github.com/leynos/cuprum/issues/89 +[issue 125]: https://github.com/leynos/cuprum/issues/125 +[issue 233]: https://github.com/leynos/cuprum/issues/233 + +## Hosted installation and code-health follow-up + +The first hosted boundary run, `34247183097`, passed Ubuntu and Windows native +contracts. Its Verus installation failed because `UV_PYTHON=3.13` selected an +interpreter incompatible with the pinned `rust-prover-tools` package, which +requires Python 3.14. The Makefile now selects that installer runtime +explicitly; `test_prover_tools_selects_its_required_python` failed before this +correction. The failure log is `/tmp/issue379-hosted-verus-failure.log`. A +fresh hosted proof run remains required; a successful cached local installation +is not its substitute. + +CodeScene identified duplicated platform dispatch and complex verification +orchestration. Windows I/O now lives beside its handle adapter, and the scripts +separate response classification, redirect traversal, progress faults, +ownership faults, and unsafe-probe classification. All five affected files +scored 10.0 without findings in `/tmp/issue379-codescene-after-*.json`. The +underlying ownership and progress kernels, symbolic domains, and unsafe +contracts are unchanged. Fresh deterministic gates and a further CodeRabbit +review remain required before publishing these follow-up changes. diff --git a/docs/users-guide.md b/docs/users-guide.md index c65f380dd..5c6398da6 100644 --- a/docs/users-guide.md +++ b/docs/users-guide.md @@ -1862,6 +1862,21 @@ place — a non-integer, or a Python integer outside the signed 64-bit range — may instead fail earlier, during PyO3 argument conversion, with a different exception. +The internal pump validates the buffer size and reader ABI representation +before transferring ownership of its duplicated writer. If either check fails, +the duplicate is closed before native work begins. These checks establish only +representability and error ordering; they do not prove that a descriptor or +handle is valid, remains live, or is exclusively owned. The pipeline keeps the +reader paused and waits for native cleanup after cancellation so those lifetime +obligations remain in force. + +Before borrowing the reader's raw descriptor, the pipeline rejects an asyncio +transport that is already closing. Asyncio may silently accept +`pause_reading()` while a queued close callback can still close the descriptor; +the hop therefore reports `READER_PAUSE_FAILED` and uses the Python fall-back, +which retains the reader's buffered prefix. This protects the hand-off contract +without identifying the cause of any historical native payload mismatch. + ### Rust stream consumption (internal) The Rust extension also exposes `cuprum._streams_rs.rust_consume_stream`, which @@ -1932,18 +1947,29 @@ Failures that never reached the operating system — an internal overflow or bounds condition — surface as a plain `OSError` with a descriptive message and no `errno`, because there is no system code to report. +The scratch buffer each helper reads or writes through is allocated fallibly, +so requesting a `buffer_size` the allocator cannot satisfy is one of those +systemless failures rather than a crash: it raises an `OSError` with the stable +message `failed to allocate the stream buffer` and no `errno`. It is an error, +not a signal, so it unwinds normally and never aborts the process embedding the +extension. The production cap of 1 GiB (see `MAX_BUFFER_SIZE`) means a request +large enough to be refused is far outside the range the helpers are designed to +serve, so the message is the stable interface here — unlike a system failure, +there is no number to branch on. + ### Rust stream observability (internal) Both internal helpers emit `tracing` diagnostics; the crate installs no subscriber, so the embedding application owns subscriber configuration. Each successful read or write logs a `debug` event (with the byte count and platform), every `EINTR` retry logs a `warn`, and fatal I/O failures, -zero-progress writes, and length-conversion overflows log an `error`. The pump -and consume loops run inside an operation span that carries the `operation` and -`buffer_size` fields and, on completion, records `total_bytes` and the -cumulative `EINTR` `read_retries`/`write_retries` counts. The span sits at -`error` level so the `warn`/`error` events retain their operation context even -when the subscriber is filtered to `warn`/`error`; it emits no log line itself. +zero-progress writes, length-conversion overflows, and refused scratch-buffer +allocations log an `error`. The pump and consume loops run inside an operation +span that carries the `operation` and `buffer_size` fields and, on completion, +records `total_bytes` and the cumulative `EINTR` `read_retries`/`write_retries` +counts. The span sits at `error` level so the `warn`/`error` events retain +their operation context even when the subscriber is filtered to `warn`/`error`; +it emits no log line itself. One further `debug` event reports the pump's own state rather than an individual read or write. When a downstream stage hangs up early — the @@ -1978,6 +2004,29 @@ itself succeeded. Both the `splice` fast path and the read/write fallback emit the event identically, so the message does not depend on which path handled the transfer. +One `error` event reports a refusal rather than a kernel failure, so it carries +a stable category instead of a system message: + +```text +scratch buffer allocation refused + error_category=buffer_allocation_failed + buffer_size= platform= +``` + +`error_category` is always `buffer_allocation_failed` for this event. The +event's `buffer_size` is the requested size rounded up to a power-of-two bucket +— a bounded magnitude, not the exact caller-supplied number and not any +allocator internal. `platform` matches the field on the read/write events. The +event is emitted from inside the operation span, so the span's `operation` and +exact requested `buffer_size` fields are in scope alongside it, exactly as for +the fatal I/O failures above. + +This event accompanies the `OSError` described under +[Rust stream error handling (internal)](#rust-stream-error-handling-internal): +the same refused allocation produces both. A caught exception then still leaves +a trace for the subscriber, rather than disappearing with the caller's `except` +clause. + ### Why a hop fell back to Python Selecting the `rust` backend does not guarantee every inter-stage hop takes it. diff --git a/pyproject.toml b/pyproject.toml index aac482c35..52b745214 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -394,6 +394,8 @@ also_copy = [ "CHANGELOG.md", "Makefile", "rust/cuprum-rust/", + "rust/cuprum-streams/", + "rust/cuprum-native-io/", "rust/Cargo.toml", "rust/Cargo.lock", "rust/dylint.toml", @@ -480,6 +482,21 @@ build-backend = "uv_build" [tool.uv.build-backend] module-root = "" source-exclude = ["cuprum/unittests/**"] +# Ship the optional native workspace without including Cargo build artefacts. +source-include = [ + "rust/Cargo.toml", + "rust/Cargo.lock", + "rust/rust-toolchain.toml", + "rust/dylint.toml", + "rust/cuprum-rust/Cargo.toml", + "rust/cuprum-rust/src/**", + "rust/cuprum-rust/tests/**", + "rust/cuprum-streams/Cargo.toml", + "rust/cuprum-streams/src/**", + "rust/cuprum-streams/tests/**", + "rust/cuprum-native-io/Cargo.toml", + "rust/cuprum-native-io/src/**", +] [tool.maturin] bindings = "pyo3" @@ -487,3 +504,7 @@ manifest-path = "rust/cuprum-rust/Cargo.toml" module-name = "cuprum._rust_backend_native" python-source = "." exclude = ["cuprum/unittests/**"] +include = [ + { path = "rust/rust-toolchain.toml", format = "sdist" }, + { path = "rust/dylint.toml", format = "sdist" }, +] diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 76fd3f9eb..4f6314890 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -11,6 +11,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "ambient-authority" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b" + [[package]] name = "autocfg" version = "1.5.1" @@ -38,21 +44,74 @@ version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +[[package]] +name = "cap-primitives" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b5f74729fd2f44701d1a8eb47e906cdb3ccd9ec0f02baad85a744b791940b18" +dependencies = [ + "ambient-authority", + "fs-set-times", + "io-extras", + "io-lifetimes 3.0.1", + "ipnet", + "maybe-owned", + "rustix", + "rustix-linux-procfs", + "windows-sys 0.61.2", + "winx", +] + +[[package]] +name = "cap-std" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ec78e242cfa2cfe276807ac2ecc00315a6c97786977414bcd1c3963b6c91b8" +dependencies = [ + "cap-primitives", + "io-extras", + "io-lifetimes 3.0.1", + "rustix", +] + [[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cuprum-native-io" +version = "0.1.0" +dependencies = [ + "cap-std", + "libc", + "rstest 0.26.1", + "windows-sys 0.61.2", +] + [[package]] name = "cuprum-rust" version = "0.1.0" dependencies = [ + "cuprum-native-io", + "cuprum-streams", + "proptest", + "pyo3", + "rstest 0.27.0", + "trybuild", +] + +[[package]] +name = "cuprum-streams" +version = "0.1.0" +dependencies = [ + "cap-std", + "cuprum-native-io", "insta", "libc", "proptest", - "pyo3", - "rstest", + "rstest 0.26.1", "tempfile", "thiserror", "tracing", @@ -78,7 +137,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -93,12 +152,34 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "fs-set-times" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" +dependencies = [ + "io-lifetimes 2.0.4", + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "futures-core" version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "futures-task" version = "0.3.32" @@ -118,8 +199,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-core", + "futures-macro", "futures-task", "pin-project-lite", + "slab", ] [[package]] @@ -184,6 +267,34 @@ dependencies = [ "tempfile", ] +[[package]] +name = "io-extras" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f" +dependencies = [ + "io-lifetimes 3.0.1", + "windows-sys 0.60.2", +] + +[[package]] +name = "io-lifetimes" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06432fb54d3be7964ecd3649233cddf80db2832f47fec34c01f65b3d9d774983" + +[[package]] +name = "io-lifetimes" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0fb0570afe1fed943c5c3d4102d5358592d8625fda6a0007fdbe65a92fba96" + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + [[package]] name = "itoa" version = "1.0.18" @@ -202,6 +313,12 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "maybe-owned" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" + [[package]] name = "memchr" version = "2.8.3" @@ -438,6 +555,17 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" +[[package]] +name = "rstest" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" +dependencies = [ + "futures-timer", + "futures-util", + "rstest_macros 0.26.1", +] + [[package]] name = "rstest" version = "0.27.0" @@ -446,7 +574,25 @@ checksum = "948203e6d13b83e51a90d7cf236dd58a0065f23f4b902f00f306e7eb2bd09b9c" dependencies = [ "futures-timer", "futures-util", - "rstest_macros", + "rstest_macros 0.27.0", +] + +[[package]] +name = "rstest_macros" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn 2.0.118", + "unicode-ident", ] [[package]] @@ -486,7 +632,17 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustix-linux-procfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fc84bf7e9aa16c4f2c758f27412dc9841341e16aa682d9c7ac308fe3ee12056" +dependencies = [ + "once_cell", + "rustix", ] [[package]] @@ -564,6 +720,12 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "syn" version = "2.0.118" @@ -608,7 +770,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -762,7 +924,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -771,6 +933,24 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -780,6 +960,135 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "winnow" version = "1.0.4" @@ -789,6 +1098,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winx" +version = "0.36.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" +dependencies = [ + "bitflags", + "windows-sys 0.59.0", +] + [[package]] name = "wit-bindgen" version = "0.57.1" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 825fac89e..5ac168553 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "2" -members = ["cuprum-rust"] +members = ["cuprum-rust", "cuprum-streams", "cuprum-native-io"] [workspace.lints.clippy] pedantic = { level = "warn", priority = -1 } diff --git a/rust/cuprum-native-io/Cargo.toml b/rust/cuprum-native-io/Cargo.toml new file mode 100644 index 000000000..553cc2534 --- /dev/null +++ b/rust/cuprum-native-io/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "cuprum-native-io" +version = "0.1.0" +edition = "2024" +license = "ISC" +description = "Audited native resource and syscall boundary for Cuprum" +publish = false + +[target.'cfg(unix)'.dependencies] +libc = "0.2.186" + +[dev-dependencies] +rstest = "0.26.1" + +[lints] +workspace = true + +[target.'cfg(windows)'.dependencies] +cap-std = "4.0.2" +windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Security", "Win32_System_Pipes"] } diff --git a/rust/cuprum-rust/src/fd_ownership_kani_proofs.rs b/rust/cuprum-native-io/src/fd_ownership_kani_proofs.rs similarity index 84% rename from rust/cuprum-rust/src/fd_ownership_kani_proofs.rs rename to rust/cuprum-native-io/src/fd_ownership_kani_proofs.rs index a35387180..57ee51bfa 100644 --- a/rust/cuprum-rust/src/fd_ownership_kani_proofs.rs +++ b/rust/cuprum-native-io/src/fd_ownership_kani_proofs.rs @@ -2,8 +2,8 @@ //! //! The invariant under proof, over every modelled exit path: //! -//! - a reader passed through `with_borrowed_reader` is **never** closed by -//! Rust — it stays caller-owned on normal completion and on unwind; +//! - a reader passed through the production retention kernel is **never** closed by +//! Rust — it stays caller-owned on normal completion and error return; //! - the `pump_stream` writer is **consumed** and closes exactly once on //! every exit path, which is what signals EOF downstream. //! @@ -11,7 +11,7 @@ //! [`super::fd_ownership_model`], not over real descriptors: Kani does not //! interpret I/O, so no real `close(2)` effect is exercised here. Actual //! descriptor behaviour stays covered by the real-FD regression tests in -//! `lib_tests.rs`. Tracking issue: . +//! `ownership_tests.rs`. Tracking issue: . //! //! Bounds. The exit space is finite and fully enumerated (two modes, both //! reached — see the `kani::cover!` statements). Only @@ -28,7 +28,7 @@ fn any_exit_mode() -> ExitMode { if kani::any::() { ExitMode::Normal } else { - ExitMode::Unwind + ExitMode::Error } } @@ -45,7 +45,10 @@ fn pump_borrows_reader_and_consumes_writer() { // Prove both boundary cases are reachable, so neither assertion below is // vacuously satisfied by an unreachable branch. kani::cover!(exit == ExitMode::Normal, "reaches normal completion"); - kani::cover!(exit == ExitMode::Unwind, "reaches the modelled unwind exit"); + kani::cover!( + exit == ExitMode::Error, + "reaches the operation error return" + ); let outcome = model_pump_stream(ModelFd::new(&reader_log), ModelFd::new(&writer_log), exit); @@ -58,8 +61,8 @@ fn pump_borrows_reader_and_consumes_writer() { "the pump writer is consumed and closes exactly once, signalling EOF", ); kani::assert( - outcome.is_err() == (exit == ExitMode::Unwind), - "the modelled unwind exit propagates out of the helper", + outcome.is_err() == (exit == ExitMode::Error), + "the operation error return propagates out of the helper", ); } @@ -73,7 +76,10 @@ fn consume_borrows_its_only_reader() { let exit = any_exit_mode(); kani::cover!(exit == ExitMode::Normal, "reaches normal completion"); - kani::cover!(exit == ExitMode::Unwind, "reaches the modelled unwind exit"); + kani::cover!( + exit == ExitMode::Error, + "reaches the operation error return" + ); let outcome = model_consume_stream(ModelFd::new(&reader_log), exit); @@ -82,8 +88,8 @@ fn consume_borrows_its_only_reader() { "a borrowed reader FD is never closed by Rust, on any exit path", ); kani::assert( - outcome.is_err() == (exit == ExitMode::Unwind), - "the modelled unwind exit propagates out of the helper", + outcome.is_err() == (exit == ExitMode::Error), + "the operation error return propagates out of the helper", ); } @@ -112,7 +118,7 @@ fn repeated_borrows_never_close_the_reader() { let outcome = model_with_borrowed_reader(ModelFd::new(&reader_log), |_reader| exit.outcome()); kani::assert( - outcome.is_err() == (exit == ExitMode::Unwind), + outcome.is_err() == (exit == ExitMode::Error), "each borrow reports its own exit mode", ); } diff --git a/rust/cuprum-rust/src/fd_ownership_model.rs b/rust/cuprum-native-io/src/fd_ownership_model.rs similarity index 62% rename from rust/cuprum-rust/src/fd_ownership_model.rs rename to rust/cuprum-native-io/src/fd_ownership_model.rs index b39fa960d..67b2c8359 100644 --- a/rust/cuprum-rust/src/fd_ownership_model.rs +++ b/rust/cuprum-native-io/src/fd_ownership_model.rs @@ -1,30 +1,13 @@ -//! Pure ownership model for the borrowed-reader / consumed-writer contract. +//! Close-effect model instantiated with the production retained-owner kernel. //! -//! Kani cannot model real operating-system descriptor effects: closing an FD -//! is I/O, which the bounded model checker does not interpret. This module -//! therefore replaces the descriptor with [`ModelFd`], whose `Drop` records a -//! close in a [`CloseLog`] instead of issuing one. Everything else — the -//! [`ManuallyDrop`] wrapper, the closure call, the early-exit edge — is real -//! Rust, so Rust's own drop elaboration decides the outcome rather than any -//! handwritten accounting. -//! -//! # Modelling the unwind edge -//! -//! The hazard `with_borrowed_reader` exists to prevent only appears when a -//! panic unwinds *through the helper's own frame*: the superseded -//! `mem::forget`-after-the-call pattern was skipped on that path, so the -//! caller-owned reader was dropped and closed. Kani builds with panics as -//! aborts, so a literal `panic!` cannot be used to reach that path. -//! -//! [`ModelUnwind`] and the `?` operator stand in for it. Both a `?` early -//! return and a real unwind leave the frame *without executing the statements -//! that follow the operation*, and both run drop glue for every live local. -//! That correspondence is what gives the proofs their teeth: a `mem::forget` -//! placed after the operation is skipped by `?` exactly as it is skipped by an -//! unwind, so reintroducing the old pattern makes the proofs fail. +//! `ModelFd` substitutes an observable drop counter for an OS descriptor. +//! The production helper executes unchanged; only the resource is modelled. +//! Normal and error returns are checked here. Neither OS close effects nor +//! real panic-unwind are proved by this model: native regressions exercise +//! those paths, including the historical trailing-`mem::forget` mistake. +use crate::memory::with_retained_owner; use core::cell::Cell; -use core::mem::ManuallyDrop; /// Records how many times the modelled descriptor was closed. /// @@ -75,43 +58,35 @@ impl Drop for ModelFd<'_> { } } -/// Stands in for a panic-unwind leaving the frame early. +/// Represents an ordinary operation error, not panic-unwind. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct ModelUnwind; +pub(crate) struct ModelError; /// How a modelled operation leaves its scope. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum ExitMode { /// The operation ran to completion and returned normally. Normal, - /// The operation aborted, standing in for a panic-unwind through the - /// helper's frame. - Unwind, + /// The operation returned an error. + Error, } impl ExitMode { /// Outcome an operation returns under this exit mode. - pub(crate) const fn outcome(self) -> Result<(), ModelUnwind> { + pub(crate) const fn outcome(self) -> Result<(), ModelError> { match self { Self::Normal => Ok(()), - Self::Unwind => Err(ModelUnwind), + Self::Error => Err(ModelError), } } } -/// Model of `with_borrowed_reader`. -/// -/// Structurally identical to production: reconstruct the handle, wrap it in -/// [`ManuallyDrop`], run the operation. The `?` is the modelled unwind edge — -/// it leaves this frame before any trailing statement could run, so the -/// wrapper is the only thing standing between the reader and a close. +/// Run the production retained-owner kernel with an observable resource. pub(crate) fn model_with_borrowed_reader<'log, T>( fd: ModelFd<'log>, - operation: impl FnOnce(&mut ModelFd<'log>) -> Result, -) -> Result { - let mut handle = ManuallyDrop::new(fd); - let value = operation(&mut handle)?; - Ok(value) + operation: impl FnOnce(&mut ModelFd<'log>) -> Result, +) -> Result { + with_retained_owner(fd, operation) } /// Model of `pump_stream`'s descriptor handling. @@ -123,7 +98,7 @@ pub(crate) fn model_pump_stream( reader: ModelFd<'_>, writer: ModelFd<'_>, exit: ExitMode, -) -> Result<(), ModelUnwind> { +) -> Result<(), ModelError> { // Held to the end of the scope so it drops on the normal return and on // the early exit alike, exactly as the real writer handle does. let _writer_handle = writer; @@ -134,7 +109,7 @@ pub(crate) fn model_pump_stream( /// /// `consume_stream` takes no writer at all: the reader is its only /// descriptor, and it is borrowed. -pub(crate) fn model_consume_stream(reader: ModelFd<'_>, exit: ExitMode) -> Result<(), ModelUnwind> { +pub(crate) fn model_consume_stream(reader: ModelFd<'_>, exit: ExitMode) -> Result<(), ModelError> { model_with_borrowed_reader(reader, |_reader_handle| exit.outcome()) } @@ -148,7 +123,7 @@ mod tests { #[rstest] #[case(ExitMode::Normal)] - #[case(ExitMode::Unwind)] + #[case(ExitMode::Error)] fn pump_borrows_reader_and_consumes_writer(#[case] exit: ExitMode) { let reader_log = CloseLog::new(); let writer_log = CloseLog::new(); @@ -161,19 +136,19 @@ mod tests { 1, "writer must be consumed exactly once" ); - assert_eq!(outcome.is_err(), exit == ExitMode::Unwind); + assert_eq!(outcome.is_err(), exit == ExitMode::Error); } #[rstest] #[case(ExitMode::Normal)] - #[case(ExitMode::Unwind)] + #[case(ExitMode::Error)] fn consume_borrows_its_only_reader(#[case] exit: ExitMode) { let reader_log = CloseLog::new(); let outcome = model_consume_stream(ModelFd::new(&reader_log), exit); assert_eq!(reader_log.closes(), 0, "borrowed reader must stay open"); - assert_eq!(outcome.is_err(), exit == ExitMode::Unwind); + assert_eq!(outcome.is_err(), exit == ExitMode::Error); } #[test] diff --git a/rust/cuprum-native-io/src/kani_proofs.rs b/rust/cuprum-native-io/src/kani_proofs.rs new file mode 100644 index 000000000..a0af61759 --- /dev/null +++ b/rust/cuprum-native-io/src/kani_proofs.rs @@ -0,0 +1,84 @@ +//! Bounded proofs over production kernels, with explicit reachability checks. +//! +//! No OS calls occur here. Drop counters witness Rust drop elaboration, not +//! kernel close effects; early errors are not claims about actual unwind. + +use core::cell::Cell; + +use super::{ + progress::{checked_count, checked_progress}, + with_owned_writer, +}; + +struct Resource<'a>(&'a Cell); + +impl Drop for Resource<'_> { + fn drop(&mut self) { + self.0.set(self.0.get() + 1); + } +} + +#[kani::proof] +fn production_scope_preserves_reader_and_drops_writer() { + let reader_closes = Cell::new(0); + let writer_closes = Cell::new(0); + let reader = Resource(&reader_closes); + let fail: bool = kani::any(); + let result = with_owned_writer(&reader, Resource(&writer_closes), |_reader, _writer| { + if fail { Err(()) } else { Ok(()) } + }); + kani::cover!(result.is_err(), "operation error is reachable"); + kani::cover!(result.is_ok(), "normal completion is reachable"); + assert_eq!(reader_closes.get(), 0); + assert_eq!(writer_closes.get(), 1); + drop(reader); + assert_eq!(reader_closes.get(), 1); +} + +#[kani::proof] +fn counts_stay_in_bounds() { + let count: usize = kani::any(); + let capacity: usize = kani::any(); + let result = checked_count(count, capacity); + assert_eq!(result.is_some(), count <= capacity); + if let Some(value) = result { + assert_eq!(value, count); + } + kani::cover!(count == 0 && result.is_some(), "EOF"); + kani::cover!( + count > 0 && count < capacity && result.is_some(), + "short IO" + ); + kani::cover!(result.is_none(), "invalid external count rejected"); +} + +#[kani::proof] +fn progress_is_exact_or_rejected() { + let total: u64 = kani::any(); + let remaining: u64 = kani::any(); + let written: u64 = kani::any(); + let result = checked_progress(total, remaining, written); + assert_eq!( + result.is_some(), + written <= remaining && u128::from(total) + u128::from(written) <= u128::from(u64::MAX) + ); + if let Some((new_total, tail)) = result { + assert_eq!( + u128::from(new_total), + u128::from(total) + u128::from(written) + ); + assert_eq!( + u128::from(tail) + u128::from(written), + u128::from(remaining) + ); + } + kani::cover!(result.is_some() && written > 0, "positive progress"); + kani::cover!( + result.is_none() && written <= remaining, + "overflow rejection" + ); + kani::cover!( + result.is_none() && written > remaining, + "invalid length rejection" + ); +} diff --git a/rust/cuprum-native-io/src/lib.rs b/rust/cuprum-native-io/src/lib.rs new file mode 100644 index 000000000..7d15cd03a --- /dev/null +++ b/rust/cuprum-native-io/src/lib.rs @@ -0,0 +1,213 @@ +//! Audited resource ownership and single-call native I/O for Cuprum. +//! +//! Safe operations accept lifetime-bound OS borrows. Only the Python +//! integration boundary may reconstruct resources from raw integers. +#![deny(unsafe_op_in_unsafe_fn)] + +#[cfg(unix)] +use std::io; +#[cfg(kani)] +mod kani_proofs; +mod memory; +pub mod progress; +pub use memory::with_owned_writer; +#[cfg(kani)] +mod fd_ownership_kani_proofs; +#[cfg(any(test, kani))] +mod fd_ownership_model; + +#[cfg(unix)] +pub use std::os::fd::{AsFd as AsStream, BorrowedFd as BorrowedStream, OwnedFd as OwnedStream}; +#[cfg(windows)] +pub use std::os::windows::io::{ + AsHandle as AsStream, BorrowedHandle as BorrowedStream, OwnedHandle as OwnedStream, +}; + +#[cfg(unix)] +use std::os::fd::{AsRawFd, FromRawFd}; +#[cfg(windows)] +use std::os::windows::io::{FromRawHandle, RawHandle}; + +/// Integer representation at the Python ABI boundary, not an ownership token. +#[cfg(unix)] +pub type PlatformFd = i32; +/// Pointer-width integer representation at the Python ABI boundary. +#[cfg(windows)] +pub type PlatformFd = usize; + +/// Borrow a resource without transferring or extending its ownership. +pub fn borrow(stream: &impl AsStream) -> BorrowedStream<'_> { + #[cfg(unix)] + { + stream.as_fd() + } + #[cfg(windows)] + { + stream.as_handle() + } +} + +/// Accept the uniquely owned writer transferred by the integration boundary. +/// +/// # Safety +/// `raw` must denote a valid open resource owned exclusively by the caller. +/// The caller relinquishes ownership and must never close or reuse it after +/// this call. Range validation alone does not establish these obligations. +#[must_use] +pub unsafe fn adopt_writer(raw: PlatformFd) -> OwnedStream { + #[cfg(unix)] + { + // SAFETY: the caller transfers a valid, uniquely owned descriptor. + unsafe { OwnedStream::from_raw_fd(raw) } + } + #[cfg(windows)] + { + // SAFETY: the caller transfers a valid independently duplicated Win32 + // handle, not a CRT descriptor. The cast preserves pointer width. + unsafe { OwnedStream::from_raw_handle(raw as RawHandle) } + } +} + +/// Borrow a raw reader whose owner is maintained by the integration layer. +/// +/// # Safety +/// `raw` must stay open and refer to the same resource throughout `'owner`. +/// The caller must prevent close/reuse, including during GIL release and +/// cancellation. It retains responsibility for closing the resource. +#[must_use] +pub const unsafe fn borrow_reader<'owner>(raw: PlatformFd) -> BorrowedStream<'owner> { + #[cfg(unix)] + { + // SAFETY: the caller guarantees validity for the returned lifetime. + unsafe { BorrowedStream::borrow_raw(raw) } + } + #[cfg(windows)] + { + // SAFETY: the caller guarantees validity for the returned lifetime; + // the integer is a pointer-width Win32 handle, not a CRT descriptor. + unsafe { BorrowedStream::borrow_raw(raw as RawHandle) } + } +} + +/// Read once into initialized storage, retaining the resource borrow. +/// +/// # Errors +/// Returns the native I/O error, including interruption, without retrying. +#[cfg(unix)] +pub fn read_once(stream: BorrowedStream<'_>, buffer: &mut [u8]) -> io::Result { + // SAFETY: the descriptor borrow remains live; the exclusive slice is + // writable for its length. read does not retain the buffer pointer. + let result = + unsafe { libc::read(stream.as_raw_fd(), buffer.as_mut_ptr().cast(), buffer.len()) }; + syscall_result(result, buffer.len()) +} + +/// Write once from initialized storage, retaining the resource borrow. +/// +/// # Errors +/// Returns the native I/O error, including interruption, without retrying. +#[cfg(unix)] +pub fn write_once(stream: BorrowedStream<'_>, buffer: &[u8]) -> io::Result { + // SAFETY: the descriptor borrow remains live; the shared slice is + // readable for its length. write does not retain or mutate the buffer. + let result = unsafe { libc::write(stream.as_raw_fd(), buffer.as_ptr().cast(), buffer.len()) }; + syscall_result(result, buffer.len()) +} + +#[cfg(unix)] +fn syscall_result(result: isize, capacity: usize) -> io::Result { + if result < 0 { + Err(io::Error::last_os_error()) + } else { + let count = usize::try_from(result).map_err(io::Error::other)?; + progress::checked_count(count, capacity).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "native count exceeded buffer") + })?; + Ok(result) + } +} + +/// Transfer once between borrowed Linux descriptors using kernel splice. +/// +/// # Errors +/// Returns the syscall error unchanged; policy decides retry and fallback. +#[cfg(target_os = "linux")] +pub fn splice_once( + reader: BorrowedStream<'_>, + writer: BorrowedStream<'_>, + length: usize, +) -> io::Result { + // SAFETY: both descriptor borrows remain live. Null offsets request the + // file offsets (and are required for pipes); no userspace buffer escapes. + let result = unsafe { + libc::splice( + reader.as_raw_fd(), + std::ptr::null_mut(), + writer.as_raw_fd(), + std::ptr::null_mut(), + length, + libc::SPLICE_F_MOVE | libc::SPLICE_F_MORE, + ) + }; + syscall_result(result, length) +} + +/// Create two uniquely owned pipe endpoints. +/// +/// # Errors +/// Returns the pipe creation error without constructing owners on failure. +#[cfg(unix)] +pub fn pipe() -> io::Result<(OwnedStream, OwnedStream)> { + let mut endpoints = [0; 2]; + // SAFETY: endpoints provides writable storage for exactly two integers. + let result = unsafe { libc::pipe(endpoints.as_mut_ptr()) }; + if result != 0 { + return Err(io::Error::last_os_error()); + } + let [reader, writer] = endpoints; + // SAFETY: successful pipe returns two distinct, newly owned descriptors. + Ok(unsafe { + ( + OwnedStream::from_raw_fd(reader), + OwnedStream::from_raw_fd(writer), + ) + }) +} + +/// Observe whether a descriptor is currently open, for native regressions. +/// +/// This observation grants no ownership or lifetime guarantee and must never +/// be used to justify subsequent raw-resource reconstruction. It is test +/// support, so it is neither exported nor compiled into the shipped library. +/// +/// # Errors +/// Returns the kernel's error for any failure that is neither an interruption +/// nor the closed-descriptor condition. Folding those into "closed" would +/// report an ownership verdict the observation never established. +#[cfg(all(test, unix))] +pub(crate) fn fd_is_open(fd: i32) -> io::Result { + loop { + // SAFETY: F_GETFD accepts arbitrary descriptor integers, has no pointer + // argument, and reports EBADF without touching memory for closed FDs. + let result = unsafe { libc::fcntl(fd, libc::F_GETFD) }; + if result != -1 { + return Ok(true); + } + let error = io::Error::last_os_error(); + match error.raw_os_error() { + // A delivered signal interrupted the call; observe again. + Some(libc::EINTR) => continue, + Some(libc::EBADF) => return Ok(false), + _ => return Err(error), + } + } +} + +#[cfg(test)] +mod ownership_tests; +#[cfg(windows)] +mod windows; +#[cfg(all(test, windows))] +pub(crate) use windows::fd_is_open; +#[cfg(windows)] +pub use windows::{pipe, read_once, write_once}; diff --git a/rust/cuprum-native-io/src/memory.rs b/rust/cuprum-native-io/src/memory.rs new file mode 100644 index 000000000..d3e587fe8 --- /dev/null +++ b/rust/cuprum-native-io/src/memory.rs @@ -0,0 +1,50 @@ +//! Production owner-scope kernels shared with Kani and unwind tests. +#![forbid(unsafe_code)] + +/// Run an operation with a borrowed reader and a scoped owned writer. +/// +/// The operation may return an error or unwind. Rust drops `writer` in every +/// returning/unwinding path, while this function cannot drop `reader`'s owner. +/// The shared writer borrow prevents replacing or moving out an `OwnedStream` +/// through the callback. Process abort and deliberately leaking resources are +/// outside this contract. +pub fn with_owned_writer(reader: &R, writer: W, operation: impl FnOnce(&R, &W) -> T) -> T { + let result = operation(reader, &writer); + drop(writer); + result +} + +/// Retain an externally owned value on return, error, and real unwind. +/// +/// Only the Windows read/write adapter reconstructs an owner for this helper; +/// it never exposes the owner to external callbacks that could move it out. +#[cfg(any(windows, test, kani))] +pub(crate) fn with_retained_owner(value: T, operation: impl FnOnce(&mut T) -> R) -> R { + let mut retained = core::mem::ManuallyDrop::new(value); + operation(&mut retained) +} + +#[cfg(test)] +mod tests { + //! Real unwinding through the production retention frame. + use super::with_retained_owner; + use core::cell::Cell; + use std::panic::{AssertUnwindSafe, catch_unwind}; + + struct Close<'a>(&'a Cell); + impl Drop for Close<'_> { + fn drop(&mut self) { + self.0.set(self.0.get() + 1); + } + } + + #[test] + fn retained_owner_survives_real_unwind() { + let closes = Cell::new(0); + let outcome = catch_unwind(AssertUnwindSafe(|| { + with_retained_owner(Close(&closes), |_| panic!("real unwind")); + })); + assert!(outcome.is_err()); + assert_eq!(closes.get(), 0); + } +} diff --git a/rust/cuprum-native-io/src/ownership_tests.rs b/rust/cuprum-native-io/src/ownership_tests.rs new file mode 100644 index 000000000..ca8ca2bec --- /dev/null +++ b/rust/cuprum-native-io/src/ownership_tests.rs @@ -0,0 +1,185 @@ +//! Actual resource effects and unwind regressions for the ownership boundary. + +#[cfg(unix)] +use std::os::fd::{AsRawFd, IntoRawFd}; +#[cfg(windows)] +use std::os::windows::io::{AsRawHandle, IntoRawHandle}; +use std::panic::{AssertUnwindSafe, catch_unwind}; + +use rstest::{fixture, rstest}; +use std::sync::{LockResult, Mutex, MutexGuard}; + +use super::{adopt_writer, borrow, borrow_reader, fd_is_open, pipe, read_once, with_owned_writer}; + +// Closed-descriptor observations must not race this test binary's next pipe. +// The guard is outside catch_unwind, so intentional panics do not poison it. +static DESCRIPTOR_TESTS: Mutex<()> = Mutex::new(()); + +#[fixture] +fn descriptor_guard() -> LockResult> { + DESCRIPTOR_TESTS.lock() +} + +/// Observe a descriptor, failing loudly when observation itself fails. +/// +/// An observation error is neither an open nor a closed verdict: the kernel, or +/// the Win32 call, refused to answer. Folding that into either branch would +/// report an ownership assertion the observation never established, so the +/// test fails here first, naming the underlying error. +fn descriptor_is_open(raw: super::PlatformFd) -> bool { + match fd_is_open(raw) { + Ok(is_open) => is_open, + Err(error) => panic!("descriptor observation failed: {error:?}"), + } +} + +#[rstest] +#[case::normal(false)] +#[case::real_unwind(true)] +fn borrowed_reader_survives( + #[from(descriptor_guard)] guard_result: LockResult>, + #[case] should_panic: bool, +) { + let guard = guard_result.unwrap_or_else(|error| panic!("descriptor test lock: {error:?}")); + let (reader, writer) = pipe().unwrap_or_else(|error| panic!("create pipe: {error:?}")); + let raw = raw_of(&reader); + let outcome = catch_unwind(AssertUnwindSafe(|| { + // SAFETY: reader remains owned outside this scope, including unwind. + let view = unsafe { borrow_reader(raw) }; + // Windows also reconstructs a temporary capability File for I/O. + // Unwind through that exact adapter before observing the real handle. + #[cfg(windows)] + if should_panic { + drop(super::windows::with_file(view, |_| { + panic!("injected real unwind") + })); + } + assert!(!should_panic, "injected real unwind"); + let written = super::write_once(borrow(&writer), b"ping") + .unwrap_or_else(|error| panic!("write payload: {error:?}")); + assert_eq!(written, 4); + drop(writer); + let mut buffer = [0; 8]; + let count = read_once(view, &mut buffer) + .unwrap_or_else(|error| panic!("read through borrow: {error:?}")); + assert_eq!(count, 4); + assert_eq!(buffer.get(..4), Some(b"ping".as_slice())); + })); + assert_eq!(outcome.is_err(), should_panic); + assert!( + descriptor_is_open(raw), + "borrow must not close the caller's reader" + ); + drop(reader); + assert!( + !descriptor_is_open(raw), + "the actual owner still closes the descriptor" + ); + drop(guard); +} + +#[rstest] +#[case::normal(0)] +#[case::error(1)] +#[case::real_unwind(2)] +fn transferred_writer_closes_and_delivers_eof( + #[from(descriptor_guard)] guard_result: LockResult>, + #[case] exit: u8, +) { + let guard = guard_result.unwrap_or_else(|error| panic!("descriptor test lock: {error:?}")); + let (reader, writer) = pipe().unwrap_or_else(|error| panic!("create pipe: {error:?}")); + let raw = into_raw(writer); + let outcome = catch_unwind(AssertUnwindSafe(|| { + // SAFETY: into_raw_fd relinquished the only owner immediately above. + let owned = unsafe { adopt_writer(raw) }; + with_owned_writer(&reader, owned, |source, sink| { + assert_eq!( + super::write_once(borrow(sink), b"pong") + .unwrap_or_else(|error| panic!("write: {error:?}")), + 4 + ); + assert!(descriptor_is_open(raw_of(source))); + match exit { + 2 => panic!("injected real unwind"), + 1 => Err("injected operation error"), + _ => Ok(()), + } + }) + })); + assert_eq!(outcome.is_err(), exit == 2); + if exit == 1 { + assert_eq!( + outcome.unwrap_or_else(|error| panic!("no panic: {error:?}")), + Err("injected operation error") + ); + } + assert!( + !descriptor_is_open(raw), + "transferred writer must close on every exit" + ); + let mut bytes = [0; 8]; + assert_eq!( + read_once(borrow(&reader), &mut bytes) + .unwrap_or_else(|error| panic!("read payload: {error:?}")), + 4 + ); + assert_eq!(bytes.get(..4), Some(b"pong".as_slice())); + assert_eq!( + read_once(borrow(&reader), &mut bytes) + .unwrap_or_else(|error| panic!("read EOF: {error:?}")), + 0 + ); + // A subsequent resource must not be closed by delayed ownership cleanup. + let (replacement, _replacement_writer) = + pipe().unwrap_or_else(|error| panic!("replacement pipe: {error:?}")); + assert!(descriptor_is_open(raw_of(&replacement))); + drop(guard); +} + +#[rstest] +fn owned_descriptor_reads_and_closes( + #[from(descriptor_guard)] guard_result: LockResult>, +) { + let guard = guard_result.unwrap_or_else(|error| panic!("descriptor test lock: {error:?}")); + let (reader, writer) = pipe().unwrap_or_else(|error| panic!("create pipe: {error:?}")); + assert_eq!( + super::write_once(borrow(&writer), b"pong") + .unwrap_or_else(|error| panic!("write: {error:?}")), + 4 + ); + drop(writer); + let raw = into_raw(reader); + // SAFETY: the sole owner was consumed immediately above. + let owned = unsafe { adopt_writer(raw) }; + let mut bytes = [0; 8]; + assert_eq!( + read_once(borrow(&owned), &mut bytes).unwrap_or_else(|error| panic!("read: {error:?}")), + 4 + ); + assert_eq!(bytes.get(..4), Some(b"pong".as_slice())); + drop(owned); + assert!(!descriptor_is_open(raw)); + drop(guard); +} + +fn raw_of(resource: &super::OwnedStream) -> super::PlatformFd { + #[cfg(unix)] + { + resource.as_raw_fd() + } + #[cfg(windows)] + { + resource.as_raw_handle() as usize + } +} + +fn into_raw(resource: super::OwnedStream) -> super::PlatformFd { + #[cfg(unix)] + { + resource.into_raw_fd() + } + #[cfg(windows)] + { + resource.into_raw_handle() as usize + } +} diff --git a/rust/cuprum-native-io/src/progress.rs b/rust/cuprum-native-io/src/progress.rs new file mode 100644 index 000000000..a775f480c --- /dev/null +++ b/rust/cuprum-native-io/src/progress.rs @@ -0,0 +1,36 @@ +//! Checked length kernels shared by production I/O and formal verification. +#![forbid(unsafe_code)] + +/// Reject a native byte count outside the supplied initialized buffer. +#[must_use] +pub const fn checked_count(count: usize, capacity: usize) -> Option { + if count <= capacity { Some(count) } else { None } +} + +/// Advance byte accounting without overflow or consuming beyond a chunk. +/// +/// Failure leaves both input values available unchanged to the caller. +#[must_use] +pub const fn checked_progress(total: u64, remaining: u64, written: u64) -> Option<(u64, u64)> { + if written > remaining || written > u64::MAX - total { + None + } else { + Some((total + written, remaining - written)) + } +} + +#[cfg(test)] +mod tests { + //! Boundary examples complement the symbolic domain checks. + use super::{checked_count, checked_progress}; + + #[test] + fn rejects_invalid_bounds_and_overflow() { + assert_eq!(checked_count(5, 4), None); + assert_eq!(checked_count(4, 4), Some(4)); + assert_eq!(checked_progress(u64::MAX, 1, 1), None); + assert_eq!(checked_progress(0, 4, 5), None); + assert_eq!(checked_progress(u64::MAX - 1, 2, 1), Some((u64::MAX, 1))); + assert_eq!(checked_progress(5, 4, 0), Some((5, 4))); + } +} diff --git a/rust/cuprum-native-io/src/windows.rs b/rust/cuprum-native-io/src/windows.rs new file mode 100644 index 000000000..00236ba00 --- /dev/null +++ b/rust/cuprum-native-io/src/windows.rs @@ -0,0 +1,92 @@ +//! Windows borrowed I/O, pipe ownership, and native handle observation. + +use std::io::{self, Read, Write}; +use std::os::windows::io::{AsRawHandle, FromRawHandle}; + +use windows_sys::Win32::System::Pipes::CreatePipe; + +#[cfg(test)] +use crate::PlatformFd; +use crate::{BorrowedStream, OwnedStream}; +#[cfg(test)] +use windows_sys::Win32::Foundation::{ERROR_INVALID_HANDLE, GetHandleInformation}; + +/// Create two uniquely owned, non-inheritable anonymous pipe handles. +/// +/// # Errors +/// Returns the Win32 error if no pipe could be created. +pub fn pipe() -> io::Result<(OwnedStream, OwnedStream)> { + let mut reader = std::ptr::null_mut(); + let mut writer = std::ptr::null_mut(); + // SAFETY: both outputs are writable HANDLE slots. Null security attributes + // select default security and non-inheritance; zero selects default size. + let result = unsafe { CreatePipe(&mut reader, &mut writer, std::ptr::null(), 0) }; + if result == 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: success produced distinct valid pipe handles owned only here. + Ok(unsafe { + ( + OwnedStream::from_raw_handle(reader), + OwnedStream::from_raw_handle(writer), + ) + }) +} + +/// Observe whether a handle is currently open, without granting ownership. +/// +/// The result must not be used as a check-then-use validity guarantee. It is +/// test support, so it is neither exported nor compiled into the shipped +/// library. +/// +/// # Errors +/// Returns the Win32 error for any failure other than the documented +/// invalid-handle condition. Recording those as "closed" would report an +/// ownership verdict the observation never established. +#[cfg(test)] +pub(crate) fn fd_is_open(raw: PlatformFd) -> io::Result { + let mut flags = 0; + // SAFETY: GetHandleInformation validates the opaque handle and writes + // flags only through the live output pointer. No ownership is acquired. + if unsafe { GetHandleInformation(raw as _, &mut flags) } != 0 { + return Ok(true); + } + let error = io::Error::last_os_error(); + // Win32 codes are unsigned; a negative `raw_os_error` cannot be one, so + // the conversion to the constant's own type is the honest comparison. + match error + .raw_os_error() + .and_then(|code| u32::try_from(code).ok()) + { + Some(ERROR_INVALID_HANDLE) => Ok(false), + _ => Err(error), + } +} + +/// Read once into initialized storage, retaining the handle borrow. +/// +/// # Errors +/// Returns the native I/O error, including interruption, without retrying. +pub fn read_once(stream: BorrowedStream<'_>, buffer: &mut [u8]) -> io::Result { + with_file(stream, |file| file.read(buffer)) +} + +/// Write once from initialized storage, retaining the handle borrow. +/// +/// # Errors +/// Returns the native I/O error, including interruption, without retrying. +pub fn write_once(stream: BorrowedStream<'_>, buffer: &[u8]) -> io::Result { + with_file(stream, |file| file.write(buffer)) +} + +pub(super) fn with_file( + stream: BorrowedStream<'_>, + operation: impl FnOnce(&mut cap_std::fs::File) -> io::Result, +) -> io::Result { + // SAFETY: the borrowed handle is valid during this scope. ManuallyDrop + // prevents closing it on normal return and unwind. This private helper + // only invokes read/write; callers cannot replace or move out the File. + let file = unsafe { cap_std::fs::File::from_raw_handle(stream.as_raw_handle()) }; + crate::memory::with_retained_owner(file, operation) + .and_then(|count| isize::try_from(count).map_err(io::Error::other)) +} diff --git a/rust/cuprum-rust/Cargo.toml b/rust/cuprum-rust/Cargo.toml index 29dc5e3b2..56082b10d 100644 --- a/rust/cuprum-rust/Cargo.toml +++ b/rust/cuprum-rust/Cargo.toml @@ -11,18 +11,13 @@ name = "_rust_backend_native" crate-type = ["cdylib", "rlib"] [dependencies] +cuprum-streams = { path = "../cuprum-streams", version = "0.1.0" } +cuprum-native-io = { path = "../cuprum-native-io", version = "0.1.0" } pyo3 = { version = "0.29.0", features = ["extension-module"] } -thiserror = "2.0.18" -tracing = { version = "0.1.44", default-features = false, features = ["std"] } - -[target.'cfg(unix)'.dependencies] -libc = "0.2.186" [dev-dependencies] -insta = { version = "1.48.0", default-features = false } proptest = "1.11.0" rstest = "0.27.0" -tempfile = "3.27.0" trybuild = { version = "1.0.116", features = ["diff"] } [lints] diff --git a/rust/cuprum-rust/src/errors.rs b/rust/cuprum-rust/src/errors.rs index 74c6a65ef..0e3f4e6de 100644 --- a/rust/cuprum-rust/src/errors.rs +++ b/rust/cuprum-rust/src/errors.rs @@ -1,69 +1,16 @@ -//! Canonical semantic error type for the stream pump and consume paths. -//! -//! `PumpError` is the single error taxonomy for the crate's internal stream -//! operations. The previously scattered `io::Error::other("...")` -//! constructions for "impossible" overflow and slice conditions are replaced -//! by machine-distinguishable variants; conversion to a Python exception -//! happens in exactly one place (`From for PyErr`). - -use std::io; - +//! Map safe stream errors to Python exceptions without losing OS codes. +use cuprum_streams::PumpError; use pyo3::PyErr; use pyo3::exceptions::PyOSError; -use thiserror::Error; - -/// Semantic error for stream pump and consume operations. -#[derive(Debug, Error)] -pub(crate) enum PumpError { - /// An integer length conversion overflowed its target type. - /// - /// This is an "impossible" condition on supported platforms (for - /// example, a non-negative `ssize_t` always fits a `usize` on Linux); - /// the variant exists so the condition stays observable rather than - /// silently truncating. - #[error("integer length conversion overflowed")] - LengthOverflow, - /// A computed range exceeded the backing buffer's bounds. - #[error("computed range exceeded the buffer bounds")] - BufferRangeExceeded, - /// An operating-system I/O failure. - #[error(transparent)] - Io(#[from] io::Error), -} - -impl PumpError { - /// Report whether this is a non-fatal write condition (broken pipe). - /// - /// These errors indicate the write end closed, which is expected when - /// downstream processes exit early. The caller should drain the reader - /// and return successfully rather than propagating the error. - pub(crate) fn is_nonfatal_write(&self) -> bool { - matches!( - self, - Self::Io(err) if matches!( - err.kind(), - io::ErrorKind::BrokenPipe | io::ErrorKind::ConnectionReset - ) - ) - } - - /// Return the Python `OSError` message for semantic non-I/O variants. - pub(crate) const fn py_os_error_message(&self) -> Option<&'static str> { - match self { - Self::LengthOverflow => Some("integer length conversion overflowed"), - Self::BufferRangeExceeded => Some("computed range exceeded the buffer bounds"), - Self::Io(_) => None, - } - } -} +use std::io; -impl From for PyErr { - fn from(err: PumpError) -> Self { - match err { - PumpError::Io(io_err) => io_error_to_py_err(io_err), - other @ (PumpError::LengthOverflow | PumpError::BufferRangeExceeded) => { - PyOSError::new_err(other.py_os_error_message().unwrap_or("stream pump failed")) - } +pub(crate) fn pump_error_to_py_err(err: PumpError) -> PyErr { + match err { + PumpError::Io(io_err) => io_error_to_py_err(io_err), + other @ (PumpError::LengthOverflow + | PumpError::BufferRangeExceeded + | PumpError::BufferAllocationFailed) => { + PyOSError::new_err(other.py_os_error_message().unwrap_or("stream pump failed")) } } } @@ -221,6 +168,23 @@ mod tests { ); } + #[test] + fn allocation_failure_is_fatal_and_stably_messaged() { + // The allocation paths reserve fallibly so this variant is reachable + // rather than the process aborting; it must stay out of the non-fatal + // write classification and carry the message the Python boundary + // raises. + assert!(!PumpError::BufferAllocationFailed.is_nonfatal_write()); + assert_eq!( + PumpError::BufferAllocationFailed.to_string(), + "failed to allocate the stream buffer", + ); + assert_eq!( + PumpError::BufferAllocationFailed.py_os_error_message(), + Some("failed to allocate the stream buffer"), + ); + } + #[rstest] #[case::raw_os_error("Bad file descriptor (os error 9)", 9, "Bad file descriptor")] #[case::other_code("Broken pipe (os error 32)", 32, "Broken pipe")] diff --git a/rust/cuprum-rust/src/lib.rs b/rust/cuprum-rust/src/lib.rs index 25fdd6884..0f6ef5d52 100644 --- a/rust/cuprum-rust/src/lib.rs +++ b/rust/cuprum-rust/src/lib.rs @@ -1,49 +1,23 @@ -//! Optional Rust extension for Cuprum stream operations. +//! Python integration boundary for Cuprum's optional native stream backend. //! -//! This crate exposes a `PyO3` module providing stream pump and consume -//! helpers alongside an availability check for the Rust backend. +//! Raw-resource obligations are confined here and in `cuprum-native-io`. +//! Stream policy lives in `cuprum-streams`, which forbids unsafe code. +#![deny(unsafe_op_in_unsafe_fn)] +use cuprum_native_io::PlatformFd; +use cuprum_streams::{BufferSize, PumpError, consume_stream, pump_stream}; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; - -use std::mem::ManuallyDrop; - -#[cfg(unix)] -use std::os::fd::{FromRawFd, OwnedFd}; - -#[cfg(windows)] -use std::fs::File; - -#[cfg(windows)] -use std::os::windows::io::{FromRawHandle, RawHandle}; - -#[cfg(test)] -mod buffer_size_tests; -#[cfg(all(test, unix))] -mod consume_snapshot_tests; mod errors; -#[cfg(kani)] -mod fd_ownership_kani_proofs; -#[cfg(any(test, kani))] -mod fd_ownership_model; #[cfg(test)] mod fd_tests; -mod io_utils; -#[cfg(all(test, unix))] -mod lib_tests; -mod pump_machine; -#[cfg(target_os = "linux")] -mod splice; -#[cfg(all(test, unix))] -mod test_support; -#[cfg(all(test, unix))] -mod tracing_capture; -mod utf8; -use errors::PumpError; -use io_utils::{StreamHandle, classify_write, operation_span, read_stream}; -use pump_machine::{Flow, PumpState, advance}; -use utf8::{FinalChunk, decode_utf8_replace}; +#[derive(Clone, Copy, Debug)] +struct ReaderFd(PlatformFd); + +fn validate_buffer_size(size: i64) -> PyResult { + BufferSize::new(size).map_err(PyValueError::new_err) +} /// Report whether the Rust extension is available. /// @@ -97,243 +71,6 @@ fn convert_platform_fd(value: i64) -> Result { usize::try_from(value).map_err(|_| "file handle out of range") } -/// Maximum accepted stream buffer size, in bytes (1 GiB). -/// -/// This guards against absurd allocations from a bad `buffer_size` while -/// comfortably exceeding any realistic transfer buffer — the default is -/// 64 KiB and even multi-megabyte buffers stay far below this cap. -const MAX_BUFFER_SIZE: usize = 1 << 30; - -/// Validate and convert a raw `buffer_size` into a bounded `usize`. -/// -/// This is the pure decision core behind [`validate_buffer_size`]: it takes no -/// Python state and returns a stable, message-carrying error so it can be -/// property tested directly. -/// -/// # Errors -/// Returns a stable error message when `buffer_size` is non-positive, overflows -/// `usize` on the target platform, or exceeds [`MAX_BUFFER_SIZE`]. -fn checked_buffer_size(buffer_size: i64) -> Result { - if buffer_size <= 0 { - return Err("buffer_size must be greater than zero"); - } - let size = usize::try_from(buffer_size).map_err(|_| "buffer_size is too large")?; - if size > MAX_BUFFER_SIZE { - return Err("buffer_size exceeds the maximum permitted size"); - } - Ok(size) -} - -/// Validate that `buffer_size` is positive, in range, and within the cap. -/// -/// # Errors -/// Returns `PyValueError` if `buffer_size` is non-positive, out of range, or -/// exceeds [`MAX_BUFFER_SIZE`]. -fn validate_buffer_size(buffer_size: i64) -> PyResult { - checked_buffer_size(buffer_size) - .map(BufferSize) - .map_err(PyValueError::new_err) -} - -#[cfg(unix)] -fn stream_from_raw(fd: PlatformFd) -> StreamHandle { - // SAFETY: The caller ensures the fd is valid and owned by the caller. - unsafe { OwnedFd::from_raw_fd(fd) } -} - -/// Run `operation` against a `StreamHandle` borrowed from a caller-owned FD. -/// -/// This is the canonical "borrow this FD without owning it" helper: the -/// handle is wrapped in [`ManuallyDrop`], so it is **never** closed when the -/// scope ends — including during unwinding from a panicking `operation`. -/// The previous pattern (a trailing `std::mem::forget` after the inner call) -/// was skipped on unwind, closing the caller-owned descriptor and exposing -/// the Python side to a double close. -/// -/// There is deliberately no writer variant: the writer FD handed to -/// [`pump_stream`] is *consumed* — it must close (on drop, including during -/// unwinding) to signal EOF downstream. -/// -/// SAFETY: the caller must guarantee that `fd` is a valid open descriptor -/// (or handle on Windows) for the duration of the call, and that ownership -/// remains with the caller; the helper guarantees it never closes `fd`. -fn with_borrowed_reader(fd: PlatformFd, operation: impl FnOnce(&mut StreamHandle) -> T) -> T { - let mut handle = ManuallyDrop::new(stream_from_raw(fd)); - // `ManuallyDrop` suppresses the close in every exit path, so no - // drop-guard or forget call is required even if `operation` panics. - operation(&mut handle) -} - -#[cfg(windows)] -fn stream_from_raw(handle: PlatformFd) -> StreamHandle { - // The usize-to-pointer cast is a deliberate, documented reinterpretation: - // Windows handles are pointer-sized opaque values, so this widens or - // narrows nothing. - // SAFETY: The caller ensures the handle is valid and owned by the caller. - unsafe { File::from_raw_handle(handle as RawHandle) } -} -/// Pump bytes between file descriptors with explicit ownership semantics. -/// -/// The reader FD is borrowed and left open. The writer FD is treated as -/// consumed and closes on drop to signal EOF downstream — including when -/// the pump unwinds. -fn pump_stream( - reader_fd: ReaderFd, - writer_fd: WriterFd, - buffer_size: BufferSize, -) -> Result { - let mut writer = stream_from_raw(writer_fd.0); - with_borrowed_reader(reader_fd.0, |reader| { - pump_stream_files(reader, &mut writer, buffer_size) - }) -} - -/// Consume bytes from a file descriptor and decode UTF-8 with replacement. -fn consume_stream(reader_fd: ReaderFd, buffer_size: BufferSize) -> Result { - with_borrowed_reader(reader_fd.0, |reader| { - consume_stream_files(reader, buffer_size) - }) -} - -#[cfg(unix)] -type PlatformFd = i32; - -#[cfg(windows)] -type PlatformFd = usize; - -#[derive(Clone, Copy, Debug)] -struct ReaderFd(PlatformFd); - -#[derive(Clone, Copy, Debug)] -struct WriterFd(PlatformFd); - -#[derive(Clone, Copy, Debug)] -struct BufferSize(usize); - -impl BufferSize { - const fn value(self) -> usize { - self.0 - } -} - -fn pump_stream_files( - reader: &mut StreamHandle, - writer: &mut StreamHandle, - buffer_size: BufferSize, -) -> Result { - // On Linux, attempt zero-copy splice first. - #[cfg(target_os = "linux")] - if let Some(result) = splice::try_splice_pump(reader, writer, buffer_size.value()) { - return result; - } - - // Fallback: read/write loop for non-Linux or unsupported FD types. - pump_stream_files_readwrite(reader, writer, buffer_size) -} - -/// Read/write loop fallback for pumping bytes between file descriptors. -/// -/// This is used when splice is not available (non-Linux) or when the file -/// descriptors do not support splice (regular files, some sockets). -fn pump_stream_files_readwrite( - reader: &mut StreamHandle, - writer: &mut StreamHandle, - buffer_size: BufferSize, -) -> Result { - // Operation span (see `operation_span`) so the EINTR (`warn!`) and - // fatal-I/O (`error!`) events emitted from the read/write seams inherit - // the operation name, `buffer_size`, and `total_bytes` context even under - // a `warn`/`error`-only production filter. - let span = operation_span("pump_stream_readwrite", buffer_size.value()); - let _guard = span.enter(); - io_utils::reset_retry_counters(); - - let mut buffer = vec![0_u8; buffer_size.value()]; - let mut state = PumpState::start(); - - loop { - let read_len = read_stream(reader, &mut buffer)?; - let writer_was_open = state.writer_open(); - - // `advance` owns both the zero-length-is-EOF translation and the write - // precondition — a chunk read while the writer is still open — so this - // loop, the property tests, and the bounded proofs share one - // definition of them. Fatal writes propagate the real error and never - // reach the pure state machine. - let flow = advance(&mut state, read_len, || { - let chunk = buffer - .get(..read_len) - .ok_or(PumpError::BufferRangeExceeded)?; - classify_write(writer, chunk) - })?; - - // The latch closing is the `head`-style early exit. Mirror splice's - // field and message so the event is not visible on one path only, and - // observe it here rather than in the deliberately pure `pump_machine`. - if writer_was_open && !state.writer_open() { - tracing::debug!( - bytes_transferred = state.total_written(), - "broken pipe; draining reader" - ); - } - - if flow == Flow::Stop { - break; - } - } - - let total_written = state.total_written(); - span.record("total_bytes", total_written); - span.record("read_retries", io_utils::read_retry_count()); - span.record("write_retries", io_utils::write_retry_count()); - Ok(total_written) -} - -fn consume_stream_files( - reader: &mut StreamHandle, - buffer_size: BufferSize, -) -> Result { - // Operation span (see `operation_span`) so the read seam's `warn!`/`error!` - // events inherit this operation's context even under a `warn`/`error`-only - // production filter. - let span = operation_span("consume_stream", buffer_size.value()); - let _guard = span.enter(); - io_utils::reset_retry_counters(); - - let mut buffer = vec![0_u8; buffer_size.value()]; - let mut pending: Vec = Vec::new(); - let mut output = String::new(); - let mut total_read = 0_u64; - - #[cfg(unix)] - let platform = "unix"; - #[cfg(windows)] - let platform = "windows"; - - loop { - let read_len = read_stream(reader, &mut buffer)?; - if read_len == 0 { - break; - } - let read_bytes = u64::try_from(read_len).map_err(|_| { - tracing::error!(platform, "read length conversion overflowed"); - PumpError::LengthOverflow - })?; - total_read = total_read.saturating_add(read_bytes); - let chunk = buffer - .get(..read_len) - .ok_or(PumpError::BufferRangeExceeded)?; - pending.extend_from_slice(chunk); - decode_utf8_replace(&mut pending, &mut output, FinalChunk::new(false)); - } - - decode_utf8_replace(&mut pending, &mut output, FinalChunk::new(true)); - - span.record("total_bytes", total_read); - span.record("read_retries", io_utils::read_retry_count()); - Ok(output) -} - /// Python module definition for the optional Rust backend. /// /// # Errors diff --git a/rust/cuprum-rust/src/stream_pyfunctions.rs b/rust/cuprum-rust/src/stream_pyfunctions.rs index 84f7837b2..30f162d6f 100644 --- a/rust/cuprum-rust/src/stream_pyfunctions.rs +++ b/rust/cuprum-rust/src/stream_pyfunctions.rs @@ -1,8 +1,8 @@ //! Contain the Python stream exports and their generated `PyO3` wrappers. use super::{ - BufferSize, PumpError, PyErr, PyResult, Python, ReaderFd, WriterFd, consume_stream, convert_fd, - pump_stream, pyfunction, validate_buffer_size, + BufferSize, PumpError, PyResult, Python, ReaderFd, consume_stream, convert_fd, pump_stream, + pyfunction, validate_buffer_size, }; /// Run a prepared stream operation after validating its buffer size. @@ -24,7 +24,7 @@ where let reader = ReaderFd(convert_fd(reader_fd)?); let operation = prepare_operation()?; let result = py.detach(move || operation(reader, validated_buffer_size)); - result.map_err(PyErr::from) + result.map_err(super::errors::pump_error_to_py_err) } /// Pump bytes between file descriptors outside the GIL. @@ -46,8 +46,17 @@ pub(super) fn rust_pump_stream( buffer_size: i64, ) -> PyResult { run_stream_operation(py, reader_fd, buffer_size, || { - let writer = WriterFd(convert_fd(writer_fd)?); - Ok(move |reader, validated_buffer_size| pump_stream(reader, writer, validated_buffer_size)) + let writer_raw = convert_fd(writer_fd)?; + // SAFETY: `_streams_rs` transfers its duplicate exactly once. + // Python retains no owner after this hand-off; see the boundary + // contract for direct native callers and submission rollback. + let writer = unsafe { cuprum_native_io::adopt_writer(writer_raw) }; + Ok(move |reader: ReaderFd, validated_buffer_size| { + // SAFETY: Python keeps the paused reader transport alive until + // the worker and cleanup finish, including cancellation. + let source = unsafe { cuprum_native_io::borrow_reader(reader.0) }; + pump_stream(&source, writer, validated_buffer_size) + }) }) } @@ -73,5 +82,12 @@ pub(super) fn rust_consume_stream( reader_fd: i64, buffer_size: i64, ) -> PyResult { - run_stream_operation(py, reader_fd, buffer_size, || Ok(consume_stream)) + run_stream_operation(py, reader_fd, buffer_size, || { + Ok(|reader: ReaderFd, size| { + // SAFETY: the Python consume caller retains its reader throughout + // this synchronous native call, including the GIL-free interval. + let source = unsafe { cuprum_native_io::borrow_reader(reader.0) }; + consume_stream(&source, size) + }) + }) } diff --git a/rust/cuprum-streams/Cargo.toml b/rust/cuprum-streams/Cargo.toml new file mode 100644 index 000000000..8f7283591 --- /dev/null +++ b/rust/cuprum-streams/Cargo.toml @@ -0,0 +1,96 @@ +[package] +name = "cuprum-streams" +version = "0.1.0" +edition = "2024" +license = "ISC" +description = "Safe stream orchestration for Cuprum" +publish = false + +[dependencies] +cuprum-native-io = { path = "../cuprum-native-io", version = "0.1.0" } +thiserror = "2.0.18" +tracing = { version = "0.1.44", default-features = false, features = ["std"] } + +[target.'cfg(unix)'.dependencies] +libc = "0.2.186" + +[dev-dependencies] +cap-std = "4.0.2" +insta = { version = "1.48.0", default-features = false } +proptest = "1.11.0" +rstest = "0.26.1" +tempfile = "3.27.0" +trybuild = { version = "1.0.116", features = ["diff"] } + +# Cargo cannot override one lint while inheriting workspace lints. The +# boundary-contract gate requires these tables to match the workspace plus +# unsafe_code = "forbid", which applies to every current and future target. +[lints.clippy] +pedantic = { level = "warn", priority = -1 } + +# 1. hygiene +allow_attributes = "deny" +allow_attributes_without_reason = "deny" +blanket_clippy_restriction_lints = "deny" +cognitive_complexity = "deny" +needless_pass_by_value = "deny" +implicit_hasher = "deny" + +# 2. debugging leftovers +dbg_macro = "deny" +print_stdout = "deny" +print_stderr = "deny" + +# 3. panic-prone operations +unwrap_used = "deny" +expect_used = "deny" +indexing_slicing = "deny" +string_slice = "deny" +integer_division = "deny" +integer_division_remainder_used = "deny" +panic_in_result_fn = "deny" +unreachable = "deny" + +# 4. portability +host_endian_bytes = "deny" +little_endian_bytes = "deny" +big_endian_bytes = "deny" + +# 5. nursery idiom polish +let_underscore_must_use = "deny" +or_fun_call = "deny" +option_if_let_else = "deny" +self_named_module_files = "deny" +shadow_reuse = "deny" +shadow_same = "deny" +shadow_unrelated = "deny" +str_to_string = "deny" +string_lit_as_bytes = "deny" +try_err = "deny" +unneeded_field_pattern = "deny" +use_self = "deny" + +# 6. numerical foot-guns +float_arithmetic = "deny" +cast_possible_truncation = "deny" +cast_possible_wrap = "deny" +cast_precision_loss = "deny" +lossy_float_literal = "deny" + +# 7. API ergonomics +missing_const_for_fn = "deny" +must_use_candidate = "deny" +unused_async = "deny" + +# 8. Error handling +missing_panics_doc = "deny" +error_impl_error = "deny" +result_large_err = "deny" + +[lints.rust] +unsafe_code = "forbid" +missing_docs = "deny" +unexpected_cfgs = { level = "warn", check-cfg = ["cfg(kani)"] } + +[lints.rustdoc] +missing_crate_level_docs = "deny" diff --git a/rust/cuprum-streams/src/buffer.rs b/rust/cuprum-streams/src/buffer.rs new file mode 100644 index 000000000..ca8c6c198 --- /dev/null +++ b/rust/cuprum-streams/src/buffer.rs @@ -0,0 +1,101 @@ +//! Fallible scratch-buffer allocation for the stream loops. +//! +//! The pump, consume, and splice paths each need a zeroed buffer sized from +//! the caller's `buffer_size`. `vec![0_u8; len]` allocates infallibly: on +//! failure it runs the allocator's error handler, which aborts the process +//! and cannot be caught, so a bad `buffer_size` would take an embedding +//! interpreter down with it. Reserving fallibly keeps that failure on +//! [`PumpError`], where the Python boundary can raise it. +use crate::PumpError; + +/// Host platform name, matching the `platform` field of the I/O seam events. +#[cfg(unix)] +const PLATFORM: &str = "unix"; +/// Host platform name, matching the `platform` field of the I/O seam events. +#[cfg(windows)] +const PLATFORM: &str = "windows"; + +/// Allocate a zeroed scratch buffer of `len` bytes. +/// +/// # Errors +/// Returns [`PumpError::BufferAllocationFailed`] when the allocation fails. +/// The failure is also reported as an `error` event carrying the stable +/// `buffer_allocation_failed` category and a power-of-two bucket of the +/// requested size, so a refused allocation is diagnosable without an unwind or +/// a core dump. +pub(crate) fn allocate_buffer(len: usize) -> Result, PumpError> { + let mut buffer = Vec::new(); + if buffer.try_reserve_exact(len).is_err() { + // Only the requested size is reported, and only as a power-of-two + // bucket: no allocator internals, no identifiers, and no payload. + tracing::error!( + error_category = ALLOCATION_FAILED_CATEGORY, + buffer_size = BufferBucket::of(len).bytes(), + platform = PLATFORM, + "scratch buffer allocation refused", + ); + return Err(PumpError::BufferAllocationFailed); + } + // The reservation above already guarantees the capacity, so growing to + // `len` here cannot reallocate and cannot reach the aborting path. + buffer.resize(len, 0_u8); + Ok(buffer) +} + +/// Stable category reported by the allocation-failure error event. +pub(crate) const ALLOCATION_FAILED_CATEGORY: &str = "buffer_allocation_failed"; + +/// A requested buffer size rounded up to a power of two, for telemetry. +/// +/// Rounding bounds the number of distinct values the event can carry and keeps +/// the logged number from leaking an exact caller-chosen size. The bucket is +/// always at least the request, so it still shows how large an allocation was +/// refused; a request too large to round is reported as itself, because no +/// larger power of two is representable. +#[derive(Clone, Copy, Debug)] +struct BufferBucket(usize); + +impl BufferBucket { + /// Round `requested` up to the next power of two, saturating at the top. + const fn of(requested: usize) -> Self { + match requested.checked_next_power_of_two() { + Some(rounded) => Self(rounded), + None => Self(requested), + } + } + + /// The rounded size in bytes. + const fn bytes(self) -> usize { + self.0 + } +} + +#[cfg(test)] +mod tests { + //! Allocation coverage: usable buffers and, crucially, a failure that + //! returns instead of aborting the test process. + use super::allocate_buffer; + use crate::PumpError; + + #[test] + fn a_requested_buffer_is_zeroed_and_exactly_sized() { + match allocate_buffer(8) { + Ok(buffer) => assert_eq!(buffer, [0_u8; 8]), + Err(error) => panic!("a small allocation must succeed, got {error:?}"), + } + } + + #[test] + fn an_impossible_allocation_reports_rather_than_aborting() { + // The production cap keeps real requests far below this, so the + // capacity overflow stands in for any allocation the allocator + // refuses: it exercises the same `try_reserve_exact` error path. A + // regression to `vec![0_u8; len]` would abort here, failing the test + // by killing the harness rather than by a failed assertion. + let result = allocate_buffer(usize::MAX); + assert!( + matches!(result, Err(PumpError::BufferAllocationFailed)), + "an unallocatable buffer must report, not abort", + ); + } +} diff --git a/rust/cuprum-rust/src/buffer_size_tests.rs b/rust/cuprum-streams/src/buffer_size_tests.rs similarity index 100% rename from rust/cuprum-rust/src/buffer_size_tests.rs rename to rust/cuprum-streams/src/buffer_size_tests.rs diff --git a/rust/cuprum-rust/src/consume_snapshot_tests.rs b/rust/cuprum-streams/src/consume_snapshot_tests.rs similarity index 98% rename from rust/cuprum-rust/src/consume_snapshot_tests.rs rename to rust/cuprum-streams/src/consume_snapshot_tests.rs index 7f6e22d8a..7e2d035ff 100644 --- a/rust/cuprum-rust/src/consume_snapshot_tests.rs +++ b/rust/cuprum-streams/src/consume_snapshot_tests.rs @@ -36,8 +36,8 @@ fn consume(payload: &[u8], buffer_size: usize) -> Result { // Close the write end so the read loop reaches EOF and terminates. drop(write_end); - let mut reader = read_end; - consume_stream_files(&mut reader, BufferSize(buffer_size)) + let reader = read_end; + consume_stream_files(&reader, BufferSize(buffer_size)) } /// ASCII payloads pass through the pipe and decoder without transformation. diff --git a/rust/cuprum-streams/src/errors.rs b/rust/cuprum-streams/src/errors.rs new file mode 100644 index 000000000..94c0c1e91 --- /dev/null +++ b/rust/cuprum-streams/src/errors.rs @@ -0,0 +1,59 @@ +//! Semantic errors for safe stream orchestration. +use std::io; +use thiserror::Error; + +/// Semantic error for stream pump and consume operations. +#[derive(Debug, Error)] +pub enum PumpError { + /// An integer length conversion overflowed its target type. + /// + /// This is an "impossible" condition on supported platforms (for + /// example, a non-negative `ssize_t` always fits a `usize` on Linux); + /// the variant exists so the condition stays observable rather than + /// silently truncating. + #[error("integer length conversion overflowed")] + LengthOverflow, + /// A computed range exceeded the backing buffer's bounds. + #[error("computed range exceeded the buffer bounds")] + BufferRangeExceeded, + /// The scratch buffer could not be allocated. + /// + /// The stream loops reserve their buffers fallibly rather than with the + /// infallible `vec![0_u8; len]`, whose allocation failure aborts the + /// process instead of unwinding. Keeping the failure on this enum lets an + /// embedding interpreter raise it rather than die with it. + #[error("failed to allocate the stream buffer")] + BufferAllocationFailed, + /// An operating-system I/O failure. + #[error(transparent)] + Io(#[from] io::Error), +} + +impl PumpError { + /// Report whether this is a non-fatal write condition (broken pipe). + /// + /// These errors indicate the write end closed, which is expected when + /// downstream processes exit early. The caller should drain the reader + /// and return successfully rather than propagating the error. + #[must_use] + pub fn is_nonfatal_write(&self) -> bool { + matches!( + self, + Self::Io(err) if matches!( + err.kind(), + io::ErrorKind::BrokenPipe | io::ErrorKind::ConnectionReset + ) + ) + } + + /// Return the Python `OSError` message for semantic non-I/O variants. + #[must_use] + pub const fn py_os_error_message(&self) -> Option<&'static str> { + match self { + Self::LengthOverflow => Some("integer length conversion overflowed"), + Self::BufferRangeExceeded => Some("computed range exceeded the buffer bounds"), + Self::BufferAllocationFailed => Some("failed to allocate the stream buffer"), + Self::Io(_) => None, + } + } +} diff --git a/rust/cuprum-rust/src/io_utils/mod.rs b/rust/cuprum-streams/src/io_utils/mod.rs similarity index 86% rename from rust/cuprum-rust/src/io_utils/mod.rs rename to rust/cuprum-streams/src/io_utils/mod.rs index b8b1fa5fa..6df4072d3 100644 --- a/rust/cuprum-rust/src/io_utils/mod.rs +++ b/rust/cuprum-streams/src/io_utils/mod.rs @@ -58,19 +58,8 @@ fn record_write_retry() { } #[cfg(unix)] -use std::os::fd::{AsRawFd, OwnedFd}; - -#[cfg(windows)] -use std::fs::File; - -#[cfg(windows)] -use std::io::{Read, Write}; - -#[cfg(unix)] -pub(crate) type StreamHandle = OwnedFd; - -#[cfg(windows)] -pub(crate) type StreamHandle = File; +use cuprum_native_io::BorrowedStream; +use cuprum_native_io::{AsStream, borrow}; /// Result of a single write attempt on the stream. #[derive(Debug, PartialEq, Eq)] @@ -107,10 +96,7 @@ pub(crate) fn operation_span(operation: &'static str, buffer_size: usize) -> tra } /// Read bytes from the stream into the buffer. -pub(crate) fn read_stream( - reader: &mut StreamHandle, - buffer: &mut [u8], -) -> Result { +pub(crate) fn read_stream(reader: &impl AsStream, buffer: &mut [u8]) -> Result { #[cfg(unix)] { read_stream_unix(reader, buffer) @@ -118,13 +104,15 @@ pub(crate) fn read_stream( #[cfg(windows)] { - reader.read(buffer).map_err(PumpError::from) + cuprum_native_io::read_once(borrow(reader), buffer) + .and_then(|count| usize::try_from(count).map_err(io::Error::other)) + .map_err(PumpError::from) } } /// Write all bytes from a chunk to the writer, returning the write outcome. pub(crate) fn handle_write( - writer: &mut StreamHandle, + writer: &impl AsStream, chunk: &[u8], ) -> Result { #[cfg(unix)] @@ -159,7 +147,7 @@ pub(crate) fn handle_write( /// assert_eq!(event, WriteEvent::Closed { bytes: 0 }); /// ``` pub(crate) fn classify_write( - writer: &mut StreamHandle, + writer: &impl AsStream, chunk: &[u8], ) -> Result { classify_write_outcome(handle_write(writer, chunk)) @@ -209,8 +197,8 @@ fn classify_write_outcome( } #[cfg(unix)] -fn read_stream_unix(reader: &StreamHandle, buffer: &mut [u8]) -> Result { - read_raw_fd(reader.as_raw_fd(), buffer) +fn read_stream_unix(reader: &impl AsStream, buffer: &mut [u8]) -> Result { + read_raw_fd(borrow(reader), buffer) } /// Read from a raw descriptor, retrying on `EINTR`. @@ -219,20 +207,8 @@ fn read_stream_unix(reader: &StreamHandle, buffer: &mut [u8]) -> Result Result { - read_raw_fd_with(|| { - // SAFETY: `buffer` is valid for writes of `buffer.len()` bytes, and - // the caller guarantees `fd` stays valid for the duration of this - // call. - let read_len = - unsafe { libc::read(fd, buffer.as_mut_ptr().cast::(), buffer.len()) }; - - if read_len >= 0 { - Ok(read_len) - } else { - Err(io::Error::last_os_error()) - } - }) +pub(crate) fn read_raw_fd(fd: BorrowedStream<'_>, buffer: &mut [u8]) -> Result { + read_raw_fd_with(|| cuprum_native_io::read_once(fd, buffer)) } #[cfg(unix)] @@ -262,18 +238,9 @@ fn read_raw_fd_with( } #[cfg(unix)] -fn write_all_unix(writer: &StreamHandle, chunk: &[u8]) -> Result { - let fd = writer.as_raw_fd(); +fn write_all_unix(writer: &impl AsStream, chunk: &[u8]) -> Result { write_all_unix_with(chunk, |buffer| { - // SAFETY: `buffer` is valid for reads of `buffer.len()` bytes, and - // `fd` stays valid for the duration of this call. - let written = - unsafe { libc::write(fd, buffer.as_ptr().cast::(), buffer.len()) }; - if written >= 0 { - Ok(written) - } else { - Err(io::Error::last_os_error()) - } + cuprum_native_io::write_once(borrow(writer), buffer) }) } @@ -320,14 +287,13 @@ fn write_all_unix_with( } #[cfg(windows)] -fn write_all_windows( - writer: &mut StreamHandle, - mut chunk: &[u8], -) -> Result { +fn write_all_windows(writer: &impl AsStream, mut chunk: &[u8]) -> Result { let mut total_written = 0_u64; while !chunk.is_empty() { - match writer.write(chunk) { + match cuprum_native_io::write_once(borrow(writer), chunk) + .and_then(|count| usize::try_from(count).map_err(io::Error::other)) + { Ok(0) => { return Err(PumpError::from(io::Error::new( io::ErrorKind::WriteZero, @@ -353,10 +319,15 @@ fn record_write_progress( total_written: &mut u64, ) -> Result<(), PumpError> { let written_len_u64 = u64::try_from(written_len).map_err(|_| PumpError::LengthOverflow)?; - *total_written = total_written.saturating_add(written_len_u64); - *chunk = chunk + let remaining = u64::try_from(chunk.len()).map_err(|_| PumpError::LengthOverflow)?; + let tail = chunk .get(written_len..) .ok_or(PumpError::BufferRangeExceeded)?; + let (new_total, _) = + cuprum_native_io::progress::checked_progress(*total_written, remaining, written_len_u64) + .ok_or(PumpError::LengthOverflow)?; + *total_written = new_total; + *chunk = tail; Ok(()) } diff --git a/rust/cuprum-rust/src/io_utils/tests.rs b/rust/cuprum-streams/src/io_utils/tests.rs similarity index 88% rename from rust/cuprum-rust/src/io_utils/tests.rs rename to rust/cuprum-streams/src/io_utils/tests.rs index 42a6df988..9fbdd5426 100644 --- a/rust/cuprum-rust/src/io_utils/tests.rs +++ b/rust/cuprum-streams/src/io_utils/tests.rs @@ -1,7 +1,7 @@ //! Direct tests for descriptor-backed I/O helper contracts. use std::io; -use std::os::fd::{AsRawFd, OwnedFd}; +use std::os::fd::OwnedFd; use proptest::prelude::*; @@ -52,12 +52,12 @@ fn ssize(len: usize) -> libc::ssize_t { /// Reading from a pipe copies the complete payload into the supplied buffer. #[rstest] fn read_stream_reads_pipe_bytes(#[from(pipe)] pipe: io::Result<(OwnedFd, OwnedFd)>) { - let (mut read_end, write_end) = unwrap_ok(pipe); + let (read_end, write_end) = unwrap_ok(pipe); unwrap_ok(write_all_to(&write_end, b"chunk")); drop(write_end); let mut buffer = [0_u8; 8]; - let read_len = unwrap_ok(read_stream(&mut read_end, &mut buffer)); + let read_len = unwrap_ok(read_stream(&read_end, &mut buffer)); assert_eq!(read_len, 5); assert_eq!(buffer.get(..read_len), Some(&b"chunk"[..])); @@ -66,10 +66,10 @@ fn read_stream_reads_pipe_bytes(#[from(pipe)] pipe: io::Result<(OwnedFd, OwnedFd /// Passing a pipe's write end to the reader reports the underlying I/O error. #[rstest] fn read_stream_reports_unreadable_descriptor(#[from(pipe)] pipe: io::Result<(OwnedFd, OwnedFd)>) { - let (_read_end, mut write_end) = unwrap_ok(pipe); + let (_read_end, write_end) = unwrap_ok(pipe); let mut buffer = [0_u8; 8]; - let err = unwrap_err(read_stream(&mut write_end, &mut buffer)); + let err = unwrap_err(read_stream(&write_end, &mut buffer)); assert!(matches!(err, PumpError::Io(_))); } @@ -81,7 +81,10 @@ fn read_raw_fd_reports_eof(#[from(pipe)] pipe: io::Result<(OwnedFd, OwnedFd)>) { drop(write_end); let mut buffer = [0_u8; 8]; - let read_len = unwrap_ok(read_raw_fd(read_end.as_raw_fd(), &mut buffer)); + let read_len = unwrap_ok(read_raw_fd( + cuprum_native_io::borrow(&read_end), + &mut buffer, + )); assert_eq!(read_len, 0); } @@ -105,9 +108,9 @@ fn read_raw_fd_retries_after_interruption() { /// Writing to an open pipe reports a complete write with its byte count. #[rstest] fn handle_write_returns_complete_outcome(#[from(pipe)] pipe: io::Result<(OwnedFd, OwnedFd)>) { - let (read_end, mut write_end) = unwrap_ok(pipe); + let (read_end, write_end) = unwrap_ok(pipe); - let outcome = unwrap_ok(handle_write(&mut write_end, b"chunk")); + let outcome = unwrap_ok(handle_write(&write_end, b"chunk")); assert_eq!(outcome, WriteOutcome::Complete(5)); drop(read_end); @@ -116,9 +119,9 @@ fn handle_write_returns_complete_outcome(#[from(pipe)] pipe: io::Result<(OwnedFd /// Passing a pipe's read end to the writer propagates the fatal I/O error. #[rstest] fn handle_write_reports_unwritable_descriptor(#[from(pipe)] pipe: io::Result<(OwnedFd, OwnedFd)>) { - let (mut read_end, _write_end) = unwrap_ok(pipe); + let (read_end, _write_end) = unwrap_ok(pipe); - let err = unwrap_err(handle_write(&mut read_end, b"chunk")); + let err = unwrap_err(handle_write(&read_end, b"chunk")); assert!(matches!(err, PumpError::Io(_))); } @@ -345,3 +348,29 @@ fn classify_write_with_propagates_a_fatal_error() { assert!(matches!(err, PumpError::Io(_))); } + +/// Invalid progress must not partially commit the buffer or accounting state. +#[rstest] +#[case::overflow(u64::MAX, 1, "integer length conversion overflowed")] +#[case::oversized(7, 2, "computed range exceeded the buffer bounds")] +fn invalid_progress_leaves_chunk_and_count_unchanged( + #[case] starting_total: u64, + #[case] written: usize, + #[case] expected_message: &str, +) { + let original = b"x".as_slice(); + let mut chunk = original; + let mut total = starting_total; + let error = unwrap_err(super::record_write_progress( + &mut chunk, written, &mut total, + )); + assert_eq!(error.to_string(), expected_message); + assert_eq!( + chunk, original, + "rejected progress must retain the original tail" + ); + assert_eq!( + total, starting_total, + "rejected progress must not commit accounting" + ); +} diff --git a/rust/cuprum-rust/src/io_utils/tracing_tests.rs b/rust/cuprum-streams/src/io_utils/tracing_tests.rs similarity index 100% rename from rust/cuprum-rust/src/io_utils/tracing_tests.rs rename to rust/cuprum-streams/src/io_utils/tracing_tests.rs diff --git a/rust/cuprum-streams/src/lib.rs b/rust/cuprum-streams/src/lib.rs new file mode 100644 index 000000000..2353a1622 --- /dev/null +++ b/rust/cuprum-streams/src/lib.rs @@ -0,0 +1,217 @@ +//! Safe stream orchestration, decoding, and error policy for Cuprum. +#![forbid(unsafe_code)] + +mod buffer; +#[cfg(test)] +mod buffer_size_tests; +#[cfg(all(test, unix))] +mod consume_snapshot_tests; +mod errors; +mod io_utils; +#[cfg(all(test, unix))] +mod lib_tests; +mod pump_machine; +#[cfg(target_os = "linux")] +mod splice; +#[cfg(all(test, unix))] +mod test_support; +#[cfg(all(test, unix))] +mod tracing_capture; +mod utf8; + +use cuprum_native_io::{AsStream, OwnedStream}; +pub use errors::PumpError; +use io_utils::{classify_write, operation_span, read_stream}; +use pump_machine::{Flow, PumpState, advance}; +use utf8::{FinalChunk, decode_utf8_replace}; + +/// Maximum accepted stream buffer size, in bytes (1 GiB). +/// +/// This guards against absurd allocations from a bad `buffer_size` while +/// comfortably exceeding any realistic transfer buffer — the default is +/// 64 KiB and even multi-megabyte buffers stay far below this cap. +const MAX_BUFFER_SIZE: usize = 1 << 30; + +/// Validate and convert a raw `buffer_size` into a bounded `usize`. +/// +/// This is the pure decision core behind [`validate_buffer_size`]: it takes no +/// Python state and returns a stable, message-carrying error so it can be +/// property tested directly. +/// +/// # Errors +/// Returns a stable error message when `buffer_size` is non-positive, overflows +/// `usize` on the target platform, or exceeds [`MAX_BUFFER_SIZE`]. +fn checked_buffer_size(buffer_size: i64) -> Result { + if buffer_size <= 0 { + return Err("buffer_size must be greater than zero"); + } + let size = usize::try_from(buffer_size).map_err(|_| "buffer_size is too large")?; + if size > MAX_BUFFER_SIZE { + return Err("buffer_size exceeds the maximum permitted size"); + } + Ok(size) +} + +/// Validated positive buffer allocation size, capped at 1 GiB. +#[derive(Clone, Copy, Debug)] +pub struct BufferSize(usize); + +impl BufferSize { + /// Validate a buffer size supplied by the integration layer. + /// + /// # Errors + /// Rejects non-positive sizes and sizes above the allocation cap. + pub fn new(size: i64) -> Result { + checked_buffer_size(size).map(Self) + } + + const fn value(self) -> usize { + self.0 + } +} + +/// Pump from a borrowed reader, consuming the uniquely owned writer. +/// +/// The writer drops on success, error, and unwind. The reader's owner must +/// outlive this borrow and remains responsible for closing it. +/// +/// # Errors +/// Returns a semantic I/O, buffer-range, or accounting failure. +pub fn pump_stream( + reader: &impl AsStream, + writer: OwnedStream, + buffer_size: BufferSize, +) -> Result { + cuprum_native_io::with_owned_writer(reader, writer, |source, sink| { + pump_stream_files(source, sink, buffer_size) + }) +} + +/// Decode a borrowed stream as UTF-8 with replacement semantics. +/// +/// # Errors +/// Returns a semantic I/O, buffer-range, or accounting failure. +pub fn consume_stream( + reader: &impl AsStream, + buffer_size: BufferSize, +) -> Result { + consume_stream_files(reader, buffer_size) +} + +fn pump_stream_files( + reader: &impl AsStream, + writer: &impl AsStream, + buffer_size: BufferSize, +) -> Result { + // On Linux, attempt zero-copy splice first. + #[cfg(target_os = "linux")] + if let Some(result) = splice::try_splice_pump(reader, writer, buffer_size.value()) { + return result; + } + + // Fallback: read/write loop for non-Linux or unsupported FD types. + pump_stream_files_readwrite(reader, writer, buffer_size) +} + +/// Read/write loop fallback for pumping bytes between file descriptors. +/// +/// This is used when splice is not available (non-Linux) or when the file +/// descriptors do not support splice (regular files, some sockets). +fn pump_stream_files_readwrite( + reader: &impl AsStream, + writer: &impl AsStream, + buffer_size: BufferSize, +) -> Result { + // Operation span (see `operation_span`) so the EINTR (`warn!`) and + // fatal-I/O (`error!`) events emitted from the read/write seams inherit + // the operation name, `buffer_size`, and `total_bytes` context even under + // a `warn`/`error`-only production filter. + let span = operation_span("pump_stream_readwrite", buffer_size.value()); + let _guard = span.enter(); + io_utils::reset_retry_counters(); + + let mut buffer = buffer::allocate_buffer(buffer_size.value())?; + let mut state = PumpState::start(); + + loop { + let read_len = read_stream(reader, &mut buffer)?; + let writer_was_open = state.writer_open(); + + // `advance` owns both the zero-length-is-EOF translation and the write + // precondition — a chunk read while the writer is still open — so this + // loop, the property tests, and the bounded proofs share one + // definition of them. Fatal writes propagate the real error and never + // reach the pure state machine. + let flow = advance(&mut state, read_len, || { + let chunk = buffer + .get(..read_len) + .ok_or(PumpError::BufferRangeExceeded)?; + classify_write(writer, chunk) + })?; + + // The latch closing is the `head`-style early exit. Mirror splice's + // field and message so the event is not visible on one path only, and + // observe it here rather than in the deliberately pure `pump_machine`. + if writer_was_open && !state.writer_open() { + tracing::debug!( + bytes_transferred = state.total_written(), + "broken pipe; draining reader" + ); + } + + if flow == Flow::Stop { + break; + } + } + + let total_written = state.total_written(); + span.record("total_bytes", total_written); + span.record("read_retries", io_utils::read_retry_count()); + span.record("write_retries", io_utils::write_retry_count()); + Ok(total_written) +} + +fn consume_stream_files( + reader: &impl AsStream, + buffer_size: BufferSize, +) -> Result { + // Operation span (see `operation_span`) so the read seam's `warn!`/`error!` + // events inherit this operation's context even under a `warn`/`error`-only + // production filter. + let span = operation_span("consume_stream", buffer_size.value()); + let _guard = span.enter(); + io_utils::reset_retry_counters(); + + let mut buffer = buffer::allocate_buffer(buffer_size.value())?; + let mut pending: Vec = Vec::new(); + let mut output = String::new(); + let mut total_read = 0_u64; + + #[cfg(unix)] + let platform = "unix"; + #[cfg(windows)] + let platform = "windows"; + + loop { + let read_len = read_stream(reader, &mut buffer)?; + if read_len == 0 { + break; + } + let read_bytes = u64::try_from(read_len).map_err(|_| { + tracing::error!(platform, "read length conversion overflowed"); + PumpError::LengthOverflow + })?; + total_read = total_read.saturating_add(read_bytes); + let chunk = buffer + .get(..read_len) + .ok_or(PumpError::BufferRangeExceeded)?; + pending.extend_from_slice(chunk); + decode_utf8_replace(&mut pending, &mut output, FinalChunk::new(false)); + } + + decode_utf8_replace(&mut pending, &mut output, FinalChunk::new(true)); + + span.record("total_bytes", total_read); + span.record("read_retries", io_utils::read_retry_count()); + Ok(output) +} diff --git a/rust/cuprum-rust/src/lib_tests.rs b/rust/cuprum-streams/src/lib_tests.rs similarity index 60% rename from rust/cuprum-rust/src/lib_tests.rs rename to rust/cuprum-streams/src/lib_tests.rs index 99d037b71..4726d562c 100644 --- a/rust/cuprum-rust/src/lib_tests.rs +++ b/rust/cuprum-streams/src/lib_tests.rs @@ -1,99 +1,15 @@ //! Tests for the borrowed file-descriptor ownership contract. -use std::io::{self, Read}; -use std::os::fd::{AsRawFd, FromRawFd, IntoRawFd, OwnedFd}; -use std::panic::{AssertUnwindSafe, catch_unwind}; - +use crate::buffer::ALLOCATION_FAILED_CATEGORY; use crate::errors::PumpError; use crate::io_utils::{classify_write, read_stream}; use crate::pump_machine::WriteEvent; -use crate::test_support::{fd_is_open, make_pipe, read_all_from, write_all_to}; +use crate::test_support::{make_pipe, read_all_from, unwrap_ok, write_all_to}; use crate::tracing_capture::capture; -use crate::{ - BufferSize, consume_stream_files, pump_stream_files_readwrite, stream_from_raw, - with_borrowed_reader, -}; -use rstest::{fixture, rstest}; +use crate::{BufferSize, consume_stream_files, pump_stream_files_readwrite}; +use rstest::rstest; use tracing::Level; -struct BorrowedReaderPipe { - read_end: OwnedFd, - write_end: OwnedFd, - raw_fd: i32, -} - -enum BorrowedReaderScenario { - Panic, - Success, -} - -/// Builds a pipe whose read descriptor can be borrowed across an operation. -#[fixture] -fn borrowed_reader_pipe() -> io::Result { - let (read_end, write_end) = make_pipe()?; - let raw_fd = read_end.as_raw_fd(); - - Ok(BorrowedReaderPipe { - read_end, - write_end, - raw_fd, - }) -} - -/// A borrowed reader keeps its descriptor open after success or panic. -#[rstest] -#[case::panicking_operation(BorrowedReaderScenario::Panic)] -#[case::successful_operation(BorrowedReaderScenario::Success)] -fn borrowed_reader_stays_open_after_operation( - #[from(borrowed_reader_pipe)] borrowed_reader_pipe_result: io::Result, - #[case] scenario: BorrowedReaderScenario, -) { - let BorrowedReaderPipe { - read_end, - write_end, - raw_fd, - } = crate::test_support::unwrap_ok(borrowed_reader_pipe_result); - - match scenario { - BorrowedReaderScenario::Panic => assert_panicking_reader_keeps_fd_open(raw_fd), - BorrowedReaderScenario::Success => { - let collected = - crate::test_support::unwrap_ok(successful_reader_bytes(raw_fd, write_end)); - assert_eq!( - collected, b"ping", - "the borrowed reader must retain the pipe bytes", - ); - } - } - - assert!(fd_is_open(raw_fd), "the borrowed FD must remain open"); - drop(read_end); -} - -/// Transferring a raw descriptor creates an owning stream that reads its data. -#[rstest] -fn stream_from_raw_owns_and_reads_the_descriptor() { - let (read_end, write_end) = crate::test_support::unwrap_ok(make_pipe()); - crate::test_support::unwrap_ok(write_all_to(&write_end, b"pong")); - drop(write_end); - - // Transfer ownership of the read descriptor into the constructed handle - // exactly once, then read the pending bytes back through it. - let raw_fd = read_end.into_raw_fd(); - let mut handle = stream_from_raw(raw_fd); - let mut buffer = [0_u8; 8]; - - let read_len = match read_stream(&mut handle, &mut buffer) { - Ok(read_len) => read_len, - Err(err) => panic!("read through stream_from_raw handle failed: {err:?}"), - }; - - assert_eq!(buffer.get(..read_len), Some(&b"pong"[..])); - // Dropping `handle` closes the owned descriptor. - drop(handle); - assert!(!fd_is_open(raw_fd), "the owned FD must be closed on drop"); -} - /// Consuming a pipe records its byte total and zero read retries in the span. #[rstest] fn consume_records_total_bytes_and_retries_on_span() { @@ -103,10 +19,10 @@ fn consume_records_total_bytes_and_retries_on_span() { // Close the write end so the read loop reaches EOF and terminates. drop(write_end); - let mut reader = read_end; + let reader = read_end; let expected = std::str::from_utf8(payload).unwrap_or(""); let captured = capture(Level::INFO, || { - match consume_stream_files(&mut reader, BufferSize(64)) { + match consume_stream_files(&reader, BufferSize(64)) { Ok(text) => assert_eq!(text, expected, "consume must decode the exact payload"), Err(err) => panic!("consume over a closed pipe failed: {err:?}"), } @@ -141,10 +57,10 @@ fn pump_records_span_fields_under_error_filter() { // Close the source's write end so the read loop reaches EOF. drop(source_write); - let mut reader = source_read; - let mut writer = sink_write; + let reader = source_read; + let writer = sink_write; let captured = capture(Level::ERROR, || { - match pump_stream_files_readwrite(&mut reader, &mut writer, BufferSize(64)) { + match pump_stream_files_readwrite(&reader, &writer, BufferSize(64)) { Ok(total) => assert_eq!(total, u64::try_from(payload.len()).unwrap_or(u64::MAX)), Err(err) => panic!("pump over pipes failed: {err:?}"), } @@ -152,9 +68,8 @@ fn pump_records_span_fields_under_error_filter() { // Close the write end, then confirm the sink received exactly the source // payload — a data oracle so same-length corruption cannot pass. drop(writer); - let received = crate::test_support::unwrap_ok(read_all_from(&sink_read)); assert_eq!( - received.as_slice(), + crate::test_support::unwrap_ok(read_all_from(&sink_read)).as_slice(), &payload[..], "the pump must deliver the source bytes unchanged to the sink", ); @@ -185,22 +100,87 @@ fn pump_records_span_fields_under_error_filter() { ); } -fn assert_panicking_reader_keeps_fd_open(raw_fd: i32) { - let outcome = catch_unwind(AssertUnwindSafe(|| { - with_borrowed_reader(raw_fd, |_reader| { - panic!("simulated failure inside the borrowed scope"); - }); - })); +/// A refused scratch allocation emits a bounded, categorized `error` event. +/// +/// Captured at `ERROR` level, exactly the filter a production subscriber is +/// most likely to run: an event that needed a finer level to be seen would +/// have diagnostic value that depends on configuration. +#[rstest] +fn allocation_failure_reports_a_bounded_category_event() { + let captured = capture(Level::ERROR, || { + match crate::buffer::allocate_buffer(usize::MAX) { + Ok(buffer) => panic!( + "an unallocatable buffer must fail, got {} bytes", + buffer.len() + ), + Err(error) => assert!( + matches!(error, PumpError::BufferAllocationFailed), + "the refusal must surface as the typed error, got {error:?}", + ), + } + }); + + assert!( + captured.event_matches( + Level::ERROR, + "scratch buffer allocation refused", + &[ + ("error_category", ALLOCATION_FAILED_CATEGORY), + // The request cannot be rounded up past the top of the range, + // so the bucket reports the request itself: an exact, bounded + // value, never an allocator internal. + ("buffer_size", &usize::MAX.to_string()), + // The platform field matches the I/O seam events, so an + // embedding application filters every stream event alike. + ("platform", "unix"), + ], + ), + "a refused allocation must emit its stable category and bounded buffer \ + size at error level", + ); +} + +/// The pump's allocation failure keeps the operation span's context. +#[rstest] +fn allocation_failure_inside_the_pump_span_keeps_its_context() { + // The realistic wiring: `pump_stream_files_readwrite` enters its operation + // span before allocating, so the event is emitted inside that span and + // inherits the operation context even under an ERROR-only filter. + let (reader, _reader_writer) = unwrap_ok(make_pipe()); + let (_sink_reader, writer) = unwrap_ok(make_pipe()); + let captured = capture(Level::ERROR, || { + match pump_stream_files_readwrite(&reader, &writer, BufferSize(usize::MAX)) { + Ok(total) => panic!("an unallocatable buffer must fail the pump, moved {total}"), + Err(error) => assert!( + matches!(error, PumpError::BufferAllocationFailed), + "the pump must propagate the allocation failure, got {error:?}", + ), + } + }); - assert!(outcome.is_err(), "the panic must propagate to the caller"); + assert!( + captured.event_matches( + Level::ERROR, + "scratch buffer allocation refused", + &[("error_category", ALLOCATION_FAILED_CATEGORY)], + ), + "the allocation event must fire from the pump with its stable category", + ); + assert_eq!( + captured + .span_field("pump_stream_readwrite", "operation") + .as_deref(), + Some("pump_stream_readwrite"), + "the event must sit inside the pump's operation span", + ); } /// A successful write is classified with the number of bytes delivered. #[rstest] fn classify_write_reports_a_completed_write() { - let (read_end, mut write_end) = crate::test_support::unwrap_ok(make_pipe()); + let (read_end, write_end) = crate::test_support::unwrap_ok(make_pipe()); - let event = match classify_write(&mut write_end, b"chunk") { + let event = match classify_write(&write_end, b"chunk") { Ok(event) => event, Err(err) => panic!("write to an open pipe failed: {err:?}"), }; @@ -215,29 +195,14 @@ fn classify_write_reports_a_completed_write() { fn classify_write_propagates_a_fatal_error() { // Writing to the read end of a pipe is a fatal `EBADF`, which must // propagate rather than latch the writer closed. - let (mut read_end, _write_end) = crate::test_support::unwrap_ok(make_pipe()); + let (read_end, _write_end) = crate::test_support::unwrap_ok(make_pipe()); - match classify_write(&mut read_end, b"chunk") { + match classify_write(&read_end, b"chunk") { Ok(event) => panic!("expected a fatal write error, got {event:?}"), Err(err) => assert!(matches!(err, PumpError::Io(_))), } } -/// Writes a probe payload and reads it through the borrowed descriptor. -fn successful_reader_bytes(raw_fd: i32, write_end: OwnedFd) -> io::Result> { - write_all_to(&write_end, b"ping")?; - drop(write_end); - - with_borrowed_reader(raw_fd, |reader| { - // SAFETY: reading through the borrowed handle's raw descriptor. - let mut file = - unsafe { std::mem::ManuallyDrop::new(std::fs::File::from_raw_fd(reader.as_raw_fd())) }; - let mut data = Vec::new(); - file.read_to_end(&mut data)?; - Ok(data) - }) -} - /// A broken writer still drains the reader before the pump reports completion. #[rstest] fn pump_drains_the_reader_after_the_writer_breaks() { @@ -254,13 +219,13 @@ fn pump_drains_the_reader_after_the_writer_breaks() { // Close the sink's read end so every write fails with a broken pipe. drop(sink_read); - let mut reader = source_read; - let mut writer = sink_write; + let reader = source_read; + let writer = sink_write; // A small buffer forces several read iterations, so the drain path runs // repeatedly after the writer has latched closed rather than just once. let mut total = 0_u64; let captured = capture(Level::DEBUG, || { - total = match pump_stream_files_readwrite(&mut reader, &mut writer, BufferSize(8)) { + total = match pump_stream_files_readwrite(&reader, &writer, BufferSize(8)) { Ok(delivered) => delivered, Err(err) => panic!("a broken downstream pipe must not fail the pump: {err:?}"), }; @@ -287,7 +252,7 @@ fn pump_drains_the_reader_after_the_writer_breaks() { // The reader must have been drained to EOF: a further read returns zero // rather than the bytes the pump skipped. let mut buffer = [0_u8; 8]; - let remaining = match read_stream(&mut reader, &mut buffer) { + let remaining = match read_stream(&reader, &mut buffer) { Ok(remaining) => remaining, Err(err) => panic!("reading the drained source failed: {err:?}"), }; diff --git a/rust/cuprum-rust/src/pump_machine.rs b/rust/cuprum-streams/src/pump_machine.rs similarity index 100% rename from rust/cuprum-rust/src/pump_machine.rs rename to rust/cuprum-streams/src/pump_machine.rs diff --git a/rust/cuprum-rust/src/pump_machine_kani_proofs.rs b/rust/cuprum-streams/src/pump_machine_kani_proofs.rs similarity index 100% rename from rust/cuprum-rust/src/pump_machine_kani_proofs.rs rename to rust/cuprum-streams/src/pump_machine_kani_proofs.rs diff --git a/rust/cuprum-rust/src/pump_machine_tests.rs b/rust/cuprum-streams/src/pump_machine_tests.rs similarity index 100% rename from rust/cuprum-rust/src/pump_machine_tests.rs rename to rust/cuprum-streams/src/pump_machine_tests.rs diff --git a/rust/cuprum-rust/src/splice/mod.rs b/rust/cuprum-streams/src/splice/mod.rs similarity index 84% rename from rust/cuprum-rust/src/splice/mod.rs rename to rust/cuprum-streams/src/splice/mod.rs index 2c62f101c..8c2b7444d 100644 --- a/rust/cuprum-rust/src/splice/mod.rs +++ b/rust/cuprum-streams/src/splice/mod.rs @@ -15,18 +15,12 @@ //! Splice requires at least one pipe endpoint. Regular files, most sockets, //! and other file descriptor types return `EINVAL` and trigger the fallback. +use cuprum_native_io::{AsStream, BorrowedStream, borrow}; use std::io; -use std::os::fd::{AsRawFd, OwnedFd}; use crate::errors::PumpError; use crate::io_utils::read_raw_fd; -/// Flag for splice: move pages instead of copying (advisory). -const SPLICE_F_MOVE: libc::c_uint = 0x01; - -/// Flag for splice: more data will be sent (for TCP corking). -const SPLICE_F_MORE: libc::c_uint = 0x04; - /// Attempt to pump data using `splice()`. Returns None if splice is not supported. /// /// # Parameters @@ -41,12 +35,12 @@ const SPLICE_F_MORE: libc::c_uint = 0x04; /// - `Some(Err(e))`: Fatal I/O error occurred /// - `None`: splice not supported for these FDs; caller should use read/write pub(crate) fn try_splice_pump( - reader: &OwnedFd, - writer: &OwnedFd, + reader: &impl AsStream, + writer: &impl AsStream, chunk_size: usize, ) -> Option> { - let reader_fd = reader.as_raw_fd(); - let writer_fd = writer.as_raw_fd(); + let reader_fd = borrow(reader); + let writer_fd = borrow(writer); // The first splice call detects support: `EINVAL` here means the FD // types cannot splice and the caller must fall back to read/write. Once @@ -66,27 +60,12 @@ pub(crate) fn try_splice_pump( /// write policies in [`crate::io_utils`]: a signal delivered mid-transfer /// re-issues the splice rather than surfacing a spurious failure that would /// abort an otherwise-healthy pump. -fn splice_once(fd_in: libc::c_int, fd_out: libc::c_int, len: usize) -> Result { - let flags = SPLICE_F_MOVE | SPLICE_F_MORE; - splice_once_with(|| { - // SAFETY: splice is a well-defined syscall; null offsets are valid for pipes. - let result = unsafe { - libc::splice( - fd_in, - std::ptr::null_mut(), // No offset for pipes - fd_out, - std::ptr::null_mut(), // No offset for pipes - len, - flags, - ) - }; - - if result >= 0 { - Ok(result) - } else { - Err(io::Error::last_os_error()) - } - }) +fn splice_once( + fd_in: BorrowedStream<'_>, + fd_out: BorrowedStream<'_>, + len: usize, +) -> Result { + splice_once_with(|| cuprum_native_io::splice_once(fd_in, fd_out, len)) } /// Retry an interrupted splice syscall and map its result into [`PumpError`]. @@ -121,8 +100,8 @@ fn splice_once_with( /// reader so upstream does not block, then reports the bytes transferred so /// far. fn splice_loop( - fd_in: libc::c_int, - fd_out: libc::c_int, + fd_in: BorrowedStream<'_>, + fd_out: BorrowedStream<'_>, chunk_size: usize, first: Result, ) -> Result { @@ -178,8 +157,8 @@ fn accumulate_splices( /// canonical raw-fd read helper, so interrupted reads (`EINTR`) retry /// instead of silently ending the drain, end of file terminates it, and /// any other error propagates. -fn drain_reader(fd_in: libc::c_int, chunk_size: usize) -> Result<(), PumpError> { - let mut buf = vec![0_u8; chunk_size]; +fn drain_reader(fd_in: BorrowedStream<'_>, chunk_size: usize) -> Result<(), PumpError> { + let mut buf = crate::buffer::allocate_buffer(chunk_size)?; loop { let n = read_raw_fd(fd_in, &mut buf)?; if n == 0 { diff --git a/rust/cuprum-rust/src/splice/tests.rs b/rust/cuprum-streams/src/splice/tests.rs similarity index 98% rename from rust/cuprum-rust/src/splice/tests.rs rename to rust/cuprum-streams/src/splice/tests.rs index 2b7c0154d..310711d95 100644 --- a/rust/cuprum-rust/src/splice/tests.rs +++ b/rust/cuprum-streams/src/splice/tests.rs @@ -3,7 +3,7 @@ //! unsupported descriptor types, and broken-pipe draining. use std::io::{self, Write}; -use std::os::fd::{AsRawFd, OwnedFd}; +use std::os::fd::OwnedFd; use proptest::prelude::*; use rstest::{fixture, rstest}; @@ -65,7 +65,7 @@ fn unsupported_descriptors_signal_fallback() { // panic leaves nothing behind. let mut reader_file = unwrap_ok(NamedTempFile::new()); let writer_file = unwrap_ok(NamedTempFile::new()); - unwrap_ok(reader_file.as_file_mut().write_all(b"file payload")); + unwrap_ok(reader_file.write_all(b"file payload")); // `reopen()` hands back an independent descriptor to the same inode, // verified against replacement, rather than a racy path lookup. let reader = OwnedFd::from(unwrap_ok(reader_file.reopen())); @@ -116,7 +116,7 @@ fn drain_reader_consumes_to_eof(#[from(pipe)] pipe_result: io::Result) unwrap_ok(write_all_to(&write_end, b"residual data")); drop(write_end); - unwrap_ok(drain_reader(read_end.as_raw_fd(), 8)); + unwrap_ok(drain_reader(cuprum_native_io::borrow(&read_end), 8)); let leftover = unwrap_ok(read_all_from(&read_end)); assert!(leftover.is_empty(), "drain must consume the pipe to EOF"); diff --git a/rust/cuprum-rust/src/test_support.rs b/rust/cuprum-streams/src/test_support.rs similarity index 50% rename from rust/cuprum-rust/src/test_support.rs rename to rust/cuprum-streams/src/test_support.rs index e7ee761b6..d3dbaa827 100644 --- a/rust/cuprum-rust/src/test_support.rs +++ b/rust/cuprum-streams/src/test_support.rs @@ -1,39 +1,23 @@ //! Shared Unix file-descriptor fixtures and assertions for Rust unit tests. +use cap_std::fs::File; use std::fmt::Debug; -use std::fs::File; use std::io::{self, Read, Write}; -use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; +use std::os::fd::OwnedFd; /// Create an anonymous pipe as `(read_end, write_end)`. /// /// Returns the operating-system error from `pipe(2)` so fixtures can pass it /// to the test body, where the failure receives assertion context. pub(crate) fn make_pipe() -> io::Result<(OwnedFd, OwnedFd)> { - let mut fds = [0_i32; 2]; - // SAFETY: `fds` is a valid two-element array for `pipe(2)` to fill. - let rc = unsafe { libc::pipe(fds.as_mut_ptr()) }; - if rc == -1 { - return Err(io::Error::last_os_error()); - } - // SAFETY: on success `pipe(2)` returned two freshly opened descriptors - // that this process exclusively owns. - Ok(unsafe { (OwnedFd::from_raw_fd(fds[0]), OwnedFd::from_raw_fd(fds[1])) }) + cuprum_native_io::pipe() } -/// Duplicate a descriptor view into an independently owned [`File`]. +/// Duplicate a typed descriptor into an independently owned [`File`]. /// -/// The caller retains ownership of its descriptor. Returning the `dup(2)` -/// error also permits tests to exercise invalid descriptor handling safely. -pub(crate) fn dup_as_file(fd: &impl AsRawFd) -> io::Result { - // SAFETY: duplicating the supplied descriptor view for a scoped `File` wrapper. - let duplicated_fd = unsafe { libc::dup(fd.as_raw_fd()) }; - if duplicated_fd == -1 { - return Err(io::Error::last_os_error()); - } - // SAFETY: `duplicated_fd` was checked for `dup(2)` failure above and is - // now owned by this scoped `File`. - Ok(unsafe { File::from_raw_fd(duplicated_fd) }) +/// The caller retains its descriptor; cloning failures propagate to the test. +pub(crate) fn dup_as_file(fd: &OwnedFd) -> io::Result { + fd.try_clone().map(File::from) } /// Write every byte of `payload` through a duplicated descriptor. @@ -73,22 +57,6 @@ pub(crate) fn unwrap_err(result: Result) -> E { } } -/// Report whether `fd` remains open without taking ownership of it. -pub(crate) fn fd_is_open(fd: i32) -> bool { - loop { - // SAFETY: F_GETFD on an arbitrary integer reports EBADF for a closed - // descriptor without dereferencing memory. - let result = unsafe { libc::fcntl(fd, libc::F_GETFD) }; - if result != -1 { - return true; - } - - if io::Error::last_os_error().raw_os_error() != Some(libc::EINTR) { - return false; - } - } -} - #[cfg(test)] #[path = "test_support_tests.rs"] mod tests; diff --git a/rust/cuprum-rust/src/test_support_tests.rs b/rust/cuprum-streams/src/test_support_tests.rs similarity index 75% rename from rust/cuprum-rust/src/test_support_tests.rs rename to rust/cuprum-streams/src/test_support_tests.rs index b0acee16d..86ab66e24 100644 --- a/rust/cuprum-rust/src/test_support_tests.rs +++ b/rust/cuprum-streams/src/test_support_tests.rs @@ -17,12 +17,16 @@ fn assert_bad_file_descriptor(error: &io::Error, operation: &str) { ); } -/// Invalid descriptors preserve the native error returned by `dup(2)`. +/// Dropping a duplicated reader leaves the original owner usable. #[test] -fn dup_as_file_returns_an_error_for_an_invalid_raw_descriptor() { - let error = unwrap_err(dup_as_file(&-1)); - - assert_bad_file_descriptor(&error, "dup(2)"); +fn dup_as_file_preserves_the_original_owner() { + let (read_end, write_end) = unwrap_ok(make_pipe()); + let duplicate = unwrap_ok(dup_as_file(&read_end)); + drop(duplicate); + unwrap_ok(write_all_to(&write_end, b"retained owner")); + drop(write_end); + + assert_eq!(unwrap_ok(read_all_from(&read_end)), b"retained owner"); } /// Attempting to write through a pipe read end returns `EBADF`. diff --git a/rust/cuprum-rust/src/tracing_capture.rs b/rust/cuprum-streams/src/tracing_capture.rs similarity index 100% rename from rust/cuprum-rust/src/tracing_capture.rs rename to rust/cuprum-streams/src/tracing_capture.rs diff --git a/rust/cuprum-rust/src/tracing_capture_tests.rs b/rust/cuprum-streams/src/tracing_capture_tests.rs similarity index 100% rename from rust/cuprum-rust/src/tracing_capture_tests.rs rename to rust/cuprum-streams/src/tracing_capture_tests.rs diff --git a/rust/cuprum-rust/src/utf8.rs b/rust/cuprum-streams/src/utf8.rs similarity index 96% rename from rust/cuprum-rust/src/utf8.rs rename to rust/cuprum-streams/src/utf8.rs index 353d8a921..8e548f005 100644 --- a/rust/cuprum-rust/src/utf8.rs +++ b/rust/cuprum-streams/src/utf8.rs @@ -79,13 +79,14 @@ fn append_valid_prefix(pending: &[u8], output: &mut String, valid_up_to: ValidUp if valid_up_to.value() == 0 { return; } - // SAFETY: `valid_up_to` comes from a `Utf8Error`, so this prefix is known - // to be valid UTF-8. + // The prefix normally comes from Utf8Error; keep validation local so + // an incorrect bound cannot introduce undefined behaviour. let Some(prefix_bytes) = pending.get(..valid_up_to.value()) else { return; }; - let valid_prefix = unsafe { std::str::from_utf8_unchecked(prefix_bytes) }; - output.push_str(valid_prefix); + if let Ok(valid_prefix) = std::str::from_utf8(prefix_bytes) { + output.push_str(valid_prefix); + } } /// Handle a UTF-8 decoding error by replacing invalid bytes. diff --git a/rust/cuprum-rust/src/utf8_kani_proofs.rs b/rust/cuprum-streams/src/utf8_kani_proofs.rs similarity index 91% rename from rust/cuprum-rust/src/utf8_kani_proofs.rs rename to rust/cuprum-streams/src/utf8_kani_proofs.rs index 3bd0240f9..2d7afc7bf 100644 --- a/rust/cuprum-rust/src/utf8_kani_proofs.rs +++ b/rust/cuprum-streams/src/utf8_kani_proofs.rs @@ -133,10 +133,9 @@ fn utf8_error_drain_bound_stays_in_range() { } } -/// Symbolic bound: the unsafe `from_utf8_unchecked` inside -/// `append_valid_prefix` produces exactly the same string as a checked -/// decode of the same prefix, for *any* short payload. This upholds the -/// SAFETY claim that the `valid_up_to` prefix is genuinely valid UTF-8. +/// For every three-byte payload, append exactly its valid UTF-8 prefix. +/// The checked decoder replaces the former unchecked conversion; this proof +/// retains the prefix-selection contract rather than claiming residual unsafety. #[kani::proof] #[kani::unwind(4)] fn append_valid_prefix_matches_checked_decode() { @@ -147,11 +146,11 @@ fn append_valid_prefix_matches_checked_decode() { }; // Cover both branches of `append_valid_prefix`: the non-empty prefix that - // reaches the `unsafe from_utf8_unchecked` call, and the `valid_up_to == 0` + // reaches the checked decode, and the `valid_up_to == 0` // early return. Pairing them proves neither became unreachable. kani::cover!( valid_up_to > 0, - "reaches the unchecked decode of a non-empty prefix" + "reaches the checked decode of a non-empty prefix" ); kani::cover!(valid_up_to == 0, "reaches the empty-prefix early return"); @@ -159,11 +158,11 @@ fn append_valid_prefix_matches_checked_decode() { append_valid_prefix(&bytes, &mut output, ValidUpTo(valid_up_to)); // The prefix is valid UTF-8 by construction, so the checked decode - // cannot fail; the unchecked path must produce the identical string. + // cannot fail; the append operation must produce the identical string. let prefix = &bytes[..valid_up_to]; let checked = core::str::from_utf8(prefix).expect("the valid_up_to prefix decodes as UTF-8"); kani::assert( output == checked, - "unchecked prefix decode must equal the checked decode", + "appended prefix must equal the checked decode", ); } diff --git a/rust/cuprum-streams/tests/compile_tests.rs b/rust/cuprum-streams/tests/compile_tests.rs new file mode 100644 index 000000000..f52935fdc --- /dev/null +++ b/rust/cuprum-streams/tests/compile_tests.rs @@ -0,0 +1,6 @@ +//! Compile contracts for safe stream policy. +#![forbid(unsafe_code)] +#[test] +fn transition_privacy() { + trybuild::TestCases::new().compile_fail("tests/ui/fail/*.rs"); +} diff --git a/rust/cuprum-rust/tests/ui/fail/pump_transition_unreachable.rs b/rust/cuprum-streams/tests/ui/fail/pump_transition_unreachable.rs similarity index 100% rename from rust/cuprum-rust/tests/ui/fail/pump_transition_unreachable.rs rename to rust/cuprum-streams/tests/ui/fail/pump_transition_unreachable.rs diff --git a/rust/cuprum-rust/tests/ui/fail/pump_transition_unreachable.stderr b/rust/cuprum-streams/tests/ui/fail/pump_transition_unreachable.stderr similarity index 100% rename from rust/cuprum-rust/tests/ui/fail/pump_transition_unreachable.stderr rename to rust/cuprum-streams/tests/ui/fail/pump_transition_unreachable.stderr diff --git a/rust/cuprum-streams/tests/ui/fail/replace_scoped_writer.rs b/rust/cuprum-streams/tests/ui/fail/replace_scoped_writer.rs new file mode 100644 index 000000000..42f740bc5 --- /dev/null +++ b/rust/cuprum-streams/tests/ui/fail/replace_scoped_writer.rs @@ -0,0 +1,16 @@ +//! A scoped writer callback must not transfer its handle back out. + +use cuprum_native_io::with_owned_writer; + +fn escape_writer(writer: W, replacement: W) -> W { + with_owned_writer(&(), writer, |_, scoped_writer| { + std::mem::replace(scoped_writer, replacement) + }) +} + +fn main() -> Result<(), std::io::Error> { + let (_, writer) = cuprum_native_io::pipe()?; + let (_, replacement) = cuprum_native_io::pipe()?; + drop(escape_writer(writer, replacement)); + Ok(()) +} diff --git a/rust/cuprum-streams/tests/ui/fail/replace_scoped_writer.stderr b/rust/cuprum-streams/tests/ui/fail/replace_scoped_writer.stderr new file mode 100644 index 000000000..dd9951218 --- /dev/null +++ b/rust/cuprum-streams/tests/ui/fail/replace_scoped_writer.stderr @@ -0,0 +1,12 @@ +error[E0308]: mismatched types + --> tests/ui/fail/replace_scoped_writer.rs:7:27 + | +7 | std::mem::replace(scoped_writer, replacement) + | ----------------- ^^^^^^^^^^^^^ types differ in mutability + | | + | arguments to this function are incorrect + | + = note: expected mutable reference `&mut W` + found reference `&W` +note: function defined here + --> $RUST/core/src/mem/mod.rs diff --git a/rust/dylint.toml b/rust/dylint.toml index 575fe853d..0321fa3b5 100644 --- a/rust/dylint.toml +++ b/rust/dylint.toml @@ -1,5 +1,4 @@ [no_std_fs_operations] -# _rust_backend_native never opens filesystem paths itself; it wraps raw file -# descriptors and handles received from the Python caller in std::fs::File, so -# capability-based alternatives such as cap-std do not apply here. -excluded_crates = ["_rust_backend_native"] +# Descriptor fixtures and the Windows adapter use cap-std. Neither the safe +# policy crate nor either audited boundary requires a filesystem exclusion. +excluded_crates = [] diff --git a/scripts/check_boundary_contract.py b/scripts/check_boundary_contract.py new file mode 100644 index 000000000..1867a9cf1 --- /dev/null +++ b/scripts/check_boundary_contract.py @@ -0,0 +1,165 @@ +#!/usr/bin/env -S uv run python +# /// script +# requires-python = ">=3.13" +# dependencies = ["cuprum==0.1.0"] +# /// +"""Compile deliberate unsafe probes against a copy of the actual safe crate. + +The workspace copy avoids touching a developer's sources while proving that +the real library and integration-test targets reject unsafe code. Cargo keeps +its shared package cache. Only build output uses a separate target directory. +""" + +from __future__ import annotations + +import contextlib +import shutil +import sys +import tomllib +import typing as typ +from pathlib import Path + +if typ.TYPE_CHECKING: + import collections.abc as cabc + +from cuprum import Program, ProgramCatalogue, ProjectSettings, ScopeConfig, scoped, sh + +ROOT = Path(__file__).resolve().parent.parent +BOUNDARIES = frozenset({"cuprum-rust", "cuprum-native-io"}) +SAFE_CRATES = frozenset({"cuprum-streams"}) +PROBES = ( + "pub fn unsafe_probe() { unsafe {} }", + "pub unsafe fn unsafe_probe() {}", + "unsafe trait Probe {} unsafe impl Probe for () {}", +) + + +def check_members(manifest: str) -> None: + """Require an explicit review before the approved crate set can grow. + + Parameters + ---------- + manifest : str + Workspace manifest text. + + Raises + ------ + ValueError + If workspace membership differs from the audited set. + """ + members = tomllib.loads(manifest)["workspace"]["members"] + if set(members) != BOUNDARIES | SAFE_CRATES or len(members) != len( + BOUNDARIES | SAFE_CRATES + ): + msg = "workspace membership differs from the audited boundary inventory" + raise ValueError(msg) + + +def check_safe_policy(workspace_manifest: str, safe_manifest: str) -> None: + """Require workspace lint parity plus an unrelaxable safe-target prohibition. + + Parameters + ---------- + workspace_manifest : str + Workspace manifest containing the shared lint policy. + safe_manifest : str + Safe crate manifest, whose lint policy applies to every Cargo target. + + Raises + ------ + ValueError + If the safe crate weakens or diverges from the workspace policy. + """ + expected = tomllib.loads(workspace_manifest)["workspace"]["lints"] + expected["rust"]["unsafe_code"] = "forbid" + actual = tomllib.loads(safe_manifest)["lints"] + if actual != expected: + msg = "safe crate must retain workspace lints and forbid unsafe in all targets" + raise ValueError(msg) + + +def _compile(workspace: Path) -> tuple[int, str]: + """Compile the copied safe library and all its targets with all features.""" + cargo_program = Program("cargo") + project = ProjectSettings( + name="boundary-contract", + programs=(cargo_program,), + documentation_locations=("docs/rust-boundary-verification.md",), + noise_rules=(), + ) + cargo = sh.make(cargo_program, catalogue=ProgramCatalogue(projects=(project,))) + context = sh.ExecutionContext( + cwd=workspace, + env={"CARGO_TARGET_DIR": str(ROOT / "rust/target/boundary-contract")}, + timeout=600, + ) + with scoped(ScopeConfig(allowlist=frozenset({cargo_program}))): + result = cargo( + "check", "--package", "cuprum-streams", "--all-targets", "--all-features" + ).run_sync(context=context) + return result.exit_code, (result.stdout or "") + (result.stderr or "") + + +def main() -> None: + """Check the allowlist and actual compiler rejection on isolated source copies.""" + manifest = (ROOT / "rust/Cargo.toml").read_text(encoding="utf-8") + check_members(manifest) + check_safe_policy( + manifest, + (ROOT / "rust/cuprum-streams/Cargo.toml").read_text(encoding="utf-8"), + ) + workspace = ROOT / ".cache/boundary-contract/workspace" + if workspace.exists(): + shutil.rmtree(workspace) + shutil.copytree(ROOT / "rust", workspace, ignore=shutil.ignore_patterns("target")) + future_target = workspace / "cuprum-streams/tests/future_target.rs" + future_target.write_text( + "//! Probe automatic Cargo target lint inheritance.\n", encoding="utf-8" + ) + logs = ROOT / "rust/target/boundary-verification" + logs.mkdir(parents=True, exist_ok=True) + code, output = _compile(workspace) + baseline_log = logs / "safe-positive.log" + baseline_log.write_text(output, encoding="utf-8") + if code != 0: + print( + f"safe baseline did not compile (cargo exit {code}); see {baseline_log}", + file=sys.stderr, + ) + raise SystemExit(code) + targets = ("src/lib.rs", "tests/compile_tests.rs", "tests/future_target.rs") + for target in targets: + _check_target(workspace, workspace / "cuprum-streams" / target, logs) + + +def _unsafe_was_forbidden(code: int, output: str) -> bool: + """Require compilation failure with an explicit unsafe-forbid diagnostic.""" + markers = ("forbid(unsafe_code)", "-F unsafe-code") + return code != 0 and any(marker in output for marker in markers) + + +@contextlib.contextmanager +def _probe_appended(path: Path, probe: str) -> cabc.Iterator[None]: + """Temporarily append a probe to a source file, then restore it.""" + original = path.read_text(encoding="utf-8") + path.write_text(original + "\n" + probe + "\n", encoding="utf-8") + try: + yield + finally: + path.write_text(original, encoding="utf-8") + + +def _check_target(workspace: Path, path: Path, logs: Path) -> None: + """Probe one actual target and restore it even if compiler checking fails.""" + for index, probe in enumerate(PROBES): + with _probe_appended(path, probe): + code, output = _compile(workspace) + name = f"{path.stem}-unsafe-{index}.log" + (logs / name).write_text(output, encoding="utf-8") + if not _unsafe_was_forbidden(code, output): + msg = f"safe target did not reject unsafe probe: {name}" + raise RuntimeError(msg) + + +if __name__ == "__main__": + main() diff --git a/scripts/check_boundary_faults.py b/scripts/check_boundary_faults.py new file mode 100644 index 000000000..0421a2edf --- /dev/null +++ b/scripts/check_boundary_faults.py @@ -0,0 +1,264 @@ +#!/usr/bin/env -S uv run python +# /// script +# requires-python = ">=3.13" +# dependencies = ["cuprum==0.1.0"] +# /// +"""Require representative production faults to fail their designated checks. + +All mutations affect a disposable workspace copy. Logs retain counterexamples +and real-unwind failures, and a successful control precedes each mutation. +""" + +from __future__ import annotations + +import dataclasses +import os +import shutil +from pathlib import Path + +from cuprum import Program, ProgramCatalogue, ProjectSettings, ScopeConfig, scoped, sh +from scripts.render_boundary_proofs import render + +ROOT = Path(__file__).resolve().parent.parent + + +@dataclasses.dataclass(frozen=True, slots=True) +class ToolPins: + """Pinned verifier versions, read from the repository's tool manifests.""" + + kani: str + verus: str + + +class BoundaryFaultError(RuntimeError): + """A verifier result the fault harness cannot classify. + + Attributes + ---------- + check_name : str + Name of the check whose verifier result was unexpected. + exit_code : int + Exit code returned by the verifier. + log_path : Path + Archived verifier output for the unexpected result. + """ + + def __init__(self, check_name: str, exit_code: int, log_path: Path) -> None: + """Record the check identity, exit code, and archived log path.""" + msg = f"unexpected fault-sensitivity result: {check_name}; inspect {log_path}" + super().__init__(msg) + self.check_name = check_name + self.exit_code = exit_code + self.log_path = log_path + + +def read_tool_pins() -> ToolPins: + """Read the pinned Kani and Verus versions from the repository. + + The manifests are read when a verification path runs, not at import time, + so importing :func:`mutate` does not require either pin file to exist. + + A missing, unreadable, or empty pin is a hard failure rather than a default: + a blank version would otherwise build a verifier path that does not exist + and report the resulting launch failure as a failed proof. + + Returns + ------- + ToolPins + The versions pinned by ``tools/kani/VERSION`` and + ``tools/verus/VERSION``. + + Raises + ------ + OSError + If a pin file cannot be read. + ValueError + If a pin file exists but records no version. + """ # ruff: ignore[docstring-extraneous-exception] - OSError/ValueError propagate from _read_pin. + return ToolPins( + kani=_read_pin("kani"), + verus=_read_pin("verus"), + ) + + +def _read_pin(tool: str) -> str: + """Return one tool's pinned version, refusing a file that records none.""" + pin = (ROOT / f"tools/{tool}/VERSION").read_text(encoding="utf-8").strip() + if not pin: + msg = f"tools/{tool}/VERSION records no version" + raise ValueError(msg) + return pin + + +def mutate(source: str, old: str, new: str) -> str: + """Apply exactly one fault, failing closed if production has changed. + + Parameters + ---------- + source : str + Actual production source text. + old : str + Unique implementation fragment to replace. + new : str + Deliberately incorrect implementation. + + Returns + ------- + str + Mutated source with exactly one replacement. + + Raises + ------ + ValueError + If the implementation fragment is absent or ambiguous. + """ + if source.count(old) != 1: + msg = "fault no longer matches exactly one production fragment" + raise ValueError(msg) + return source.replace(old, new) + + +def _run(program: str, args: tuple[str, ...], workspace: Path) -> tuple[int, str]: + """Run one allowlisted verification tool with bounded execution time.""" + pins = read_tool_pins() + executable = Program(program) + project = ProjectSettings( + name="boundary-faults", + programs=(executable,), + documentation_locations=("docs/rust-boundary-verification.md",), + noise_rules=(), + ) + command = sh.make(executable, catalogue=ProgramCatalogue(projects=(project,))) + kani = Path.home() / f".kani/kani-{pins.kani}" + context = sh.ExecutionContext( + cwd=workspace, + env={ + "CARGO_TARGET_DIR": str(ROOT / "rust/target/boundary-faults"), + "LD_LIBRARY_PATH": f"{kani}/toolchain/lib:{kani}/lib", + "VERUS_Z3_PATH": str(ROOT / ".cache/boundary-z3/z3"), + "RUSTUP_TOOLCHAIN": "1.98.0" if program != "cargo" else "1.85.0", + }, + timeout=1200, + ) + with scoped(ScopeConfig(allowlist=frozenset({executable}))): + result = command(*args).run_sync(context=context) + return result.exit_code, (result.stdout or "") + (result.stderr or "") + + +@dataclasses.dataclass(frozen=True, slots=True) +class _Check: + """One verifier invocation and its expected diagnostic.""" + + name: str + program: str + args: tuple[str, ...] + expected_failure: str | None = None + + +def _check(check: _Check, workspace: Path, logs: Path) -> None: + """Archive a control or require the specific counterexample diagnostic.""" + code, output = _run(check.program, check.args, workspace) + (logs / f"{check.name}.log").write_text(output, encoding="utf-8") + passed = ( + code == 0 + if check.expected_failure is None + else code != 0 and check.expected_failure in output + ) + if not passed: + raise BoundaryFaultError(check.name, code, logs / f"{check.name}.log") + + +def main() -> None: + """Execute controls and four deliberate faults without editing live sources.""" + workspace = ROOT / ".cache/boundary-faults/workspace" + if workspace.exists(): + shutil.rmtree(workspace) + shutil.copytree(ROOT / "rust", workspace, ignore=shutil.ignore_patterns("target")) + logs = ROOT / "rust/target/boundary-verification/faults" + logs.mkdir(parents=True, exist_ok=True) + _verify_progress_faults(workspace, logs) + _verify_ownership_faults(workspace, logs) + + +def _verify_progress_faults(workspace: Path, logs: Path) -> None: + """Check the production progress kernels against bound and accounting faults.""" + pins = read_tool_pins() + install = Path( + os.environ.get( + "VERUS_INSTALL_DIR", + str(Path.home() / f".local/share/cuprum-verus-{pins.verus}"), + ) + ) + verus = str(install / "verus/verus") + progress = (workspace / "cuprum-native-io/src/progress.rs").read_text( + encoding="utf-8" + ) + proof = workspace / "progress-fault.rs" + proof.write_text(render(progress), encoding="utf-8") + args = (str(proof), "--crate-type=lib") + _check(_Check("verus-control", verus, args), workspace, logs) + for name, old, new in ( + ("invalid-bound", "count <= capacity", "count >= capacity"), + ("incorrect-accounting", "total + written", "total - written"), + ): + proof.write_text(render(mutate(progress, old, new)), encoding="utf-8") + _check( + _Check(name, verus, args, "verification results::"), + workspace, + logs, + ) + + +def _verify_ownership_faults(workspace: Path, logs: Path) -> None: + """Check writer drop and real-unwind retention faults against their controls.""" + memory = workspace / "cuprum-native-io/src/memory.rs" + original = memory.read_text(encoding="utf-8") + ownership = ( + "kani", + "--package", + "cuprum-native-io", + "--harness", + "production_scope_preserves_reader_and_drops_writer", + ) + unwind = ( + "test", + "--package", + "cuprum-native-io", + "--lib", + "memory::tests::retained_owner_survives_real_unwind", + "--", + "--exact", + ) + for name, args, old, new, diagnostic in ( + ( + "leaked-writer", + ownership, + " drop(writer);", + " core::mem::forget(writer);", + "VERIFICATION:- FAILED", + ), + ( + "trailing-forget-unwind", + unwind, + ( + " let mut retained = core::mem::ManuallyDrop::new(value);\n" + " operation(&mut retained)" + ), + ( + " let mut retained = value;\n" + " let result = operation(&mut retained);\n" + " core::mem::forget(retained);\n result" + ), + "test result: FAILED", + ), + ): + try: + _check(_Check(f"{name}-control", "cargo", args), workspace, logs) + memory.write_text(mutate(original, old, new), encoding="utf-8") + _check(_Check(name, "cargo", args, diagnostic), workspace, logs) + finally: + memory.write_text(original, encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/scripts/install_boundary_kani.py b/scripts/install_boundary_kani.py new file mode 100644 index 000000000..2bd64a428 --- /dev/null +++ b/scripts/install_boundary_kani.py @@ -0,0 +1,237 @@ +#!/usr/bin/env -S uv run python +# /// script +# requires-python = ">=3.13" +# dependencies = [] +# /// +"""Install pinned Kani frontend binaries and fetch its verified native bundle. + +The approved frontend is the cargo-quickinstall artefact also pinned by +Netsuke. The upstream bundle supplies CBMC and the matching Rust compiler. +``make install-boundary-kani`` performs the subsequent native setup command; +this script never invokes Cargo or compiles a verifier from source. +""" + +from __future__ import annotations + +import hashlib +import http.client +import os +import shutil +import tarfile +import urllib.parse +from pathlib import Path + +# The repository root the installer reads its version pin from and writes its +# cache beneath. `CUPRUM_BOUNDARY_ROOT` redirects both, so a test or a staging +# wrapper can install into an isolated tree without touching the checkout. +_ROOT_ENV = "CUPRUM_BOUNDARY_ROOT" + +ROOT = Path(__file__).resolve().parent.parent +VERSION = "0.67.0" +TARGET = "x86_64-unknown-linux-gnu" +FRONTEND_NAME = f"kani-verifier-{VERSION}-{TARGET}.tar.gz" +BUNDLE_NAME = f"kani-{VERSION}-{TARGET}.tar.gz" +FRONTEND_MEMBERS = ("cargo-kani", "kani") +FRONTEND_DIGEST = "ed2bafc239b834e14c6b66fc4838e342e3bc0b814e548e72ea30e84f83dc0974" +BUNDLE_DIGEST = "3b5f7afd3b51603ee720db7bc1bc4fe46b5a4f5d36daad9939c4b4c658b51ac0" +FRONTEND_URL = ( + "https://github.com/cargo-bins/cargo-quickinstall/releases/download/" + f"kani-verifier-{VERSION}/{FRONTEND_NAME}" +) +BUNDLE_URL = ( + "https://github.com/model-checking/kani/releases/download/" + f"kani-{VERSION}/{BUNDLE_NAME}" +) + + +def checked_download(url: str, destination: Path, digest: str) -> None: + """Fetch an HTTPS artefact and fail closed on any checksum mismatch. + + Parameters + ---------- + url : str + HTTPS release asset URL. + destination : Path + Local cache path; an existing file is reused only if its digest matches. + digest : str + Expected SHA-256 digest from the approved release manifest. + + Raises + ------ + ValueError + If the URL is not HTTPS or downloaded bytes do not match the pin. + OSError + If an existing cache entry cannot be inspected, or the download cannot + be transferred into the cache. + """ # ruff: ignore[docstring-extraneous-exception] - OSError propagates from _cache_is_current and _digest. + if not url.startswith("https://"): + msg = "binary downloads require HTTPS" + raise ValueError(msg) + if _cache_is_current(destination, digest): + return + pending = destination.with_suffix(".download") + try: + _download(url, pending) + _require_digest(pending, destination, digest) + pending.replace(destination) + finally: + pending.unlink(missing_ok=True) + + +def _cache_is_current(destination: Path, digest: str) -> bool: + """Report whether an existing cache entry already matches the pin. + + A cache entry that cannot be read is not a cache miss. The failure is + raised, not reported as ``False``, so ``checked_download`` never answers an + unreadable cache with a fresh download that would overwrite it. + + Parameters + ---------- + destination : Path + Local cache path to inspect. + digest : str + Expected SHA-256 digest from the approved release manifest. + + Returns + ------- + bool + ``True`` only for a readable entry whose digest already matches. + + Raises + ------ + OSError + If the cache entry exists but cannot be opened or read. + """ # ruff: ignore[docstring-extraneous-exception] - OSError propagates from _digest. + try: + current = _digest(destination) + except FileNotFoundError: + return False + return current == digest + + +def _require_digest(pending: Path, destination: Path, digest: str) -> None: + """Refuse downloaded bytes that do not match the approved digest.""" + if _digest(pending) != digest: + msg = f"binary checksum mismatch: {destination.name}" + raise ValueError(msg) + + +def _download(url: str, destination: Path) -> None: + """Follow bounded HTTPS-only release redirects into a temporary file.""" + for _ in range(6): + redirect = _download_once(url, destination) + if redirect is None: + return + url = redirect + msg = "release redirect limit exceeded" + raise ValueError(msg) + + +def _connect(url: str) -> tuple[http.client.HTTPSConnection, str]: + """Validate one hop's URL and open its HTTPS connection.""" + parsed = urllib.parse.urlsplit(url) + if parsed.scheme != "https" or not parsed.hostname: + msg = "binary downloads and redirects require HTTPS" + raise ValueError(msg) + target = urllib.parse.urlunsplit(("", "", parsed.path, parsed.query, "")) or "/" + connection = http.client.HTTPSConnection(parsed.hostname, parsed.port, timeout=60) + return connection, target + + +def _redirect_url(url: str, response: http.client.HTTPResponse) -> str | None: + """Resolve a redirect target or validate a terminal download response.""" + if response.status in {301, 302, 303, 307, 308}: + location = response.getheader("Location") + if location is None: + msg = "release redirect has no Location" + raise ValueError(msg) + return urllib.parse.urljoin(url, location) + if response.status != http.client.OK: + msg = f"release download failed: HTTP {response.status}" + raise OSError(msg) + return None + + +def _fetch_body( + connection: http.client.HTTPSConnection, + url: str, + destination: Path, +) -> str | None: + """Save the response body, or resolve one redirect hop instead.""" + with connection.getresponse() as response: + redirect = _redirect_url(url, response) + if redirect is not None: + return redirect + with destination.open("wb") as output: + shutil.copyfileobj(response, output) + return None + + +def _download_once(url: str, destination: Path) -> str | None: + """Close one HTTPS connection after saving its body or resolving a redirect.""" + connection, target = _connect(url) + try: + connection.request( + "GET", + target, + headers={"User-Agent": "cuprum-boundary-verification"}, + ) + return _fetch_body(connection, url, destination) + finally: + connection.close() + + +def _digest(path: Path) -> str: + """Hash a release artefact without loading the bundle into memory.""" + with path.open("rb") as source: + return hashlib.file_digest(source, "sha256").hexdigest() + + +def boundary_root() -> Path: + """Return the repository root whose pins and cache this installer uses. + + Honours ``CUPRUM_BOUNDARY_ROOT`` so an isolated tree — a test fixture, or a + staging wrapper — can be installed into without modifying the checkout the + script lives in. + + Returns + ------- + Path + The configured root, or the repository containing this script. + """ + override = os.environ.get(_ROOT_ENV) + return Path(override) if override else ROOT + + +def _require_current_pin(pin: str) -> None: + """Refuse a VERSION file that no longer matches the packaged digests.""" + if pin != VERSION: + msg = "update Kani archive names and digests with its version pin" + raise ValueError(msg) + + +def _extract_frontend(archive: Path, destination: Path) -> None: + """Extract exactly the approved frontend executables into the cache.""" + destination.mkdir(exist_ok=True) + with tarfile.open(archive, "r:gz") as bundle: + bundle.extractall( + destination, + members=[bundle.getmember(name) for name in FRONTEND_MEMBERS], + filter="data", + ) + + +def main() -> None: + """Install the pinned frontend into a private verification tool directory.""" + root = boundary_root() + pin = (root / "tools/kani/VERSION").read_text(encoding="utf-8").strip() + _require_current_pin(pin) + cache = root / ".cache/boundary-kani" + cache.mkdir(parents=True, exist_ok=True) + checked_download(FRONTEND_URL, cache / FRONTEND_NAME, FRONTEND_DIGEST) + checked_download(BUNDLE_URL, cache / BUNDLE_NAME, BUNDLE_DIGEST) + _extract_frontend(cache / FRONTEND_NAME, cache / "bin") + + +if __name__ == "__main__": + main() diff --git a/scripts/install_boundary_z3.py b/scripts/install_boundary_z3.py new file mode 100644 index 000000000..b6b7e02f1 --- /dev/null +++ b/scripts/install_boundary_z3.py @@ -0,0 +1,44 @@ +#!/usr/bin/env -S uv run python +# /// script +# requires-python = ">=3.13" +# dependencies = [] +# [tool.ty.environment] +# extra-paths = ["."] +# /// +"""Install the prebuilt Z3 release required by the pinned Verus binary. + +Verus's September 2026 archive does not include the solver. The version follows +its upstream ``source/tools/get-z3.sh``; the SHA-256 is the release asset digest. +""" + +from __future__ import annotations + +import zipfile + +# Imported as a package by the tests and as a sibling script by the Makefile. +# `__package__` is empty for direct-script execution, where only the bare name +# resolves; leaving a bare import unconditional would break `scripts.tests`. +if __package__: + from scripts.install_boundary_kani import boundary_root, checked_download +else: + from install_boundary_kani import boundary_root, checked_download + +NAME = "z3-4.16.0-x64-glibc-2.39" +DIGEST = "7288c49a5bd6dbafd7b0b0d1f65956b91672da24b08f09242919af159be3418e" +URL = f"https://github.com/Z3Prover/z3/releases/download/z3-4.16.0/{NAME}.zip" + + +def main() -> None: + """Install only the verified solver executable in the repository cache.""" + cache = boundary_root() / ".cache/boundary-z3" + cache.mkdir(parents=True, exist_ok=True) + archive_path = cache / f"{NAME}.zip" + checked_download(URL, archive_path, DIGEST) + binary = cache / "z3" + with zipfile.ZipFile(archive_path) as archive: + binary.write_bytes(archive.read(f"{NAME}/bin/z3")) + binary.chmod(0o755) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_boundary_proofs.py b/scripts/render_boundary_proofs.py new file mode 100644 index 000000000..e5f96aaf5 --- /dev/null +++ b/scripts/render_boundary_proofs.py @@ -0,0 +1,128 @@ +#!/usr/bin/env -S uv run python +# /// script +# requires-python = ">=3.13" +# dependencies = [] +# /// +"""Attach Verus specifications to the actual production length kernels. + +Only function signatures acquire specifications; executable bodies are copied +verbatim from production. Generated output belongs under ``rust/target`` and +is regenerated on every proof run, so a stale mirror cannot pass verification. +""" + +from __future__ import annotations + +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +COUNT_SIGNATURE = "-> Option {" +PROGRESS_SIGNATURE = "-> Option<(u64, u64)> {" +COUNT_CONTRACT = """-> (result: Option) + ensures match result { + Some(value) => value == count && count <= capacity, + None => count > capacity, + }, +{""" +PROGRESS_CONTRACT = """-> (result: Option<(u64, u64)>) + ensures match result { + Some((new_total, tail)) => + new_total as int == total as int + written as int + && tail as int + written as int == remaining as int, + None => written > remaining || total as int + written as int > u64::MAX, + }, +{""" + + +def _production_half(source: str) -> str: + """Return the non-test half of a module, failing closed on the boundary. + + Parameters + ---------- + source : str + Complete production module text. + + Returns + ------- + str + Text preceding the module's ``#[cfg(test)]`` boundary. + + Raises + ------ + ValueError + If the test-module boundary is absent or ambiguous. + """ + marker = "#[cfg(test)]" + if source.count(marker) != 1: + msg = "production module must have exactly one test-module boundary" + raise ValueError(msg) + return source.split(marker, 1)[0] + + +def render(source: str) -> str: + """Render production functions with specifications and unchanged bodies. + + Parameters + ---------- + source : str + Complete production ``progress.rs`` module. + + Returns + ------- + str + Standalone Verus input containing the production function bodies. + + Raises + ------ + ValueError + If a signature or the test-module boundary changed unexpectedly. + """ + production = _production_half(source) + for signature in (COUNT_SIGNATURE, PROGRESS_SIGNATURE): + if production.count(signature) != 1: + msg = f"production signature changed: {signature}" + raise ValueError(msg) + # Verus uses exec functions; const qualification affects compile-time use, + # not the executable bodies being verified. Inner module attributes cannot + # occur inside verus!, so the lint remains on the generated crate root. + production = production.replace("#![forbid(unsafe_code)]\n", "") + production = production.replace("pub const fn", "pub fn") + production = production.replace(COUNT_SIGNATURE, COUNT_CONTRACT) + production = production.replace(PROGRESS_SIGNATURE, PROGRESS_CONTRACT) + production = production.replace("//!", "//") + return ( + "#![forbid(unsafe_code)]\nuse vstd::prelude::*;\nverus! {\n" + + production + + "}\n" + ) + + +def main() -> None: + """Write regenerated Verus input beneath the workspace build directory.""" + source = ROOT / "rust/cuprum-native-io/src/progress.rs" + destination = ROOT / "rust/target/boundary-verification/progress.rs" + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(render(source.read_text(encoding="utf-8")), encoding="utf-8") + memory_source = ROOT / "rust/cuprum-native-io/src/memory.rs" + memory = _production_half(memory_source.read_text(encoding="utf-8")) + memory = memory.replace("#![forbid(unsafe_code)]", "") + memory = memory.replace("#[cfg(any(windows, test, kani))]", "").replace("//!", "//") + assessment = destination.with_name("memory-assessment.rs") + assessment.write_text( + "use vstd::prelude::*;\nverus! {\n" + memory + "}\n", encoding="utf-8" + ) + + resource = (ROOT / "rust/cuprum-native-io/src/lib.rs").read_text(encoding="utf-8") + start = resource.index("pub unsafe fn adopt_writer") + end = resource.index("/// Borrow a raw reader", start) + resource_assessment = ( + "use vstd::prelude::*;\n" + "use std::os::fd::{OwnedFd as OwnedStream, FromRawFd};\n" + "type PlatformFd = i32;\nverus! {\n" + resource[start:end] + "}\n" + ) + destination.with_name("resource-assessment.rs").write_text( + resource_assessment, encoding="utf-8" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/tests/boundary_harness_support.py b/scripts/tests/boundary_harness_support.py new file mode 100644 index 000000000..6c30e683e --- /dev/null +++ b/scripts/tests/boundary_harness_support.py @@ -0,0 +1,35 @@ +"""Build isolated repository fixtures for the boundary harness tests. + +The harnesses copy the real Rust workspace and run verifier commands through +the ``cuprum`` execution seam. Pointing them at a temporary root keeps their +orchestration exercised against production source text while confining every +write, and every recorded command, to the test. +""" + +from __future__ import annotations + +import shutil +import typing as typ + +from scripts import check_boundary_contract + +if typ.TYPE_CHECKING: + from pathlib import Path + +# Build output is the only part of the workspace a harness must never copy. +IGNORED = shutil.ignore_patterns("target") + + +def copy_boundary_repository(tmp_path: Path) -> Path: + """Copy the Rust workspace and the tool pins into a temporary root. + + Returns + ------- + Path + Temporary root laid out like the repository, minus any build output. + """ + shutil.copytree( + check_boundary_contract.ROOT / "rust", tmp_path / "rust", ignore=IGNORED + ) + shutil.copytree(check_boundary_contract.ROOT / "tools", tmp_path / "tools") + return tmp_path diff --git a/scripts/tests/test_boundary_binary_install.py b/scripts/tests/test_boundary_binary_install.py new file mode 100644 index 000000000..52c8873ec --- /dev/null +++ b/scripts/tests/test_boundary_binary_install.py @@ -0,0 +1,339 @@ +"""Binary tool installation must reject mismatches and never source-build. + +The installer entry points are exercised end to end against synthetic release +archives served through a stubbed download seam, so the tests cover what +``main()`` installs — and how it fails — without network access. +""" + +from __future__ import annotations + +import hashlib +import io +import tarfile +import typing as typ +import zipfile + +import pytest + +from scripts import install_boundary_kani as installer +from scripts import install_boundary_z3 as z3_installer + +if typ.TYPE_CHECKING: + from pathlib import Path + +# The root environment variable that isolates both installers' pin file and +# cache. Spelled literally so the tests exercise the documented contract +# rather than the constant that implements it. +ROOT_ENV = "CUPRUM_BOUNDARY_ROOT" + +KANI_FRONTEND_MEMBERS = { + "cargo-kani": b"cargo-kani frontend executable", + "kani": b"kani frontend executable", +} +KANI_BUNDLE_MEMBERS = {"kani": b"bundled kani driver"} + + +def _gzip_tar(members: dict[str, bytes]) -> bytes: + """Build a gzip-compressed tar containing exactly the given members.""" + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w:gz") as archive: + for name, content in members.items(): + info = tarfile.TarInfo(name) + info.size = len(content) + archive.addfile(info, io.BytesIO(content)) + return buffer.getvalue() + + +def _zip_archive(members: dict[str, bytes]) -> bytes: + """Build a zip containing exactly the given members.""" + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + for name, content in members.items(): + archive.writestr(name, content) + return buffer.getvalue() + + +def _seed_kani_pin(root: Path, version: str = installer.VERSION) -> None: + """Write the Kani version pin the isolated root is installed from.""" + pin = root / "tools/kani/VERSION" + pin.parent.mkdir(parents=True) + pin.write_text(f"{version}\n", encoding="utf-8") + + +def _serve_assets( + monkeypatch: pytest.MonkeyPatch, assets: dict[str, bytes] +) -> list[str]: + """Serve synthetic release bytes for `assets` and record every request. + + Each asset is keyed by the release URL it stands in for. The pinned digest + constants are re-pointed at the synthetic bytes, so ``main()`` verifies and + caches them exactly as it would a real release. The stub then checks each + requested digest against the content it is asked to supply, and fails the + test if the installer ever requests a URL the fixture does not cover. + + Returns + ------- + list[str] + The URLs the installer requested, in request order. + """ + digests: dict[str, str] = {} + for url, content in assets.items(): + digests[url] = hashlib.sha256(content).hexdigest() + for url, attribute in ( + (installer.FRONTEND_URL, "FRONTEND_DIGEST"), + (installer.BUNDLE_URL, "BUNDLE_DIGEST"), + ): + if url in digests: + monkeypatch.setattr(installer, attribute, digests[url]) + if z3_installer.URL in digests: + monkeypatch.setattr(z3_installer, "DIGEST", digests[z3_installer.URL]) + + requested: list[str] = [] + + def checked_download(url: str, destination: Path, digest: str) -> None: + """Supply the synthetic release body without network access.""" + requested.append(url) + content = assets.get(url) + assert content is not None, f"no synthetic asset for {url}" + assert hashlib.sha256(content).hexdigest() == digest, ( + "the installer must verify the patched expected digest" + ) + destination.write_bytes(content) + + monkeypatch.setattr(installer, "checked_download", checked_download) + monkeypatch.setattr(z3_installer, "checked_download", checked_download) + return requested + + +class TestBoundaryBinaryInstall: + """Installer entry points install only pinned, verified release content.""" + + def test_cached_binary_must_match_digest(self, tmp_path: Path) -> None: + """A verified cache is reusable without performing a download.""" + archive = tmp_path / "binary.tar.gz" + archive.write_bytes(b"verified archive") + digest = hashlib.sha256(b"verified archive").hexdigest() + installer.checked_download("https://example.invalid/asset", archive, digest) + assert archive.read_bytes() == b"verified archive", ( + "verified cache was modified" + ) + + def test_mismatch_preserves_existing_cache( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Corrupt downloads cannot replace a cached artefact or leave a partial.""" + archive = tmp_path / "binary.tar.gz" + archive.write_bytes(b"old cache") + + def fetch(_url: str, destination: Path) -> None: + """Supply a corrupt release body without network access.""" + destination.write_bytes(b"corrupt body") + + monkeypatch.setattr(installer, "_download", fetch) + with pytest.raises(ValueError, match="checksum mismatch"): + installer.checked_download( + "https://example.invalid/asset", archive, "0" * 64 + ) + assert archive.read_bytes() == b"old cache", ( + "corrupt download replaced the cache" + ) + assert not archive.with_suffix(".download").exists(), "partial download leaked" + + def test_an_unreadable_cache_is_refused_rather_than_replaced( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A cache entry that cannot be read must not be answered with a download.""" + archive = tmp_path / "binary.tar.gz" + archive.write_bytes(b"cached archive") + downloads: list[str] = [] + + def fetch(url: str, _destination: Path) -> None: + """Record any attempted download so the test can reject it.""" + downloads.append(url) + + def unreadable(_path: Path) -> str: + """Stand in for a cache entry the process cannot read.""" + raise PermissionError("unreadable") + + monkeypatch.setattr(installer, "_download", fetch) + monkeypatch.setattr(installer, "_digest", unreadable) + + with pytest.raises(PermissionError): + installer.checked_download( + "https://example.invalid/asset", archive, "0" * 64 + ) + + assert not downloads, "an unreadable cache must not start a download" + assert archive.read_bytes() == b"cached archive", ( + "an unreadable cache must be left in place" + ) + + def test_a_missing_cache_entry_is_a_miss_not_a_failure( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """An absent cache entry downloads normally instead of raising.""" + archive = tmp_path / "binary.tar.gz" + content = b"fresh archive" + digest = hashlib.sha256(content).hexdigest() + + def fetch(url: str, destination: Path) -> None: + """Supply a fresh release body without network access.""" + destination.write_bytes(content) + + monkeypatch.setattr(installer, "_download", fetch) + installer.checked_download("https://example.invalid/asset", archive, digest) + assert archive.read_bytes() == content, "the missing entry was not installed" + + def test_plain_http_is_rejected(self, tmp_path: Path) -> None: + """The installer cannot fetch a verifier over an unauthenticated scheme.""" + with pytest.raises(ValueError, match="HTTPS"): + installer.checked_download( + "http://example.invalid/asset", tmp_path / "binary", "0" * 64 + ) + + @pytest.mark.parametrize( + "redirect", + [ + ("http://example.invalid/asset", "require HTTPS"), + (None, "no Location"), + ("/another-redirect", "redirect limit"), + ], + ) + def test_release_redirects_fail_closed( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + redirect: tuple[str | None, str], + ) -> None: + """Insecure, malformed, and looping redirects cannot install a binary.""" + from unittest import mock + + location, message = redirect + + response = mock.MagicMock() + response.__enter__.return_value = response + response.status = 302 + response.getheader.return_value = location + connection = mock.Mock() + connection.getresponse.return_value = response + monkeypatch.setattr( + installer.http.client, + "HTTPSConnection", + lambda *_args, **_kwargs: connection, + ) + with pytest.raises(ValueError, match=message): + installer._download("https://example.invalid/asset", tmp_path / "binary") + connection.close.assert_called() + assert not (tmp_path / "binary").exists(), "rejected redirect created a binary" + + def test_main_installs_only_the_pinned_kani_frontend( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The installer extracts the two approved executables and nothing else.""" + _seed_kani_pin(tmp_path) + frontend = _gzip_tar(KANI_FRONTEND_MEMBERS) + bundle = _gzip_tar(KANI_BUNDLE_MEMBERS) + assets = {installer.FRONTEND_URL: frontend, installer.BUNDLE_URL: bundle} + requested = _serve_assets(monkeypatch, assets) + monkeypatch.setenv(ROOT_ENV, str(tmp_path)) + + installer.main() + + # Both pinned archives were fetched, in order, from their release URLs. + assert requested == [installer.FRONTEND_URL, installer.BUNDLE_URL], ( + "the installer must fetch the pinned frontend and bundle, and nothing else" + ) + cache = tmp_path / ".cache/boundary-kani" + assert (cache / installer.FRONTEND_NAME).read_bytes() == frontend, ( + "the verified frontend archive must be cached unchanged" + ) + assert (cache / installer.BUNDLE_NAME).read_bytes() == bundle, ( + "the verified bundle archive must be cached unchanged" + ) + bin_dir = cache / "bin" + assert sorted(path.name for path in bin_dir.iterdir()) == [ + "cargo-kani", + "kani", + ], "only the two pinned Kani executables may be extracted" + for name, content in KANI_FRONTEND_MEMBERS.items(): + assert (bin_dir / name).read_bytes() == content, ( + f"the extracted {name} must be the archived member, byte for byte" + ) + + def test_main_installs_the_z3_solver_executable( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The solver lands in the isolated cache with its exact bytes and mode.""" + content = b"z3 solver executable" + members = { + f"{z3_installer.NAME}/bin/z3": content, + f"{z3_installer.NAME}/LICENSE.txt": b"z3 license", + } + archive = _zip_archive(members) + requested = _serve_assets(monkeypatch, {z3_installer.URL: archive}) + monkeypatch.setenv(ROOT_ENV, str(tmp_path)) + + z3_installer.main() + + assert requested == [z3_installer.URL], ( + "the installer must fetch only the pinned solver release" + ) + cache = tmp_path / ".cache/boundary-z3" + assert (cache / f"{z3_installer.NAME}.zip").read_bytes() == archive, ( + "the verified solver archive must be cached unchanged" + ) + binary = cache / "z3" + assert binary.read_bytes() == content, ( + "the solver binary must be the archived member, byte for byte" + ) + assert binary.stat().st_mode & 0o777 == 0o755, "the solver must be executable" + + def test_main_rejects_a_stale_kani_version_pin( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A pin that disagrees with the packaged digests downloads nothing.""" + _seed_kani_pin(tmp_path, version="0.0.0-not-pinned") + requested = _serve_assets( + monkeypatch, + { + installer.FRONTEND_URL: _gzip_tar(KANI_FRONTEND_MEMBERS), + installer.BUNDLE_URL: _gzip_tar(KANI_BUNDLE_MEMBERS), + }, + ) + monkeypatch.setenv(ROOT_ENV, str(tmp_path)) + + with pytest.raises(ValueError, match="update Kani archive names and digests"): + installer.main() + + assert not requested, "a rejected version pin must not fetch any archive" + assert not (tmp_path / ".cache").exists(), ( + "a rejected version pin must not create the installer cache" + ) + + def test_main_rejects_a_truncated_kani_frontend( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A truncated archive fails loudly instead of installing part of a tool.""" + _seed_kani_pin(tmp_path) + frontend = _gzip_tar(KANI_FRONTEND_MEMBERS) + _serve_assets( + monkeypatch, + { + # Truncating the compressed stream mid-archive stands in for a + # release body that verified yet cannot be unpacked. + installer.FRONTEND_URL: frontend[: len(frontend) // 2], + installer.BUNDLE_URL: _gzip_tar(KANI_BUNDLE_MEMBERS), + }, + ) + monkeypatch.setenv(ROOT_ENV, str(tmp_path)) + + # Which layer reports the truncation is an implementation detail of the + # standard library: the tar reader raises `ReadError` when it notices, + # and the gzip reader raises `EOFError` mid-stream before it does. + with pytest.raises((tarfile.ReadError, EOFError)): + installer.main() + + bin_dir = tmp_path / ".cache/boundary-kani/bin" + assert not any(bin_dir.iterdir()), ( + "a truncated archive must not install a partial frontend" + ) diff --git a/scripts/tests/test_boundary_contract.py b/scripts/tests/test_boundary_contract.py new file mode 100644 index 000000000..b15b7df45 --- /dev/null +++ b/scripts/tests/test_boundary_contract.py @@ -0,0 +1,199 @@ +"""The unsafe-bearing crate allowlist must reject unreviewed workspace growth. + +The compiler path is driven through a recorded command seam, so the tests +assert what the harness compiles and requires of each result — including that +every probe is restored — without invoking Cargo. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts import check_boundary_contract as contract +from scripts.check_boundary_contract import check_members, check_safe_policy +from scripts.tests.boundary_harness_support import copy_boundary_repository + +# The one diagnostic string that distinguishes a rejected probe from a compile +# that failed for some unrelated reason. +UNSAFE_MARKERS = ("forbid(unsafe_code)", "-F unsafe-code") + +TARGETS = ("src/lib.rs", "tests/compile_tests.rs", "tests/future_target.rs") + + +class _RecordedCompile: + """Stand in for the harness compiler seam, recording every invocation. + + The result reflects the sources under test: a run whose probed target still + contains one of the probes fails with the unsafe-forbid diagnostic, and any + other run compiles cleanly. Deriving the result from the sources rather than + from call order keeps the seam honest — a harness that skipped a probe, or + that failed to remove one, changes the sequence the test observes. + """ + + def __init__(self) -> None: + self.probes: list[str] = [] + + def __call__(self, workspace: Path) -> tuple[int, str]: + """Record one compile and return the result its sources imply.""" + source = "\n".join( + path.read_text(encoding="utf-8") + for path in sorted((workspace / "cuprum-streams").rglob("*.rs")) + ) + self.probes.append(source) + if any(probe in source for probe in contract.PROBES): + return 1, f"error: use of unsafe code is forbidden by {UNSAFE_MARKERS[0]}" + return 0, "finished checking" + + +def _drive_contract( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> tuple[Path, _RecordedCompile]: + """Run the whole contract harness against a temporary repository copy.""" + root = copy_boundary_repository(tmp_path) + compiler = _RecordedCompile() + monkeypatch.setattr(contract, "ROOT", root) + monkeypatch.setattr(contract, "_compile", compiler) + contract.main() + return root, compiler + + +@pytest.mark.parametrize("extra", [", 'new-boundary'", ", 'cuprum-native-io'", ", '*'"]) +def test_unreviewed_members_are_rejected(extra: str) -> None: + """Extra, repeated, and wildcard members all require an explicit audit.""" + manifest = ( + "[workspace]\nmembers = ['cuprum-rust', 'cuprum-native-io', 'cuprum-streams'" + + extra + + "]" + ) + with pytest.raises(ValueError, match="audited boundary inventory"): + check_members(manifest) + + +def test_audited_members_are_accepted() -> None: + """The documented three-crate dependency split is accepted.""" + check_members( + "[workspace]\nmembers = ['cuprum-rust', 'cuprum-native-io', 'cuprum-streams']" + ) + + +@pytest.mark.parametrize("replacement", ["deny", "allow"]) +def test_safe_targets_cannot_weaken_unsafe_policy(replacement: str) -> None: + """A target-wide deny can be locally relaxed, so only forbid is accepted.""" + root = Path(__file__).resolve().parents[2] / "rust" + workspace = (root / "Cargo.toml").read_text(encoding="utf-8") + safe = (root / "cuprum-streams/Cargo.toml").read_text(encoding="utf-8") + check_safe_policy(workspace, safe) + weakened = safe.replace('unsafe_code = "forbid"', f'unsafe_code = "{replacement}"') + with pytest.raises(ValueError, match="forbid unsafe in all targets"): + check_safe_policy(workspace, weakened) + + +def test_main_probes_every_target_with_every_unsafe_form( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The baseline compiles, then each target is probed with all three forms.""" + root, compiler = _drive_contract(tmp_path, monkeypatch) + + for probe in contract.PROBES: + assert sum(probe in text for text in compiler.probes) == len(TARGETS), ( + f"the {probe!r} probe must be compiled against every target" + ) + logs = root / "rust/target/boundary-verification" + assert (logs / "safe-positive.log").read_text(encoding="utf-8") == ( + "finished checking" + ), "the baseline result must be archived" + expected = [ + f"{Path(target).stem}-unsafe-{index}.log" + for target in TARGETS + for index in range(3) + ] + assert sorted(path.name for path in logs.glob("*.log")) == sorted([ + "safe-positive.log", + *expected, + ]), "every probe result must be archived under its target and form" + + +def test_every_probed_target_is_restored_afterwards( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A probe left behind would poison every later compile.""" + root, _compiler = _drive_contract(tmp_path, monkeypatch) + workspace = root / ".cache/boundary-contract/workspace" + for target in TARGETS: + source = (workspace / "cuprum-streams" / target).read_text(encoding="utf-8") + for probe in contract.PROBES: + assert probe not in source, f"{target} was not restored after probing" + + +def test_a_probe_that_compiles_must_fail_the_check( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A safe target that accepts unsafe code must abort the harness.""" + root = copy_boundary_repository(tmp_path) + monkeypatch.setattr(contract, "ROOT", root) + # A compiler that never rejects anything stands in for a target whose lint + # stopped applying; the harness must not report success. + monkeypatch.setattr( + contract, "_compile", lambda _workspace: (0, "finished checking") + ) + with pytest.raises(RuntimeError, match="did not reject unsafe probe"): + contract.main() + + +def test_a_failing_baseline_aborts_before_any_probe( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A baseline that cannot compile leaves the targets unprobed.""" + root = copy_boundary_repository(tmp_path) + probed: list[Path] = [] + monkeypatch.setattr(contract, "ROOT", root) + monkeypatch.setattr( + contract, "_compile", lambda _workspace: (101, "error: no compiler") + ) + monkeypatch.setattr( + contract, + "_check_target", + lambda _workspace, path, _logs: probed.append(path), + ) + with pytest.raises(SystemExit): + contract.main() + assert not probed, "a broken baseline must not be reported as a probed workspace" + + +def test_unsafe_detection_requires_both_failure_and_the_forbid_diagnostic() -> None: + """An unrelated compile failure must not be read as a successful rejection.""" + assert contract._unsafe_was_forbidden(1, "error: forbid(unsafe_code)"), ( + "a failed compile naming forbid(unsafe_code) is a rejection" + ) + assert contract._unsafe_was_forbidden(1, "error: -F unsafe-code"), ( + "the -F unsafe-code spelling is the same rejection" + ) + assert not contract._unsafe_was_forbidden(0, "error: forbid(unsafe_code)"), ( + "a diagnostic without a failure exit is not a rejection" + ) + assert not contract._unsafe_was_forbidden(1, "error: unused import"), ( + "an unrelated failure is not a rejection" + ) + + +def test_a_probe_is_removed_even_when_compilation_fails(tmp_path: Path) -> None: + """The context manager restores the source on both exit paths.""" + target = tmp_path / "lib.rs" + target.write_text("pub fn production() {}\n", encoding="utf-8") + original = target.read_text(encoding="utf-8") + probe = "pub fn unsafe_probe() { unsafe {} }" + with contract._probe_appended(target, probe): + assert probe in target.read_text(encoding="utf-8"), "the probe was not applied" + assert target.read_text(encoding="utf-8") == original, ( + "the probe was not removed on the normal path" + ) + with ( + pytest.raises(ValueError, match="boom"), + contract._probe_appended(target, probe), + ): + raise ValueError("boom") + assert target.read_text(encoding="utf-8") == original, ( + "the probe was not removed when the body raised" + ) diff --git a/scripts/tests/test_boundary_faults.py b/scripts/tests/test_boundary_faults.py new file mode 100644 index 000000000..702e6549e --- /dev/null +++ b/scripts/tests/test_boundary_faults.py @@ -0,0 +1,310 @@ +"""Fault injection fails closed when the production correspondence changes. + +The executable paths run against a temporary workspace copy through a recorded +command seam, so the tests assert what the harness would run, in which order, +and what it requires of each result — without launching Verus, Kani, or Cargo. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts import check_boundary_faults as faults +from scripts.check_boundary_faults import mutate +from scripts.tests.boundary_harness_support import copy_boundary_repository + +ROOT = Path(__file__).resolve().parents[2] + +# Repository-relative, for reading the fixture root the harness is pointed at. +PROGRESS = "rust/cuprum-native-io/src/progress.rs" +MEMORY = "rust/cuprum-native-io/src/memory.rs" +# Workspace-relative, because the harness copies the crate tree to its root. +MEMORY_IN_WORKSPACE = "cuprum-native-io/src/memory.rs" + +# What the harness archives each proof fault under, and the fragments that must +# only ever appear in a rendered fault. +PROOF_FAULTS = { + "invalid-bound": "count >= capacity", + "incorrect-accounting": "total - written", +} + +# Fragments proving an ownership fault is currently planted in the workspace. +OWNERSHIP_FAULTS = ("core::mem::forget(writer)", "core::mem::forget(retained)") + +# Exit codes chosen to be distinct, so a control that returned a fault's code +# would not have run and a fault that returned a control's code was not caught. +CONTROL_EXIT = 0 +FAULT_EXIT = 7 + +# The diagnostic each verifier emits for a rejected check. Only the demanded +# one is reported, so a harness that accepted any failure would fail. +# +# A rendered proof is checked by Verus (its `program` is the absolute verus +# path). An ownership fault is a Cargo run, and which diagnostic it must +# produce depends on the subcommand: `cargo kani` reports a failed +# verification, while `cargo test` reports a failed test result. +VERUS_DIAGNOSTIC = "verification results::" +KANI_DIAGNOSTIC = "VERIFICATION:- FAILED" +TEST_DIAGNOSTIC = "test result: FAILED" +CARGO_DIAGNOSTICS = {"kani": KANI_DIAGNOSTIC, "test": TEST_DIAGNOSTIC} +PROOF_TARGET = "progress-fault.rs" + + +class _RecordedRun: + """Record every invocation while returning a result derived from the workspace. + + A control is identified by the workspace still holding production source; a + fault is identified by the planted mutation. Deriving the result from the + workspace rather than from call order keeps the seam honest: if the harness + ran a control while a fault was planted, or ran without planting anything, + the expected diagnostic would not match and the test would fail. + """ + + def __init__(self, workspace: Path) -> None: + self.workspace = workspace + self.calls: list[tuple[str, tuple[str, ...]]] = [] + + def __call__( + self, program: str, args: tuple[str, ...], workspace: Path + ) -> tuple[int, str]: + """Record one invocation and return the result its workspace implies. + + Only the diagnostic that this specific check demands is returned, so a + harness that accepted any failure — rather than the designated one — + would still be caught. + + Returns + ------- + tuple[int, str] + The exit code and output the harness will classify. + """ + self.calls.append((program, args)) + proof = Path(args[0]) + is_proof = proof.name == PROOF_TARGET + if is_proof: + source = proof.read_text(encoding="utf-8") + planted = any(fragment in source for fragment in PROOF_FAULTS.values()) + rejected = VERUS_DIAGNOSTIC + else: + memory = (workspace / MEMORY_IN_WORKSPACE).read_text(encoding="utf-8") + planted = any(fragment in memory for fragment in OWNERSHIP_FAULTS) + rejected = CARGO_DIAGNOSTICS[args[0]] + if not planted: + return CONTROL_EXIT, f"{VERUS_DIAGNOSTIC}\n" + return FAULT_EXIT, f"{rejected}\n" + + def names(self) -> list[str]: + """Return the invocation programs in call order.""" + return [program for program, _args in self.calls] + + def proof_targets(self) -> list[str]: + """Return the rendered proof file paths in call order.""" + return [args[0].rsplit("/", 1)[-1] for _program, args in self.calls] + + +def _drive( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> tuple[Path, _RecordedRun]: + """Run the whole harness against a temporary repository copy.""" + root = copy_boundary_repository(tmp_path) + runner = _RecordedRun(root / "rust") + monkeypatch.setattr(faults, "ROOT", root) + monkeypatch.setattr(faults, "_run", runner) + faults.main() + return root, runner + + +def test_the_harness_launches_the_pinned_installer_versions() -> None: + """A drifted pin must fail here rather than at verifier launch.""" + makefile = (ROOT / "Makefile").read_text(encoding="utf-8") + pins = faults.read_tool_pins() + assert f"KANI_VERSION = {pins.kani}" in makefile, ( + "the harness Kani pin drifted from the Makefile's installer" + ) + assert f"cuprum-verus-{pins.verus}" in makefile, ( + "the harness Verus root drifted from the Makefile's installer" + ) + + +def test_every_pin_is_read_from_its_manifest() -> None: + """Pins come from the tool manifests and are never blank.""" + pins = faults.read_tool_pins() + for name, version in (("kani", pins.kani), ("verus", pins.verus)): + expected = (ROOT / f"tools/{name}/VERSION").read_text(encoding="utf-8").strip() + assert version == expected != "", f"the {name} pin is not the manifest version" + + +def test_an_empty_pin_is_refused_rather_than_defaulted( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A blank pin must fail closed instead of naming a verifier that cannot exist.""" + root = copy_boundary_repository(tmp_path) + (root / "tools/kani/VERSION").write_text("\n", encoding="utf-8") + monkeypatch.setattr(faults, "ROOT", root) + with pytest.raises(ValueError, match="records no version"): + faults.read_tool_pins() + + +def test_a_missing_pin_is_a_hard_failure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """An absent pin propagates rather than silently defaulting a version.""" + root = copy_boundary_repository(tmp_path) + (root / "tools/verus/VERSION").unlink() + monkeypatch.setattr(faults, "ROOT", root) + with pytest.raises(FileNotFoundError): + faults.read_tool_pins() + + +@pytest.mark.parametrize("source", ["absent", "old old"]) +def test_fault_rejects_missing_or_ambiguous_kernel(source: str) -> None: + """A stale mutation must fail rather than claiming harness sensitivity.""" + with pytest.raises(ValueError, match="exactly one"): + mutate(source, "old", "new") + + +def test_fault_changes_only_the_named_fragment() -> None: + """Surrounding executable code remains the production implementation.""" + assert mutate("before old after", "old", "new") == "before new after", ( + "fault changed surrounding production code" + ) + + +def test_main_runs_both_fault_families_over_a_copied_workspace( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """main() copies the workspace and drives the proof and ownership faults.""" + root, runner = _drive(tmp_path, monkeypatch) + workspace = root / ".cache/boundary-faults/workspace" + assert workspace.is_dir(), "the harness must copy the workspace before mutating" + assert not (workspace / "target").exists(), ( + "the workspace copy must exclude Cargo build output" + ) + # One control plus the two progress faults, and a control plus the fault for + # each of the two ownership faults. + assert runner.proof_targets().count("progress-fault.rs") == 3, ( + "each progress fault must be preceded by a control, plus the baseline" + ) + assert runner.names().count("cargo") == 4, ( + "each ownership fault must be preceded by a control" + ) + + +def test_every_proof_fault_reaches_the_verus_input( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A fault is only detected if the mutation survives rendering into the proof.""" + root = copy_boundary_repository(tmp_path) + production = (root / PROGRESS).read_text(encoding="utf-8") + rendered: list[str] = [] + runner = _RecordedRun(root / "rust") + logs = root / "logs" + logs.mkdir() + monkeypatch.setattr(faults, "ROOT", root) + monkeypatch.setattr(faults, "_run", runner) + monkeypatch.setattr( + faults, "render", lambda source: rendered.append(source) or source + ) + faults._verify_progress_faults(root / "rust", logs) + + for name, fragment in PROOF_FAULTS.items(): + assert any(fragment in source for source in rendered), ( + f"the {name} fault never reached the Verus input" + ) + assert (root / PROGRESS).read_text(encoding="utf-8") == production, ( + "the proof fault was not restored after checking" + ) + + +def test_ownership_faults_are_detected_and_their_mutations_restored( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Both ownership faults are caught, and the workspace keeps production source.""" + root = copy_boundary_repository(tmp_path) + production = (root / MEMORY).read_text(encoding="utf-8") + runner = _RecordedRun(root / "rust") + logs = root / "logs" + logs.mkdir() + monkeypatch.setattr(faults, "ROOT", root) + monkeypatch.setattr(faults, "_run", runner) + faults._verify_ownership_faults(root / "rust", logs) + + # Each fault logs its control and the fault itself, in that order: the + # control proves the check passes on production, and the fault's own log + # proves the mutation was planted before it was required to fail. + assert [path.name for path in sorted(logs.iterdir())] == [ + "leaked-writer-control.log", + "leaked-writer.log", + "trailing-forget-unwind-control.log", + "trailing-forget-unwind.log", + ], "each ownership fault must run its control first" + assert (root / MEMORY).read_text(encoding="utf-8") == production, ( + "the ownership fault was not restored after checking" + ) + + +def test_unexpected_verifier_results_archive_the_log_and_fail_closed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """An unclassifiable result names the check and the archived log.""" + logs = tmp_path / "logs" + logs.mkdir() + monkeypatch.setattr( + faults, "_run", lambda _program, _args, _workspace: (3, "no verdict") + ) + with pytest.raises(faults.BoundaryFaultError) as caught: + faults._check( + faults._Check("compactness-control", "verus", ("proof.rs",)), + tmp_path, + logs, + ) + assert caught.value.check_name == "compactness-control", "the check must be named" + assert caught.value.exit_code == 3, "the verifier exit code must be retained" + assert caught.value.log_path == logs / "compactness-control.log", ( + "the archived log path must be reported" + ) + assert "no verdict" in caught.value.log_path.read_text(encoding="utf-8"), ( + "the verifier output must be archived for inspection" + ) + + +def test_a_control_that_fails_closed_is_not_accepted( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A control must pass, so a failing control cannot stand in for a fault.""" + logs = tmp_path / "logs" + logs.mkdir() + monkeypatch.setattr( + faults, "_run", lambda _program, _args, _workspace: (1, "unexpected failure") + ) + with pytest.raises(faults.BoundaryFaultError): + faults._check( + faults._Check("verus-control", "verus", ("proof.rs",)), tmp_path, logs + ) + + +def test_a_fault_is_only_accepted_for_its_own_diagnostic( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """An unrelated failure must not satisfy a check that names a diagnostic.""" + logs = tmp_path / "logs" + logs.mkdir() + check = faults._Check( + "invalid-bound", "verus", ("proof.rs",), "verification results::" + ) + monkeypatch.setattr( + faults, "_run", lambda _program, _args, _workspace: (1, "error: build failed") + ) + with pytest.raises(faults.BoundaryFaultError) as caught: + faults._check(check, tmp_path, logs) + assert caught.value.check_name == "invalid-bound", "the check must be named" + + monkeypatch.setattr( + faults, + "_run", + lambda _program, _args, _workspace: (1, "verification results::"), + ) + faults._check(check, tmp_path, logs) diff --git a/scripts/tests/test_boundary_proof_render.py b/scripts/tests/test_boundary_proof_render.py new file mode 100644 index 000000000..1186bc599 --- /dev/null +++ b/scripts/tests/test_boundary_proof_render.py @@ -0,0 +1,116 @@ +"""Guard the correspondence between Verus inputs and production kernels.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts import render_boundary_proofs as renderer +from scripts.tests.boundary_harness_support import copy_boundary_repository + +SOURCE = Path(__file__).resolve().parents[2] / "rust/cuprum-native-io/src/progress.rs" +MEMORY = Path(__file__).resolve().parents[2] / "rust/cuprum-native-io/src/memory.rs" + + +def test_production_fault_reaches_verus_input() -> None: + """A production accounting fault must survive rendering unchanged.""" + source = SOURCE.read_text(encoding="utf-8") + mutant = source.replace("total + written", "total - written") + assert mutant != source, "the deliberate fault no longer matches production" + generated = renderer.render(mutant) + assert "Some((total - written, remaining - written))" in generated, ( + "renderer erased the production fault" + ) + assert "new_total as int == total as int + written as int" in generated, ( + "fault changed the specification instead of the implementation" + ) + + +@pytest.mark.parametrize( + "signature", [renderer.COUNT_SIGNATURE, renderer.PROGRESS_SIGNATURE] +) +def test_changed_signature_fails_closed(signature: str) -> None: + """An unrecognized signature must not silently lose its specification.""" + source = SOURCE.read_text(encoding="utf-8").replace(signature, "-> bool {") + with pytest.raises(ValueError, match="production signature changed"): + renderer.render(source) + + +@pytest.mark.parametrize( + "source", + [ + pytest.param("pub const fn count() {}", id="absent"), + pytest.param( + "#[cfg(test)]\nmod first;\n#[cfg(test)]\nmod second;\n", + id="duplicated", + ), + ], +) +def test_ambiguous_test_boundary_fails_closed(source: str) -> None: + """An absent or duplicated marker must not silently truncate the module.""" + with pytest.raises(ValueError, match="exactly one test-module boundary"): + renderer._production_half(source) + + +def test_memory_assessment_carries_no_test_module() -> None: + """The memory proof renders production retention, not its tests.""" + production = renderer._production_half(MEMORY.read_text(encoding="utf-8")) + assert "with_retained_owner" in production, ( + "the memory proof lost the retention kernel it exists to verify" + ) + assert "#[cfg(test)]" not in production, "the test module leaked into the proof" + + +def test_main_writes_every_assessment_without_test_modules( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """main() emits all three Verus inputs from production source alone.""" + root = copy_boundary_repository(tmp_path) + monkeypatch.setattr(renderer, "ROOT", root) + renderer.main() + + destination = root / "rust/target/boundary-verification" + generated = { + path.name: path.read_text(encoding="utf-8") for path in destination.iterdir() + } + assert sorted(generated) == [ + "memory-assessment.rs", + "progress.rs", + "resource-assessment.rs", + ], "every assessment must be regenerated beneath the build directory" + for name, text in generated.items(): + assert "#[cfg(test)]" not in text, f"{name} carries a test module" + assert "use vstd::prelude::*;" in text, f"{name} is not a Verus input" + assert "total + written" in generated["progress.rs"], ( + "the progress proof must carry the production accounting" + ) + assert "with_retained_owner" in generated["memory-assessment.rs"], ( + "the memory proof must carry the production retention kernel" + ) + assert "adopt_writer" in generated["resource-assessment.rs"], ( + "the resource proof must carry the production adoption helper" + ) + assert "Borrow a raw reader" not in generated["resource-assessment.rs"], ( + "the resource proof must stop at the reader helper's documentation" + ) + + +def test_executable_bodies_are_preserved() -> None: + """Removing only the added contracts recovers the production module.""" + source = SOURCE.read_text(encoding="utf-8").split("#[cfg(test)]", 1)[0] + generated = renderer.render(SOURCE.read_text(encoding="utf-8")) + recovered = generated.removeprefix( + "#![forbid(unsafe_code)]\nuse vstd::prelude::*;\nverus! {\n" + ).removesuffix("}\n") + recovered = recovered.replace(renderer.COUNT_CONTRACT, renderer.COUNT_SIGNATURE) + recovered = recovered.replace( + renderer.PROGRESS_CONTRACT, renderer.PROGRESS_SIGNATURE + ) + expected = ( + source + .replace("#![forbid(unsafe_code)]\n", "") + .replace("pub const fn", "pub fn") + .replace("//!", "//") + ) + assert recovered == expected, "Verus executable bodies differ from production" diff --git a/tests/behaviour/test_native_pump_cleanup_behaviour.py b/tests/behaviour/test_native_pump_cleanup_behaviour.py index 733f9de20..89595cf23 100644 --- a/tests/behaviour/test_native_pump_cleanup_behaviour.py +++ b/tests/behaviour/test_native_pump_cleanup_behaviour.py @@ -12,7 +12,6 @@ import pytest from cuprum import ( - ECHO, ExecutionContext, ScopeConfig, _pipeline_native_pump_runtime, @@ -86,14 +85,18 @@ def _make_cleanup_scenario() -> _CleanupScenario: """Create a two-stage pipeline whose pump remains active until cancelled.""" _, python_program = python_catalogue() catalogue = combine_programs_into_catalogue( - ECHO, python_program, project_name="native-pump-cleanup-behaviour", ) - echo = sh.make(ECHO, catalogue=catalogue) python = sh.make(python_program, catalogue=catalogue) - pipeline = echo("-n", "payload") | python("-c", "import sys; sys.stdin.read()") - return _CleanupScenario(pipeline, frozenset((ECHO, python_program))) + # The scenario requires a live reader at hand-off, not an echo process + # whose EOF may already have queued the transport's close callback. + producer = ( + "import sys, threading; sys.stdout.write('payload'); " + "sys.stdout.flush(); threading.Event().wait()" + ) + pipeline = python("-c", producer) | python("-c", "import sys; sys.stdin.read()") + return _CleanupScenario(pipeline, frozenset((python_program,))) async def _cancel_public_pipeline( diff --git a/tests/behaviour/test_stream_parity_behaviour.py b/tests/behaviour/test_stream_parity_behaviour.py index fb19a4da9..af07adc8e 100644 --- a/tests/behaviour/test_stream_parity_behaviour.py +++ b/tests/behaviour/test_stream_parity_behaviour.py @@ -270,8 +270,12 @@ def _assert_stdout_matches( description : str Human-readable description of the payload for error messages. """ - assert pipeline_result.stdout == expected, ( - f"expected {description} to survive pipeline intact" + # Avoid quadratic full-string diff rendering hiding the useful diagnostic. + matches = pipeline_result.stdout == expected + assert matches, ( + f"expected {description} to survive pipeline intact: " + f"expected={len(expected)}, actual={len(pipeline_result.stdout or '')}, " + f"stage_exits={[stage.exit_code for stage in pipeline_result.stages]}" ) diff --git a/tests/helpers/ci_runners.py b/tests/helpers/ci_runners.py index e88ad850c..b494e3df3 100644 --- a/tests/helpers/ci_runners.py +++ b/tests/helpers/ci_runners.py @@ -106,6 +106,8 @@ "delayed-pr-comment.yml": ("delay_and_comment",), "get-codescene-sha.yml": ("refresh-sha",), "release.yml": ("publish",), + # Issue379 requires verifier schedules on GitHub-hosted Linux. + "rust-boundaries.yml": ("verus", "extended"), } #: Windows-native validation needs GitHub's hosted Windows image; Ubicloud #: offers Linux capacity only. @@ -138,6 +140,7 @@ #: is installed conditionally there and a job-wide contract cannot describe it. #: `test_the_typecheck_only_leg_installs_no_wrapper` covers that case instead. SCCACHE_JOBS: typ.Final = ( + ("rust-boundaries.yml", "verus"), ("ci.yml", "lint-test"), ("ci.yml", "extension-tests"), ("ci.yml", "coverage"), diff --git a/tests/test_ci_runner_placement.py b/tests/test_ci_runner_placement.py index b240b532f..d55db9f0b 100644 --- a/tests/test_ci_runner_placement.py +++ b/tests/test_ci_runner_placement.py @@ -133,6 +133,7 @@ def test_every_workflow_job_appears_in_one_placement_manifest() -> None: | set(WINDOWS_HOSTED_CASES) | { ("build-wheels.yml", "build-native-wheels"), + ("rust-boundaries.yml", "native"), # Callers of a reusable workflow declare no runner of their own. ("ci.yml", "build-wheels"), ("release.yml", "build-wheels"), diff --git a/tests/test_ci_rust_boundaries.py b/tests/test_ci_rust_boundaries.py new file mode 100644 index 000000000..8d38ea1f9 --- /dev/null +++ b/tests/test_ci_rust_boundaries.py @@ -0,0 +1,108 @@ +"""Keep formal boundary validation hosted, bounded, and free of target archives.""" + +from __future__ import annotations + +import shlex +from pathlib import Path, PurePosixPath + +import pytest + +from tests.helpers.ci_workflows import ( + cache_paths, + cache_steps, + job, + step_inputs, + steps, +) + +WORKFLOW = "rust-boundaries.yml" + + +def test_native_boundary_matrix_is_hosted_and_bounded() -> None: + """Native resource effects need each supported kernel within the CI budget.""" + native = job(WORKFLOW, "native") + assert native["runs-on"] == "${{ matrix.os }}", "native checks lost their OS matrix" + strategy = native.get("strategy") + assert isinstance(strategy, dict), "native strategy must be a mapping" + matrix = strategy.get("matrix") + assert isinstance(matrix, dict), "native matrix must be a mapping" + assert set(matrix["os"]) == {"ubuntu-latest", "windows-2022", "macos-latest"}, ( + "native boundary validation must retain all supported hosted platforms" + ) + assert strategy["max-parallel"] == 2, "native jobs exceeded the reviewed fan-out" + assert native.get("timeout-minutes") == 20, "native validation lost its timeout" + + +def test_expensive_boundary_checks_are_scheduled_or_manual() -> None: + """Kani/Miri and fault sensitivity must not add an unbounded PR workload.""" + extended = job(WORKFLOW, "extended") + assert extended.get("if") == "github.event_name != 'pull_request'", ( + "heavy proofs entered every PR" + ) + assert extended.get("timeout-minutes") == 45, ( + "extended verification lost its time limit" + ) + strategy = extended.get("strategy") + assert isinstance(strategy, dict), "extended strategy must be a mapping" + matrix = strategy.get("matrix") + assert isinstance(matrix, dict), "extended matrix must be a mapping" + assert strategy["max-parallel"] == 1, "heavy proof jobs must run serially" + assert set(matrix["check"]) == {"kani", "miri"}, "a required verifier disappeared" + + +@pytest.mark.parametrize("name", ["native", "verus", "extended"]) +def test_boundary_checkouts_do_not_persist_credentials(name: str) -> None: + """Keep the checkout token out of .git/config where built code could read it.""" + checkouts = [ + step + for step in steps(WORKFLOW, name) + if str(step.get("uses", "")).startswith("actions/checkout@") + ] + assert checkouts, f"{name} must check the repository out for its boundary checks" + for checkout in checkouts: + inputs = step_inputs( + checkout, f"{WORKFLOW}:{name} checkout must declare inputs" + ) + assert inputs.get("persist-credentials") is False, ( + f"{name} must set persist-credentials: false" + ) + + +@pytest.mark.parametrize("name", ["native", "verus", "extended"]) +def test_boundary_cache_ownership_is_limited(name: str) -> None: + """Only scheduled main runs publish tool/object caches, never target trees.""" + for step in cache_steps(WORKFLOW, name): + for path in cache_paths(step, f"{name} cache"): + assert "target" not in PurePosixPath(path).parts, ( + "Cargo target archive bypassed sccache" + ) + saves = [ + step + for step in steps(WORKFLOW, name) + if "actions/cache/save@" in str(step.get("uses", "")) + ] + assert saves, "the scheduled tool/compiler cache has no owner" + for save in saves: + condition = str(save.get("if", "")) + assert "github.event_name == 'schedule'" in condition, ( + "a PR can publish the trusted cache" + ) + assert "github.ref == 'refs/heads/main'" in condition, ( + "a branch can publish the trusted cache" + ) + + +def test_prover_tools_selects_its_required_python() -> None: + """The installer must override CI's Python 3.13 with its required Python 3.14.""" + makefile = (Path(__file__).resolve().parents[1] / "Makefile").read_text( + encoding="utf-8" + ) + assignment = next( + line for line in makefile.splitlines() if line.startswith("PROVER_TOOLS =") + ) + arguments = shlex.split(assignment.partition("=")[2]) + assert "--python" in arguments, ( + "ambient UV_PYTHON can select an incompatible runtime" + ) + runtime = arguments[arguments.index("--python") + 1] + assert runtime == "3.14", "the pinned prover-tools package requires Python 3.14" diff --git a/tests/test_native_sdist.py b/tests/test_native_sdist.py new file mode 100644 index 000000000..db8c6fc13 --- /dev/null +++ b/tests/test_native_sdist.py @@ -0,0 +1,73 @@ +"""Both source distributions must retain the optional native workspace.""" + +from __future__ import annotations + +import tarfile +from pathlib import Path + +import pytest + +from cuprum import Program, ProgramCatalogue, ProjectSettings, ScopeConfig, scoped, sh +from tests.helpers.docs import repo_root + + +def _build_source_archive(backend: str, destination: Path) -> Path: + """Build one real source archive using the repository's packaging frontend.""" + program = Program("uv") + catalogue = ProgramCatalogue( + projects=( + ProjectSettings( + name="sdist-test", + programs=(program,), + documentation_locations=("docs/rust-boundary-verification.md",), + noise_rules=(), + ), + ) + ) + command = sh.make(program, catalogue=catalogue) + arguments = ( + ("build", "--sdist", "--out-dir", str(destination)) + if backend == "uv" + else ( + "run", + "maturin", + "sdist", + "--manifest-path", + "rust/cuprum-rust/Cargo.toml", + "--out", + str(destination), + ) + ) + with scoped(ScopeConfig(allowlist=frozenset({program}))): + result = command(*arguments).run_sync( + context=sh.ExecutionContext(cwd=repo_root(), timeout=180) + ) + assert result.exit_code == 0, f"source build failed: {result.stderr}" + archives = list(destination.glob("*.tar.gz")) + assert len(archives) == 1, "source build must produce exactly one archive" + return archives[0] + + +@pytest.mark.timeout(240) +@pytest.mark.parametrize("backend", ["uv", "maturin"]) +def test_source_distribution_retains_native_workspace( + backend: str, + tmp_path: Path, +) -> None: + """Every crate source and lockfile survives packaging without build output.""" + with tarfile.open(_build_source_archive(backend, tmp_path)) as archive: + members = {Path(name).parts[1:] for name in archive.getnames()} + workspace = repo_root() / "rust" + expected: set[tuple[str, ...]] = { + ("rust", name) + for name in ("Cargo.toml", "Cargo.lock", "rust-toolchain.toml", "dylint.toml") + } + for crate in ("cuprum-rust", "cuprum-streams", "cuprum-native-io"): + expected.add(("rust", crate, "Cargo.toml")) + for source in (workspace / crate / "src").rglob("*"): + if source.is_file(): + expected.add(("rust", *source.relative_to(workspace).parts)) + assert expected <= members, f"source archive omitted: {sorted(expected - members)}" + assert not any("target" in member for member in members), ( + "source archive must not contain Rust build output" + ) diff --git a/tools/kani/VERSION b/tools/kani/VERSION new file mode 100644 index 000000000..328185caa --- /dev/null +++ b/tools/kani/VERSION @@ -0,0 +1 @@ +0.67.0 diff --git a/tools/verus/SHA256SUMS b/tools/verus/SHA256SUMS new file mode 100644 index 000000000..0475c9df0 --- /dev/null +++ b/tools/verus/SHA256SUMS @@ -0,0 +1 @@ +13d01e134c0620c3b29770874707d16c33b3d227c843a489c8ceb744d43c0a16 verus-0.2026.09.06.8dea4a2-x86-linux.zip diff --git a/tools/verus/VERSION b/tools/verus/VERSION new file mode 100644 index 000000000..5e696e347 --- /dev/null +++ b/tools/verus/VERSION @@ -0,0 +1 @@ +0.2026.09.06.8dea4a2