From e64dde8c6b49ee200bd35783f93bd7ae76804313 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Sat, 17 Oct 2026 15:17:56 -0400 Subject: [PATCH 1/5] ci: run one full wheel suite and prove failure propagation (#99) --- .github/workflows/ci.yml | 9 +-- tests/test_ci_invocation.py | 107 +++++++++++++++++++++++++++++++++++ tests/test_release_policy.py | 10 +--- 3 files changed, 113 insertions(+), 13 deletions(-) create mode 100644 tests/test_ci_invocation.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c19be9..b62c926 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -218,14 +218,11 @@ jobs: run: >- uv run --python .artifact-venv --no-project ty check --python-version ${{ matrix.python-version }} - - name: Run the installed-wheel suite repeatedly + - name: Run the installed-wheel suite shell: bash run: | - for iteration in {1..10}; do - echo "Installed-wheel suite ${iteration}/10" - uv run --python .artifact-venv --no-project \ - python -m pytest tests/ -q --tb=short --no-cov - done + uv run --python .artifact-venv --no-project \ + python -m pytest tests/ -q --tb=short --no-cov - name: Run documented workflows from the installed candidate run: >- uv run --python .artifact-venv --no-project diff --git a/tests/test_ci_invocation.py b/tests/test_ci_invocation.py new file mode 100644 index 0000000..86ec4da --- /dev/null +++ b/tests/test_ci_invocation.py @@ -0,0 +1,107 @@ +"""Execute the configured installed-suite command, including abnormal exits.""" + +from __future__ import annotations + +import os +import shlex +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).parents[1] + + +def _run_suite(tmp_path: Path, test: str, *, hook: str = "", options: str = ""): + workflow = yaml.load((ROOT / ".github/workflows/ci.yml").read_text(), Loader=yaml.BaseLoader) + command = next( + step["run"] + for step in workflow["jobs"]["test"]["steps"] + if "python -m pytest tests/" in step.get("run", "") + ) + # Bind the runner's interpreter input, without replacing its shell/exit logic. + command = command.replace("--python .artifact-venv", f"--python {shlex.quote(sys.executable)}") + (tmp_path / "tests").mkdir() + (tmp_path / "tests/test_case.py").write_text(test) + (tmp_path / "pytest.ini").write_text(f"[pytest]\naddopts = {options}\n") + (tmp_path / "conftest.py").write_text( + "import os\nfrom pathlib import Path\n" + "def pytest_sessionstart(session):\n" + " with Path('sessions').open('a') as stream:\n" + " stream.write('started\\n')\n" + " if os.name == 'posix':\n" + " import resource\n" + " resource.setrlimit(resource.RLIMIT_CORE, (0, 0))\n" + hook + ) + return subprocess.run( + ["bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", command], + cwd=tmp_path, + env={ + **os.environ, + "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1", + "PYTEST_PLUGINS": "pytest_cov.plugin,pytest_timeout", + "PYTEST_ADDOPTS": "", + }, + capture_output=True, + text=True, + timeout=45, + check=False, + ) + + +def test_routine_wheel_cell_runs_the_full_suite_once(tmp_path: Path) -> None: + result = _run_suite(tmp_path, "def test_pass():\n assert 2 + 2 == 4\n") + assert result.returncode == 0, result.stdout + result.stderr + assert (tmp_path / "sessions").read_text().splitlines() == ["started"] + + +@pytest.mark.parametrize( + ("test", "hook", "options", "message"), + [ + ("def test_fails():\n assert False, 'real-test-failure'\n", "", "", "real-test-failure"), + ("def test_broken(:\n", "", "", "SyntaxError"), + ("VALUE = 1\n", "", "", "no tests ran"), + ("def test_pass(): pass\n", "", "--not-a-real-pytest-option", "unrecognized arguments"), + ( + "def test_pass(): pass\n", + "def pytest_collection(session):\n raise RuntimeError('real-internal-error')\n", + "", + "real-internal-error", + ), + ("def test_crashes():\n import os\n os.abort()\n", "", "", ""), + ( + "import time\nimport pytest\n@pytest.mark.timeout(0.2)\n" + "def test_times_out():\n time.sleep(30)\n", + "", + "", + "Timeout", + ), + ( + "def test_pass(): pass\n", + "def pytest_sessionfinish(session, exitstatus):\n" + " assert exitstatus == 0\n" + " print('1 passed', flush=True)\n" + " os.abort()\n", + "", + "1 passed", + ), + ], + ids=[ + "failure", + "collection", + "empty", + "usage", + "internal", + "crash", + "timeout", + "pass-then-crash", + ], +) +def test_configured_wheel_command_propagates_real_failure( + tmp_path: Path, test: str, hook: str, options: str, message: str +) -> None: + result = _run_suite(tmp_path, test, hook=hook, options=options) + assert result.returncode != 0, result.stdout + result.stderr + assert message in result.stdout + result.stderr diff --git a/tests/test_release_policy.py b/tests/test_release_policy.py index 07db42a..eef78f6 100644 --- a/tests/test_release_policy.py +++ b/tests/test_release_policy.py @@ -62,7 +62,7 @@ def test_each_matrix_cell_runs_all_release_checks_without_masking_failures() -> "Export installed-wheel test environment", "Install the candidate wheel in a clean environment", "Import the installed candidate", - "Run the installed-wheel suite repeatedly", + "Run the installed-wheel suite", "Run documented workflows from the installed candidate", } <= commands.keys() assert setup["with"] == {"python-version": "${{ matrix.python-version }}"} @@ -80,13 +80,9 @@ def test_each_matrix_cell_runs_all_release_checks_without_masking_failures() -> assert "--exclude" not in commands["Run ty check"] assert "--extra-search-path" not in commands["Run ty check"] - test_command = commands["Run the installed-wheel suite repeatedly"] - assert "for iteration in {1..10}" in test_command + test_command = commands["Run the installed-wheel suite"] assert "--python .artifact-venv --no-project" in test_command assert "python -m pytest tests/" in test_command - assert "set +e" not in test_command - assert "PYTEST_EXIT" not in test_command - assert "exit 0" not in test_command export_command = commands["Export installed-wheel test environment"] assert "--group dev --extra ta --extra store --extra viz" in export_command @@ -100,7 +96,7 @@ def test_each_matrix_cell_runs_all_release_checks_without_masking_failures() -> step_names = [step.get("name") for step in steps] assert step_names.index( "Install the candidate wheel in a clean environment" - ) < step_names.index("Run the installed-wheel suite repeatedly") + ) < step_names.index("Run the installed-wheel suite") def test_release_publishes_only_the_qualified_artifact() -> None: From 1fb082cf4888e7a995398b3a40163733c183d4ce Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Sat, 17 Oct 2026 15:26:13 -0400 Subject: [PATCH 2/5] ci: verify installed identity and separate native fresh-process stress (#99) --- .github/workflows/ci.yml | 4 +- .github/workflows/native-stress.yml | 114 ++++++++++++++++++ scripts/ci/candidate.py | 60 ++++++++- scripts/ci/native_stress.py | 54 +++++++++ tests/features/test_rolling_map_first_call.py | 37 +++--- tests/test_ci_invocation.py | 52 +++++++- 6 files changed, 298 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/native-stress.yml create mode 100644 scripts/ci/native_stress.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b62c926..ca3c3ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -213,7 +213,9 @@ jobs: uv pip install --python .artifact-venv \ --requirements "${{ runner.temp }}/test-requirements.txt" - name: Import the installed candidate - run: uv run --python .artifact-venv --no-project python -c "import ml4t.engineer" + run: >- + uv run --python .artifact-venv --no-project + python scripts/ci/candidate.py verify-installed candidate --source-root src - name: Run ty check run: >- uv run --python .artifact-venv --no-project diff --git a/.github/workflows/native-stress.yml b/.github/workflows/native-stress.yml new file mode 100644 index 0000000..8896765 --- /dev/null +++ b/.github/workflows/native-stress.yml @@ -0,0 +1,114 @@ +name: Native stress + +on: + pull_request: + branches: [main] + paths: + - "src/ml4t/engineer/features/**" + - "src/ml4t/engineer/core/**" + - "tests/features/**" + - "tests/test_all_features_integration.py" + - "scripts/ci/**" + - ".github/workflows/ci.yml" + - ".github/workflows/native-stress.yml" + - "pyproject.toml" + - "uv.lock" + push: + branches: [main] + paths: + - "src/ml4t/engineer/features/**" + - "src/ml4t/engineer/core/**" + - "tests/features/**" + - "tests/test_all_features_integration.py" + - "scripts/ci/**" + - ".github/workflows/ci.yml" + - ".github/workflows/native-stress.yml" + - "pyproject.toml" + - "uv.lock" + schedule: + - cron: "17 5 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: native-stress-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + fetch-depth: 0 + persist-credentials: false + - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 + with: + version: "0.10.9" + - run: uv sync --python 3.12 --dev + - run: uv build --out-dir native-candidate/dist + - name: Bind the stress artifact to the tested revision + run: >- + uv run python scripts/ci/candidate.py create native-candidate + --commit-sha "${{ github.sha }}" + --git-tree "$(git rev-parse 'HEAD^{tree}')" + - id: matrix + run: uv run python scripts/ci/native_stress.py --matrix >> "$GITHUB_OUTPUT" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + with: + name: native-stress-candidate + path: native-candidate/ + if-no-files-found: error + + stress: + name: Native stress (${{ matrix.os }}, Python ${{ matrix.python-version }}) + runs-on: ${{ matrix.os }} + needs: build + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.build.outputs.matrix) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + fetch-depth: 0 + persist-credentials: false + - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 + with: + version: "0.10.9" + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 + with: + name: native-stress-candidate + path: native-candidate + - name: Verify stress artifact identity + shell: bash + run: >- + python scripts/ci/candidate.py verify native-candidate + --expected-commit "${{ github.sha }}" + --expected-tree "$(git rev-parse 'HEAD^{tree}')" + - name: Install the built wheel with locked test dependencies + shell: bash + run: | + uv export --locked --group dev --extra ta --extra store --extra viz \ + --no-emit-project --no-hashes --output-file "${{ runner.temp }}/stress-requirements.txt" + uv venv --python python .artifact-venv + wheel="$(find native-candidate/dist -maxdepth 1 -name '*.whl' -print -quit)" + UV_CACHE_DIR="${{ runner.temp }}/fresh-wheel-cache" \ + uv pip install --python .artifact-venv "$wheel" + uv pip install --python .artifact-venv \ + --requirements "${{ runner.temp }}/stress-requirements.txt" + - name: Verify actual installed origin and bytes + run: >- + uv run --python .artifact-venv --no-project + python scripts/ci/candidate.py verify-installed native-candidate --source-root src + - name: Run fresh-process native and cold-call regressions + run: >- + uv run --python .artifact-venv --no-project + python scripts/ci/native_stress.py --repetitions 10 diff --git a/scripts/ci/candidate.py b/scripts/ci/candidate.py index 5ef380b..343b445 100644 --- a/scripts/ci/candidate.py +++ b/scripts/ci/candidate.py @@ -2,7 +2,12 @@ import argparse import hashlib +import importlib +import importlib.metadata import json +import os +import subprocess +import sys import tarfile import zipfile from email.message import Message @@ -178,6 +183,53 @@ def verify( raise ValueError("candidate public metadata does not match the manifest") +def verify_installed(candidate_dir: Path, source_root: Path | None = None) -> None: + """Bind the imported package and installed bytes to the verified wheel.""" + verify(candidate_dir) + manifest = _load_manifest(candidate_dir) + if manifest["name"] != "ml4t-engineer": + raise ValueError("installed candidate must be ml4t-engineer") + distribution = importlib.metadata.distribution(manifest["name"]) + if distribution.version != manifest["version"]: + raise ValueError("installed candidate version does not match the manifest") + module = importlib.import_module("ml4t.engineer") + if module.__file__ is None: + raise ValueError("candidate import origin has no package file") + origin = Path(module.__file__).resolve() + prefix = Path(sys.prefix).resolve() + installed = Path(distribution.locate_file("ml4t/engineer/__init__.py")).resolve() + if origin != installed or not origin.is_relative_to(prefix): + raise ValueError(f"candidate import origin is not the installed wheel: {origin}") + wheel, _ = _distribution_files(candidate_dir) + with zipfile.ZipFile(wheel) as archive: + members = [ + item + for item in archive.infolist() + if item.filename.startswith("ml4t/engineer/") and not item.is_dir() + ] + if not any(item.filename == "ml4t/engineer/__init__.py" for item in members): + raise ValueError("candidate wheel does not contain ml4t.engineer") + for item in members: + path = Path(distribution.locate_file(item.filename)).resolve() + if not path.is_relative_to(prefix) or path.read_bytes() != archive.read(item): + raise ValueError(f"installed candidate bytes differ from wheel: {item.filename}") + print( + json.dumps({"version": distribution.version, "origin": str(origin), "files": len(members)}) + ) + if source_root is not None: + result = subprocess.run( + [sys.executable, str(Path(__file__).resolve()), "verify-installed", str(candidate_dir)], + env={**os.environ, "PYTHONPATH": str(source_root.resolve())}, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + if result.returncode == 0 or "candidate import origin" not in result.stderr: + raise ValueError("source-tree contamination was not rejected by the installed check") + print("Source-tree import contamination rejected") + + def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser() subparsers = parser.add_subparsers(dest="command", required=True) @@ -193,6 +245,10 @@ def _parser() -> argparse.ArgumentParser: verify_parser.add_argument("--expected-tree") verify_parser.add_argument("--expected-tag") + installed_parser = subparsers.add_parser("verify-installed") + installed_parser.add_argument("candidate_dir", type=Path) + installed_parser.add_argument("--source-root", type=Path) + return parser @@ -200,13 +256,15 @@ def main() -> None: args = _parser().parse_args() if args.command == "create": create(args.candidate_dir, args.commit_sha, args.git_tree) - else: + elif args.command == "verify": verify( args.candidate_dir, expected_commit=args.expected_commit, expected_tree=args.expected_tree, expected_tag=args.expected_tag, ) + else: + verify_installed(args.candidate_dir, args.source_root) if __name__ == "__main__": diff --git a/scripts/ci/native_stress.py b/scripts/ci/native_stress.py new file mode 100644 index 0000000..2dfa380 --- /dev/null +++ b/scripts/ci/native_stress.py @@ -0,0 +1,54 @@ +"""Fresh-process regressions for native bounds, call ordering and cold compilation.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import Path + +import yaml + +MACD = [ + "tests/features/momentum/test_momentum_numba_coverage.py::TestMACDNumbaDirectCoverage", + "tests/features/momentum/test_macd.py", +] +FEATURE_ORDER = [ + "tests/test_all_features_integration.py::test_all_features_compute_individually", + "tests/test_all_features_integration.py::test_all_features_compute_together", + "tests/test_all_features_integration.py::test_all_features_isolate_panel_assets", +] + + +def _run(nodes: list[str]) -> None: + subprocess.run( + [sys.executable, "-m", "pytest", *nodes, "-q", "--tb=short", "--no-cov"], + check=True, + timeout=600, + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--matrix", action="store_true") + parser.add_argument("--repetitions", type=int, default=10) + args = parser.parse_args() + if args.matrix: + workflow = yaml.load(Path(".github/workflows/ci.yml").read_text(), Loader=yaml.BaseLoader) + print("matrix=" + json.dumps(workflow["jobs"]["test"]["strategy"]["matrix"])) + return + if args.repetitions < 1: + parser.error("repetitions must be positive") + for iteration in range(args.repetitions): + print(f"Fresh MACD process {iteration + 1}/{args.repetitions}", flush=True) + _run(MACD if iteration % 2 == 0 else list(reversed(MACD))) + for nodes in (FEATURE_ORDER, list(reversed(FEATURE_ORDER))): + print("Fresh public feature-order process", flush=True) + _run(nodes) + print("Cold-call regressions, each with a fresh interpreter and empty Numba cache", flush=True) + _run(["tests/features/test_rolling_map_first_call.py"]) + + +if __name__ == "__main__": + main() diff --git a/tests/features/test_rolling_map_first_call.py b/tests/features/test_rolling_map_first_call.py index 990b92e..5f6469e 100644 --- a/tests/features/test_rolling_map_first_call.py +++ b/tests/features/test_rolling_map_first_call.py @@ -16,11 +16,13 @@ from __future__ import annotations +import json import os import subprocess import sys import textwrap +import numpy as np import pytest # feature -> the expression to build, as source, against columns `close` and `returns` @@ -41,7 +43,7 @@ PROBE = textwrap.dedent( """ - import logging, warnings + import json, logging, warnings warnings.filterwarnings("ignore") logging.disable(logging.WARNING) @@ -56,17 +58,16 @@ def evaluate(): values = frame.select(({expr}).alias("v")).to_series().to_numpy().astype(float) - finite = values[np.isfinite(values)] - return len(np.unique(finite)), finite.size + return values.tolist() - first, n = evaluate() - second, _ = evaluate() - print(f"{{first}} {{second}} {{n}}") + first = evaluate() + second = evaluate() + print(json.dumps([first, second])) """ ) -def _run(expr: str, tmp_path) -> tuple[int, int, int]: +def _run(expr: str, tmp_path) -> tuple[np.ndarray, np.ndarray]: """Evaluate the feature twice in a fresh interpreter with an empty Numba cache.""" env = dict(os.environ, NUMBA_CACHE_DIR=str(tmp_path), PYTHONWARNINGS="ignore") completed = subprocess.run( @@ -79,18 +80,24 @@ def _run(expr: str, tmp_path) -> tuple[int, int, int]: ) if completed.returncode != 0: pytest.fail(f"probe failed:\n{completed.stderr[-2000:]}") - first, second, n = (int(part) for part in completed.stdout.strip().split()[-3:]) - return first, second, n + first, second = json.loads(completed.stdout.strip().splitlines()[-1]) + return np.asarray(first), np.asarray(second) @pytest.mark.parametrize("name", sorted(FEATURES)) def test_first_evaluation_matches_the_second(name: str, tmp_path) -> None: - first, second, n = _run(FEATURES[name], tmp_path) + first, second = _run(FEATURES[name], tmp_path) + finite = first[np.isfinite(first)] + n = finite.size assert n > 100, f"{name} produced almost nothing to compare" - assert first == second, ( - f"{name} returns {first} distinct values on its first evaluation in a process " - f"and {second} on its second, over {n} rows: the rolling window did not advance " - f"while the kernel compiled" + np.testing.assert_allclose( + first, + second, + rtol=1e-12, + atol=1e-12, + equal_nan=True, + err_msg=f"{name} changes numerical output between its cold and second evaluation", ) - assert first > 2, f"{name} came out near-constant ({first} distinct values over {n} rows)" + distinct = len(np.unique(finite)) + assert distinct > 2, f"{name} came out near-constant ({distinct} distinct values over {n} rows)" diff --git a/tests/test_ci_invocation.py b/tests/test_ci_invocation.py index 86ec4da..9a3cf74 100644 --- a/tests/test_ci_invocation.py +++ b/tests/test_ci_invocation.py @@ -14,7 +14,9 @@ ROOT = Path(__file__).parents[1] -def _run_suite(tmp_path: Path, test: str, *, hook: str = "", options: str = ""): +def _run_suite( + tmp_path: Path, test: str, *, hook: str = "", options: str = "", native: bool = False +): workflow = yaml.load((ROOT / ".github/workflows/ci.yml").read_text(), Loader=yaml.BaseLoader) command = next( step["run"] @@ -25,7 +27,7 @@ def _run_suite(tmp_path: Path, test: str, *, hook: str = "", options: str = ""): command = command.replace("--python .artifact-venv", f"--python {shlex.quote(sys.executable)}") (tmp_path / "tests").mkdir() (tmp_path / "tests/test_case.py").write_text(test) - (tmp_path / "pytest.ini").write_text(f"[pytest]\naddopts = {options}\n") + (tmp_path / "pytest.ini").write_text(f"[pytest]\naddopts = -s {options}\n") (tmp_path / "conftest.py").write_text( "import os\nfrom pathlib import Path\n" "def pytest_sessionstart(session):\n" @@ -33,8 +35,33 @@ def _run_suite(tmp_path: Path, test: str, *, hook: str = "", options: str = ""): " stream.write('started\\n')\n" " if os.name == 'posix':\n" " import resource\n" - " resource.setrlimit(resource.RLIMIT_CORE, (0, 0))\n" + hook + " resource.setrlimit(resource.RLIMIT_CORE, (0, 0))\n" + " else:\n" + " import ctypes\n" + " ctypes.windll.kernel32.SetErrorMode(3)\n" + " ctypes.CDLL('ucrtbase')._set_abort_behavior(0, 3)\n" + hook ) + if native: + from scripts.ci.native_stress import FEATURE_ORDER, MACD + + direct = tmp_path / MACD[0].split("::")[0] + direct.parent.mkdir(parents=True) + body = test.removeprefix("import time\nimport pytest\n") + if body.startswith("VALUE ="): + body = "pass\n" + direct.write_text( + "import time\nimport pytest\nclass TestMACDNumbaDirectCoverage:\n" + + "".join(" " + line + "\n" for line in body.replace("():", "(self):").splitlines()) + ) + (tmp_path / MACD[1]).write_text(test) + (tmp_path / FEATURE_ORDER[0].split("::")[0]).write_text( + "\n".join(f"def {node.split('::')[1]}(): pass" for node in FEATURE_ORDER) + ) + (tmp_path / "tests/features/test_rolling_map_first_call.py").write_text(test) + command = ( + f"{shlex.quote(sys.executable)} " + f"{shlex.quote(str(ROOT / 'scripts/ci/native_stress.py'))} --repetitions 2" + ) return subprocess.run( ["bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", command], cwd=tmp_path, @@ -57,6 +84,13 @@ def test_routine_wheel_cell_runs_the_full_suite_once(tmp_path: Path) -> None: assert (tmp_path / "sessions").read_text().splitlines() == ["started"] +def test_native_stress_runs_fresh_ordered_processes_and_cold_call_control(tmp_path: Path) -> None: + result = _run_suite(tmp_path, "def test_pass():\n assert 2 + 2 == 4\n", native=True) + assert result.returncode == 0, result.stdout + result.stderr + assert (tmp_path / "sessions").read_text().splitlines() == ["started"] * 5 + + +@pytest.mark.parametrize("native", [False, True], ids=["full-suite", "native-stress"]) @pytest.mark.parametrize( ("test", "hook", "options", "message"), [ @@ -70,7 +104,13 @@ def test_routine_wheel_cell_runs_the_full_suite_once(tmp_path: Path) -> None: "", "real-internal-error", ), - ("def test_crashes():\n import os\n os.abort()\n", "", "", ""), + ( + "def test_crashes():\n import os\n" + " print('native-abort', flush=True)\n os.abort()\n", + "", + "", + "native-abort", + ), ( "import time\nimport pytest\n@pytest.mark.timeout(0.2)\n" "def test_times_out():\n time.sleep(30)\n", @@ -100,8 +140,8 @@ def test_routine_wheel_cell_runs_the_full_suite_once(tmp_path: Path) -> None: ], ) def test_configured_wheel_command_propagates_real_failure( - tmp_path: Path, test: str, hook: str, options: str, message: str + tmp_path: Path, test: str, hook: str, options: str, message: str, native: bool ) -> None: - result = _run_suite(tmp_path, test, hook=hook, options=options) + result = _run_suite(tmp_path, test, hook=hook, options=options, native=native) assert result.returncode != 0, result.stdout + result.stderr assert message in result.stdout + result.stderr From 6267e95ca8383bb78a9d2a41013f946a24c5167c Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Sat, 17 Oct 2026 15:36:25 -0400 Subject: [PATCH 3/5] ci: bound native build and use explicit fail-closed shells (#99) --- .github/workflows/native-stress.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/native-stress.yml b/.github/workflows/native-stress.yml index 8896765..28369a8 100644 --- a/.github/workflows/native-stress.yml +++ b/.github/workflows/native-stress.yml @@ -39,6 +39,7 @@ concurrency: jobs: build: runs-on: ubuntu-latest + timeout-minutes: 20 outputs: matrix: ${{ steps.matrix.outputs.matrix }} steps: @@ -105,10 +106,12 @@ jobs: uv pip install --python .artifact-venv \ --requirements "${{ runner.temp }}/stress-requirements.txt" - name: Verify actual installed origin and bytes + shell: bash run: >- uv run --python .artifact-venv --no-project python scripts/ci/candidate.py verify-installed native-candidate --source-root src - name: Run fresh-process native and cold-call regressions + shell: bash run: >- uv run --python .artifact-venv --no-project python scripts/ci/native_stress.py --repetitions 10 From 3a26a9479d707440e1a5cf2e9cce177c29a7baba Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Sat, 17 Oct 2026 15:43:41 -0400 Subject: [PATCH 4/5] test(ci): reject real installed version and package drift (#99) --- tests/test_release_candidate.py | 121 ++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/tests/test_release_candidate.py b/tests/test_release_candidate.py index f883817..617eddf 100644 --- a/tests/test_release_candidate.py +++ b/tests/test_release_candidate.py @@ -1,10 +1,13 @@ from __future__ import annotations import json +import os import subprocess import sys import tarfile +import tempfile import zipfile +from collections.abc import Iterator from io import BytesIO from pathlib import Path from urllib.error import HTTPError @@ -42,6 +45,124 @@ def test_candidate_manifest_binds_both_artifacts_to_source(candidate_dir: Path) assert all(len(record["sha256"]) == 64 for record in manifest["artifacts"]) +@pytest.fixture(scope="module") +def installed_candidate(candidate_dir: Path) -> Iterator[Path]: + root = os.environ.get("RUNNER_TEMP") or os.environ.get("TMPDIR") + if root is None: + cache = Path(__file__).parents[1] / ".pytest_cache" + cache.mkdir(exist_ok=True) + root = str(cache) + with tempfile.TemporaryDirectory(prefix="engineer-installed-check-", dir=root) as directory: + environment = Path(directory) / "environment" + subprocess.run( + ["uv", "venv", "--python", sys.executable, str(environment)], + check=True, + capture_output=True, + ) + wheel, _ = candidate._distribution_files(candidate_dir) + subprocess.run( + [ + "uv", + "pip", + "install", + "--python", + str(environment), + "--link-mode", + "copy", + str(wheel), + ], + check=True, + capture_output=True, + ) + yield environment / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + + +def _installed_check(python: Path, directory: Path, **environment: str): + return subprocess.run( + [str(python), str(Path(candidate.__file__).resolve()), "verify-installed", str(directory)], + env={**os.environ, **environment}, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + + +def test_installed_candidate_rejects_real_source_tree_shadowing( + candidate_dir: Path, installed_candidate: Path +) -> None: + valid = _installed_check(installed_candidate, candidate_dir) + assert valid.returncode == 0, valid.stderr + source = Path(__file__).parents[1] / "src" + legacy = subprocess.run( + [str(installed_candidate), "-c", "import ml4t.engineer; print(ml4t.engineer.__file__)"], + env={**os.environ, "PYTHONPATH": str(source)}, + capture_output=True, + text=True, + check=False, + ) + assert legacy.returncode == 0, legacy.stderr + assert Path(legacy.stdout.strip()).is_relative_to(source) + rejected = _installed_check(installed_candidate, candidate_dir, PYTHONPATH=str(source)) + assert rejected.returncode != 0 + assert "candidate import origin" in rejected.stderr + + +def test_installed_candidate_rejects_modified_installed_package_bytes( + candidate_dir: Path, installed_candidate: Path +) -> None: + location = subprocess.run( + [str(installed_candidate), "-c", "import ml4t.engineer; print(ml4t.engineer.__file__)"], + check=True, + capture_output=True, + text=True, + ) + path = Path(location.stdout.strip()).parent / "features/trend/sma.py" + original = path.read_bytes() + try: + path.write_bytes(original + b"\n# Unexpected installed-file modification.\n") + legacy = subprocess.run( + [str(installed_candidate), "-c", "import ml4t.engineer"], + capture_output=True, + text=True, + check=False, + ) + assert legacy.returncode == 0, legacy.stderr + rejected = _installed_check(installed_candidate, candidate_dir) + assert rejected.returncode != 0 + assert "installed candidate bytes differ" in rejected.stderr + finally: + path.write_bytes(original) + + +def test_installed_candidate_rejects_another_real_wheel_version( + candidate_dir: Path, installed_candidate: Path, tmp_path: Path +) -> None: + subprocess.run( + ["uv", "build", "--wheel", "--out-dir", str(tmp_path)], + env={**os.environ, "SETUPTOOLS_SCM_PRETEND_VERSION": "0.0.0"}, + check=True, + capture_output=True, + ) + original, _ = candidate._distribution_files(candidate_dir) + other = next(tmp_path.glob("*.whl")) + install = ["uv", "pip", "install", "--python", str(installed_candidate), "--link-mode", "copy"] + try: + subprocess.run([*install, str(other)], check=True, capture_output=True) + legacy = subprocess.run( + [str(installed_candidate), "-c", "import ml4t.engineer"], + capture_output=True, + text=True, + check=False, + ) + assert legacy.returncode == 0, legacy.stderr + rejected = _installed_check(installed_candidate, candidate_dir) + assert rejected.returncode != 0 + assert "installed candidate version does not match" in rejected.stderr + finally: + subprocess.run([*install, str(original)], check=True, capture_output=True) + + @pytest.mark.parametrize("source_dependency", [False, True]) def test_release_selection_rejects_real_source_dependent_artifacts_before_publication( tmp_path: Path, source_dependency: bool From fde92ea963eef11cea490f08c8bd1a9ac2014ba9 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Sat, 17 Oct 2026 16:11:03 -0400 Subject: [PATCH 5/5] fix(ci): execute Git Bash rather than Windows WSL launcher (#99) --- tests/test_ci_invocation.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/tests/test_ci_invocation.py b/tests/test_ci_invocation.py index 9a3cf74..49916dc 100644 --- a/tests/test_ci_invocation.py +++ b/tests/test_ci_invocation.py @@ -4,6 +4,7 @@ import os import shlex +import shutil import subprocess import sys from pathlib import Path @@ -14,6 +15,21 @@ ROOT = Path(__file__).parents[1] +def _bash() -> str: + if os.name == "nt": + git = shutil.which("git") + if git is not None: + for parent in Path(git).resolve().parents: + for relative in ("bin/bash.exe", "usr/bin/bash.exe"): + candidate = parent / relative + if candidate.is_file(): + return str(candidate) + raise AssertionError("The configured Bash workflow requires Git Bash on Windows, not WSL") + bash = shutil.which("bash") + assert bash is not None, "The configured Bash workflow requires Bash" + return bash + + def _run_suite( tmp_path: Path, test: str, *, hook: str = "", options: str = "", native: bool = False ): @@ -24,7 +40,8 @@ def _run_suite( if "python -m pytest tests/" in step.get("run", "") ) # Bind the runner's interpreter input, without replacing its shell/exit logic. - command = command.replace("--python .artifact-venv", f"--python {shlex.quote(sys.executable)}") + python = shlex.quote(Path(sys.executable).as_posix()) + command = command.replace("--python .artifact-venv", f"--python {python}") (tmp_path / "tests").mkdir() (tmp_path / "tests/test_case.py").write_text(test) (tmp_path / "pytest.ini").write_text(f"[pytest]\naddopts = -s {options}\n") @@ -59,11 +76,11 @@ def _run_suite( ) (tmp_path / "tests/features/test_rolling_map_first_call.py").write_text(test) command = ( - f"{shlex.quote(sys.executable)} " - f"{shlex.quote(str(ROOT / 'scripts/ci/native_stress.py'))} --repetitions 2" + f"{python} " + f"{shlex.quote((ROOT / 'scripts/ci/native_stress.py').as_posix())} --repetitions 2" ) return subprocess.run( - ["bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", command], + [_bash(), "--noprofile", "--norc", "-eo", "pipefail", "-c", command], cwd=tmp_path, env={ **os.environ,