ci: qualify each wheel once and isolate native stress

.github/workflows/ci.yml

@@ -213,19 +213,18 @@ jobs:
           uv pip install --python .artifact-venv \
             --requirements "${{ runner.temp }}/test-requirements.txt"
       - name: Import the installed candidate
-        run: uv run --python .artifact-venv --no-project python -c "import ml4t.engineer"
+        run: >-
+          uv run --python .artifact-venv --no-project
+          python scripts/ci/candidate.py verify-installed candidate --source-root src
       - name: Run ty check
         run: >-
           uv run --python .artifact-venv --no-project
           ty check --python-version ${{ matrix.python-version }}
-      - name: Run the installed-wheel suite repeatedly
+      - name: Run the installed-wheel suite
         shell: bash
         run: |
-          for iteration in {1..10}; do
-            echo "Installed-wheel suite ${iteration}/10"
-            uv run --python .artifact-venv --no-project \
-              python -m pytest tests/ -q --tb=short --no-cov
-          done
+          uv run --python .artifact-venv --no-project \
+            python -m pytest tests/ -q --tb=short --no-cov
       - name: Run documented workflows from the installed candidate
         run: >-
           uv run --python .artifact-venv --no-project

.github/workflows/native-stress.yml

@@ -0,0 +1,117 @@
+name: Native stress
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - "src/ml4t/engineer/features/**"
+      - "src/ml4t/engineer/core/**"
+      - "tests/features/**"
+      - "tests/test_all_features_integration.py"
+      - "scripts/ci/**"
+      - ".github/workflows/ci.yml"
+      - ".github/workflows/native-stress.yml"
+      - "pyproject.toml"
+      - "uv.lock"
+  push:
+    branches: [main]
+    paths:
+      - "src/ml4t/engineer/features/**"
+      - "src/ml4t/engineer/core/**"
+      - "tests/features/**"
+      - "tests/test_all_features_integration.py"
+      - "scripts/ci/**"
+      - ".github/workflows/ci.yml"
+      - ".github/workflows/native-stress.yml"
+      - "pyproject.toml"
+      - "uv.lock"
+  schedule:
+    - cron: "17 5 * * 1"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: native-stress-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884
+        with:
+          version: "0.10.9"
+      - run: uv sync --python 3.12 --dev
+      - run: uv build --out-dir native-candidate/dist
+      - name: Bind the stress artifact to the tested revision
+        run: >-
+          uv run python scripts/ci/candidate.py create native-candidate
+          --commit-sha "${{ github.sha }}"
+          --git-tree "$(git rev-parse 'HEAD^{tree}')"
+      - id: matrix
+        run: uv run python scripts/ci/native_stress.py --matrix >> "$GITHUB_OUTPUT"
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9
+        with:
+          name: native-stress-candidate
+          path: native-candidate/
+          if-no-files-found: error
+
+  stress:
+    name: Native stress (${{ matrix.os }}, Python ${{ matrix.python-version }})
+    runs-on: ${{ matrix.os }}
+    needs: build
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.build.outputs.matrix) }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884
+        with:
+          version: "0.10.9"
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333
+        with:
+          name: native-stress-candidate
+          path: native-candidate
+      - name: Verify stress artifact identity
+        shell: bash
+        run: >-
+          python scripts/ci/candidate.py verify native-candidate
+          --expected-commit "${{ github.sha }}"
+          --expected-tree "$(git rev-parse 'HEAD^{tree}')"
+      - name: Install the built wheel with locked test dependencies
+        shell: bash
+        run: |
+          uv export --locked --group dev --extra ta --extra store --extra viz \
+            --no-emit-project --no-hashes --output-file "${{ runner.temp }}/stress-requirements.txt"
+          uv venv --python python .artifact-venv
+          wheel="$(find native-candidate/dist -maxdepth 1 -name '*.whl' -print -quit)"
+          UV_CACHE_DIR="${{ runner.temp }}/fresh-wheel-cache" \
+            uv pip install --python .artifact-venv "$wheel"
+          uv pip install --python .artifact-venv \
+            --requirements "${{ runner.temp }}/stress-requirements.txt"
+      - name: Verify actual installed origin and bytes
+        shell: bash
+        run: >-
+          uv run --python .artifact-venv --no-project
+          python scripts/ci/candidate.py verify-installed native-candidate --source-root src
+      - name: Run fresh-process native and cold-call regressions
+        shell: bash
+        run: >-
+          uv run --python .artifact-venv --no-project
+          python scripts/ci/native_stress.py --repetitions 10

scripts/ci/candidate.py

@@ -2,7 +2,12 @@
 
 import argparse
 import hashlib
+import importlib
+import importlib.metadata
 import json
+import os
+import subprocess
+import sys
 import tarfile
 import zipfile
 from email.message import Message
@@ -178,6 +183,53 @@ def verify(
         raise ValueError("candidate public metadata does not match the manifest")
 
 
+def verify_installed(candidate_dir: Path, source_root: Path | None = None) -> None:
+    """Bind the imported package and installed bytes to the verified wheel."""
+    verify(candidate_dir)
+    manifest = _load_manifest(candidate_dir)
+    if manifest["name"] != "ml4t-engineer":
+        raise ValueError("installed candidate must be ml4t-engineer")
+    distribution = importlib.metadata.distribution(manifest["name"])
+    if distribution.version != manifest["version"]:
+        raise ValueError("installed candidate version does not match the manifest")
+    module = importlib.import_module("ml4t.engineer")
+    if module.__file__ is None:
+        raise ValueError("candidate import origin has no package file")
+    origin = Path(module.__file__).resolve()
+    prefix = Path(sys.prefix).resolve()
+    installed = Path(distribution.locate_file("ml4t/engineer/__init__.py")).resolve()
+    if origin != installed or not origin.is_relative_to(prefix):
+        raise ValueError(f"candidate import origin is not the installed wheel: {origin}")
+    wheel, _ = _distribution_files(candidate_dir)
+    with zipfile.ZipFile(wheel) as archive:
+        members = [
+            item
+            for item in archive.infolist()
+            if item.filename.startswith("ml4t/engineer/") and not item.is_dir()
+        ]
+        if not any(item.filename == "ml4t/engineer/__init__.py" for item in members):
+            raise ValueError("candidate wheel does not contain ml4t.engineer")
+        for item in members:
+            path = Path(distribution.locate_file(item.filename)).resolve()
+            if not path.is_relative_to(prefix) or path.read_bytes() != archive.read(item):
+                raise ValueError(f"installed candidate bytes differ from wheel: {item.filename}")
+    print(
+        json.dumps({"version": distribution.version, "origin": str(origin), "files": len(members)})
+    )
+    if source_root is not None:
+        result = subprocess.run(
+            [sys.executable, str(Path(__file__).resolve()), "verify-installed", str(candidate_dir)],
+            env={**os.environ, "PYTHONPATH": str(source_root.resolve())},
+            capture_output=True,
+            text=True,
+            timeout=60,
+            check=False,
+        )
+        if result.returncode == 0 or "candidate import origin" not in result.stderr:
+            raise ValueError("source-tree contamination was not rejected by the installed check")
+        print("Source-tree import contamination rejected")
+
+
 def _parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser()
     subparsers = parser.add_subparsers(dest="command", required=True)
@@ -193,20 +245,26 @@ def _parser() -> argparse.ArgumentParser:
     verify_parser.add_argument("--expected-tree")
     verify_parser.add_argument("--expected-tag")
 
+    installed_parser = subparsers.add_parser("verify-installed")
+    installed_parser.add_argument("candidate_dir", type=Path)
+    installed_parser.add_argument("--source-root", type=Path)
+
     return parser
 
 
 def main() -> None:
     args = _parser().parse_args()
     if args.command == "create":
         create(args.candidate_dir, args.commit_sha, args.git_tree)
-    else:
+    elif args.command == "verify":
         verify(
             args.candidate_dir,
             expected_commit=args.expected_commit,
             expected_tree=args.expected_tree,
             expected_tag=args.expected_tag,
         )
+    else:
+        verify_installed(args.candidate_dir, args.source_root)
 
 
 if __name__ == "__main__":

scripts/ci/native_stress.py

@@ -0,0 +1,54 @@
+"""Fresh-process regressions for native bounds, call ordering and cold compilation."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+import yaml
+
+MACD = [
+    "tests/features/momentum/test_momentum_numba_coverage.py::TestMACDNumbaDirectCoverage",
+    "tests/features/momentum/test_macd.py",
+]
+FEATURE_ORDER = [
+    "tests/test_all_features_integration.py::test_all_features_compute_individually",
+    "tests/test_all_features_integration.py::test_all_features_compute_together",
+    "tests/test_all_features_integration.py::test_all_features_isolate_panel_assets",
+]
+
+
+def _run(nodes: list[str]) -> None:
+    subprocess.run(
+        [sys.executable, "-m", "pytest", *nodes, "-q", "--tb=short", "--no-cov"],
+        check=True,
+        timeout=600,
+    )
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--matrix", action="store_true")
+    parser.add_argument("--repetitions", type=int, default=10)
+    args = parser.parse_args()
+    if args.matrix:
+        workflow = yaml.load(Path(".github/workflows/ci.yml").read_text(), Loader=yaml.BaseLoader)
+        print("matrix=" + json.dumps(workflow["jobs"]["test"]["strategy"]["matrix"]))
+        return
+    if args.repetitions < 1:
+        parser.error("repetitions must be positive")
+    for iteration in range(args.repetitions):
+        print(f"Fresh MACD process {iteration + 1}/{args.repetitions}", flush=True)
+        _run(MACD if iteration % 2 == 0 else list(reversed(MACD)))
+    for nodes in (FEATURE_ORDER, list(reversed(FEATURE_ORDER))):
+        print("Fresh public feature-order process", flush=True)
+        _run(nodes)
+    print("Cold-call regressions, each with a fresh interpreter and empty Numba cache", flush=True)
+    _run(["tests/features/test_rolling_map_first_call.py"])
+
+
+if __name__ == "__main__":
+    main()

tests/features/test_rolling_map_first_call.py

@@ -16,11 +16,13 @@
 
 from __future__ import annotations
 
+import json
 import os
 import subprocess
 import sys
 import textwrap
 
+import numpy as np
 import pytest
 
 # feature -> the expression to build, as source, against columns `close` and `returns`
@@ -41,7 +43,7 @@
 
 PROBE = textwrap.dedent(
     """
-    import logging, warnings
+    import json, logging, warnings
     warnings.filterwarnings("ignore")
     logging.disable(logging.WARNING)
 
@@ -56,17 +58,16 @@
 
     def evaluate():
         values = frame.select(({expr}).alias("v")).to_series().to_numpy().astype(float)
-        finite = values[np.isfinite(values)]
-        return len(np.unique(finite)), finite.size
+        return values.tolist()
 
-    first, n = evaluate()
-    second, _ = evaluate()
-    print(f"{{first}} {{second}} {{n}}")
+    first = evaluate()
+    second = evaluate()
+    print(json.dumps([first, second]))
     """
 )
 
 
-def _run(expr: str, tmp_path) -> tuple[int, int, int]:
+def _run(expr: str, tmp_path) -> tuple[np.ndarray, np.ndarray]:
     """Evaluate the feature twice in a fresh interpreter with an empty Numba cache."""
     env = dict(os.environ, NUMBA_CACHE_DIR=str(tmp_path), PYTHONWARNINGS="ignore")
     completed = subprocess.run(
@@ -79,18 +80,24 @@ def _run(expr: str, tmp_path) -> tuple[int, int, int]:
     )
     if completed.returncode != 0:
         pytest.fail(f"probe failed:\n{completed.stderr[-2000:]}")
-    first, second, n = (int(part) for part in completed.stdout.strip().split()[-3:])
-    return first, second, n
+    first, second = json.loads(completed.stdout.strip().splitlines()[-1])
+    return np.asarray(first), np.asarray(second)
 
 
 @pytest.mark.parametrize("name", sorted(FEATURES))
 def test_first_evaluation_matches_the_second(name: str, tmp_path) -> None:
-    first, second, n = _run(FEATURES[name], tmp_path)
+    first, second = _run(FEATURES[name], tmp_path)
+    finite = first[np.isfinite(first)]
+    n = finite.size
 
     assert n > 100, f"{name} produced almost nothing to compare"
-    assert first == second, (
-        f"{name} returns {first} distinct values on its first evaluation in a process "
-        f"and {second} on its second, over {n} rows: the rolling window did not advance "
-        f"while the kernel compiled"
+    np.testing.assert_allclose(
+        first,
+        second,
+        rtol=1e-12,
+        atol=1e-12,
+        equal_nan=True,
+        err_msg=f"{name} changes numerical output between its cold and second evaluation",
     )
-    assert first > 2, f"{name} came out near-constant ({first} distinct values over {n} rows)"
+    distinct = len(np.unique(finite))
+    assert distinct > 2, f"{name} came out near-constant ({distinct} distinct values over {n} rows)"