From e98ac10af9b6e619513503b223b8bf31f7571393 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 25 Aug 2026 14:38:42 +0000 Subject: [PATCH 1/4] release: pin guarded MLX and label 0.6.4b1 macOS 26-only Tighten the Fusion DMG packaging pin to mlx/mlx-metal 0.32.1.dev20260825+26421e953 (jonathan308/mlx@26421e953, nanobind 2.13.0, cp311). Block stock PyPI 0.32.0 and 0.32.2. Bump version to 0.6.4b1 and name the artifact macos26-arm64, not macos15-26. Do not create tag v0.6.4b1 until the matched cp311 wheels exist. Co-authored-by: Jonathan Spangler --- .github/workflows/build-wheels.yml | 5 +- .github/workflows/release-macos-dmg.yml | 28 ++- .gitignore | 8 +- apps/omlx-mac/Scripts/build.sh | 33 ++- .../Sources/Updater/ReleasesChecker.swift | 3 +- .../oMLXTests/ReleasesCheckerTests.swift | 30 +++ apps/omlx-mac/oMLX.xcodeproj/project.pbxproj | 12 +- docs/dev-prerelease-notes-template.md | 21 +- docs/packaging-guarded-mlx.md | 69 ++++++ docs/release-public-checklist.md | 27 ++- omlx/_version.py | 2 +- omlx/custom_kernels/__init__.py | 8 +- omlx/custom_kernels/decode_fast/fast.py | 10 + packaging/.gitignore | 2 + packaging/README.md | 15 +- packaging/_wheels/guarded/README.md | 43 ++++ packaging/build.py | 222 ++++++++++++++---- packaging/venvstacks.toml | 4 + pyproject.toml | 46 ++-- scripts/build_guarded_mlx_cp311_wheels.sh | 102 ++++++++ scripts/release_macos_dmg.sh | 2 +- setup.py | 2 +- tests/test_custom_kernel_abi_probe.py | 3 +- tests/test_packaging_guarded_mlx_pin.py | 59 +++++ 24 files changed, 640 insertions(+), 116 deletions(-) create mode 100644 docs/packaging-guarded-mlx.md create mode 100644 packaging/_wheels/guarded/README.md create mode 100755 scripts/build_guarded_mlx_cp311_wheels.sh create mode 100644 tests/test_packaging_guarded_mlx_pin.py diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 9c67a168fe..554072984b 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -46,6 +46,9 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install build setuptools wheel "cmake>=3.27" "nanobind==2.13.0" "mlx==0.32.0" + # This job is not the Fusion DMG path. The signed macOS 26 DMG must + # use the guarded cp311 pair (docs/packaging-guarded-mlx.md) and + # must never package 0.32.2. export CMAKE_ARGS="-DPython_EXECUTABLE=$(python -c 'import sys; print(sys.executable)')" python -m build --wheel --no-isolation @@ -61,7 +64,7 @@ jobs: /tmp/wheel-smoke/bin/python - <<'PY' import importlib failed = [] - for pkg in ("bonsai", "glm_moe_dsa", "minimax_m3", "qwen35_prefill"): + for pkg in ("bonsai", "decode_fast", "glm_moe_dsa", "minimax_m3", "qwen35_prefill"): fast = importlib.import_module(f"omlx.custom_kernels.{pkg}.fast") ok = bool(fast.is_native_available()) print(f"{pkg}: native_available={ok} import_error={fast.import_error()}") diff --git a/.github/workflows/release-macos-dmg.yml b/.github/workflows/release-macos-dmg.yml index 8ef115ec73..d7f3b8b4d8 100644 --- a/.github/workflows/release-macos-dmg.yml +++ b/.github/workflows/release-macos-dmg.yml @@ -5,7 +5,7 @@ on: workflow_dispatch: inputs: tag: - description: Existing version tag to build (for example, v0.6.3) + description: Existing version tag to build (for example, v0.6.4b1) required: true type: string @@ -64,9 +64,25 @@ jobs: "venvstacks==0.7.0" "cmake==4.4.2" "nanobind==2.13.0" \ "setuptools==80.10.2" "wheel==0.47.0" + - name: Require guarded cp311 MLX wheels + run: | + set -euo pipefail + python -c " + import importlib.util + spec = importlib.util.spec_from_file_location( + 'omlx_packaging_build', 'packaging/build.py' + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module.require_guarded_mlx_wheels() + print('guarded mlx wheels present') + " + - name: Build staged app with native kernels env: OMLX_RELEASE_REPOSITORY: ${{ github.repository }} + MACOSX_DEPLOYMENT_TARGET: "26.0" + OMLX_CUSTOM_KERNEL_DEPLOYMENT_TARGET: "26.0" run: apps/omlx-mac/Scripts/build.sh release --with-custom-kernel --rebuild-donor - name: Confirm arm64 application output @@ -77,6 +93,16 @@ jobs: test "$(lipo -archs "$app/Contents/MacOS/oMLX")" = arm64 actual_repo=$(/usr/libexec/PlistBuddy -c 'Print :OMLXReleaseRepository' "$app/Contents/Info.plist") test "$actual_repo" = "${{ github.repository }}" + min_os=$(/usr/libexec/PlistBuddy -c 'Print :LSMinimumSystemVersion' "$app/Contents/Info.plist") + [[ "$min_os" == 26* ]] || { + echo "expected macOS 26-only LSMinimumSystemVersion, got $min_os" >&2 + exit 1 + } + py="$app/Contents/Resources/Python/cpython-3.11/bin/python3" + site="$app/Contents/Resources/Python/framework-mlx-base/lib/python3.11/site-packages" + mlx_ver=$(PYTHONPATH="$site" "$py" -c 'import mlx; print(mlx.__version__)') + test "$mlx_ver" = "0.32.1.dev20260825+26421e953" + case "$mlx_ver" in *0.32.2*) echo "refusing mlx 0.32.2" >&2; exit 1 ;; esac - name: Archive app without losing symlinks or extended attributes run: | diff --git a/.gitignore b/.gitignore index e322bd5444..eef403e8f4 100644 --- a/.gitignore +++ b/.gitignore @@ -98,8 +98,12 @@ temp/ # MLX cache .mlx_cache/ -# Build artifacts -packaging/_wheels/ +# Build artifacts. Guarded mlx wheel *bytes* stay untracked; the README +# that documents the cp311 pin is tracked. +packaging/_wheels/* +!packaging/_wheels/guarded/ +packaging/_wheels/guarded/* +!packaging/_wheels/guarded/README.md packaging/_export/ packaging/_build/ *.dmg diff --git a/apps/omlx-mac/Scripts/build.sh b/apps/omlx-mac/Scripts/build.sh index a65b2025dc..215b086bef 100755 --- a/apps/omlx-mac/Scripts/build.sh +++ b/apps/omlx-mac/Scripts/build.sh @@ -52,8 +52,9 @@ # PYTHON_BIN=/path/to/python3 # python used for venvstacks driver # (default: PATH lookup of python3) # OMLX_WITH_CUSTOM_KERNEL=1 # same as --with-custom-kernel -# OMLX_CUSTOM_KERNEL_DEPLOYMENT_TARGET=15.0 +# OMLX_CUSTOM_KERNEL_DEPLOYMENT_TARGET=26.0 # # macOS min version for custom kernels +# # (Path B Fusion beta is macOS 26-only) # OMLX_RELEASE_REPOSITORY=owner/repo # GitHub Releases source embedded in # # Info.plist (default: jonathan308/omlx) # OMLX_BUILD_ARCHS=arm64 # application architectures; the @@ -117,6 +118,8 @@ CUSTOM_KERNEL_DIRS=( "$REPO_ROOT/omlx/custom_kernels/glm_moe_dsa" "$REPO_ROOT/omlx/custom_kernels/minimax_m3" "$REPO_ROOT/omlx/custom_kernels/qwen35_prefill" + "$REPO_ROOT/omlx/custom_kernels/decode_fast" + "$REPO_ROOT/omlx/custom_kernels/bonsai" ) # OMLX_EXPORT_DIR overrides the venvstacks export tree we copy Python # layers from. Release builds use this to point at a per-target export @@ -222,7 +225,7 @@ _rebuild_venvstacks_export() { } _custom_kernel_deployment_target() { - printf "%s\n" "${OMLX_CUSTOM_KERNEL_DEPLOYMENT_TARGET:-${MACOSX_DEPLOYMENT_TARGET:-15.0}}" + printf "%s\n" "${OMLX_CUSTOM_KERNEL_DEPLOYMENT_TARGET:-${MACOSX_DEPLOYMENT_TARGET:-26.0}}" } _custom_kernel_pythonpath() { @@ -251,7 +254,9 @@ _clean_custom_kernel_build_artifacts() { for ext_name in \ "omlx.custom_kernels.glm_moe_dsa._ext" \ "omlx.custom_kernels.minimax_m3._ext" \ - "omlx.custom_kernels.qwen35_prefill._ext"; do + "omlx.custom_kernels.qwen35_prefill._ext" \ + "omlx.custom_kernels.decode_fast._ext" \ + "omlx.custom_kernels.bonsai._ext"; do find "$REPO_ROOT/build" \ -type d \ -name "$ext_name" \ @@ -328,7 +333,7 @@ import sys import mlx.core as mx failures = [] -for name in ("glm_moe_dsa", "minimax_m3", "qwen35_prefill"): +for name in ("glm_moe_dsa", "minimax_m3", "qwen35_prefill", "decode_fast", "bonsai"): ext_dir = pathlib.Path("omlx/custom_kernels") / name so = next(ext_dir.glob("_ext.*.so"), None) if so is None: @@ -387,6 +392,10 @@ _build_custom_kernels() { || die "custom kernel build finished but MiniMax M3 metallib is missing." [ -f "$REPO_ROOT/omlx/custom_kernels/qwen35_prefill/omlx_qwen35_prefill_kernels.metallib" ] \ || die "custom kernel build finished but Qwen3.5 prefill metallib is missing." + [ -f "$REPO_ROOT/omlx/custom_kernels/decode_fast/omlx_decode_fast_kernels.metallib" ] \ + || die "custom kernel build finished but decode_fast metallib is missing." + [ -f "$REPO_ROOT/omlx/custom_kernels/bonsai/omlx_bonsai_kernels.metallib" ] \ + || die "custom kernel build finished but bonsai metallib is missing." # The NAX (M5 tensor unit) metallib is SDK-gated in cmake, not # deployment-gated: any build on SDK 26.2+ must produce it. A silent # omission would ship DMGs whose M5 qmm quietly falls back to the @@ -510,6 +519,7 @@ xcodebuild \ MARKETING_VERSION="$APP_VERSION" \ CURRENT_PROJECT_VERSION="$BUILD_NUMBER" \ OMLX_RELEASE_REPOSITORY="$OMLX_RELEASE_REPOSITORY" \ + MACOSX_DEPLOYMENT_TARGET="${MACOSX_DEPLOYMENT_TARGET:-26.0}" \ ARCHS="$OMLX_BUILD_ARCHS" \ ONLY_ACTIVE_ARCH=YES \ build >"$BUILD_DIR/xcodebuild.log" 2>&1 \ @@ -558,6 +568,21 @@ if [ -d "$DONOR_LAYERS/__venvstacks__" ]; then ok " + __venvstacks__ metadata" fi +log "Verifying bundled MLX is the guarded cp311 pair…" +BUNDLE_PYTHON="$PYTHON_DIR/cpython-3.11/bin/python3" +MLX_SITE="$PYTHON_DIR/framework-mlx-base/lib/python3.11/site-packages" +[ -x "$BUNDLE_PYTHON" ] || die "bundled CPython 3.11 interpreter missing: $BUNDLE_PYTHON" +[ -d "$MLX_SITE" ] || die "bundled MLX site-packages missing: $MLX_SITE" +GUARDED_MLX_VERSION="0.32.1.dev20260825+26421e953" +actual_mlx="$(PYTHONPATH="$MLX_SITE" "$BUNDLE_PYTHON" -c 'import mlx; print(mlx.__version__)')" \ + || die "bundled environment cannot import mlx; stock PyPI resolve is forbidden." +case "$actual_mlx" in + *0.32.2*) die "bundled mlx $actual_mlx is 0.32.2; TP2 rejected that ABI. Pin $GUARDED_MLX_VERSION." ;; +esac +[ "$actual_mlx" = "$GUARDED_MLX_VERSION" ] \ + || die "bundled mlx $actual_mlx does not match guarded pin $GUARDED_MLX_VERSION." +ok " + mlx $actual_mlx" + # --- Embed omlx package --------------------------------------------------- if [ "$WITH_CUSTOM_KERNEL" = "1" ]; then diff --git a/apps/omlx-mac/Sources/Updater/ReleasesChecker.swift b/apps/omlx-mac/Sources/Updater/ReleasesChecker.swift index 81632377ec..7512226681 100644 --- a/apps/omlx-mac/Sources/Updater/ReleasesChecker.swift +++ b/apps/omlx-mac/Sources/Updater/ReleasesChecker.swift @@ -4,7 +4,8 @@ // single source of truth for distribution. No appcast XML, no EdDSA key // management on the maintainer side. Pulls a page of releases, picks the // latest stable PEP 440 tag, and selects the DMG asset whose filename -// embeds the current macOS major version (e.g. `-macos15-` or `-macos26-`). +// embeds the current macOS major version (e.g. `-macos26-` for this +// Fusion Path B beta). Historical `-macos15-` assets remain parseable. // // Channel handling: Stable accepts final tags only, Release Candidate also // accepts rc tags, and Dev accepts dev/pre-release tags. GitHub's diff --git a/apps/omlx-mac/Tests/oMLXTests/ReleasesCheckerTests.swift b/apps/omlx-mac/Tests/oMLXTests/ReleasesCheckerTests.swift index 6c6a3d3314..c229be8a6c 100644 --- a/apps/omlx-mac/Tests/oMLXTests/ReleasesCheckerTests.swift +++ b/apps/omlx-mac/Tests/oMLXTests/ReleasesCheckerTests.swift @@ -117,6 +117,36 @@ final class ReleasesCheckerTests: XCTestCase { ) } + func testFindMatchingDMGSelectsMacOS26OnlyBetaAsset() { + let tahoeOnly = "oMLX-0.6.4b1-macos26-arm64.dmg" + let mislabeledRange = "oMLX-0.6.4b1-macos15-26-arm64.dmg" + let assets = [ + asset(tahoeOnly), + asset(mislabeledRange), + ] + + XCTAssertEqual( + ReleasesChecker.findMatchingDMG( + assets: assets, + macOSMajor: 26 + )?.name, + tahoeOnly + ) + XCTAssertEqual( + ReleasesChecker.findMatchingDMG( + assets: assets, + macOSMajor: 15 + )?.name, + mislabeledRange + ) + XCTAssertNil( + ReleasesChecker.findMatchingDMG( + assets: [asset(tahoeOnly)], + macOSMajor: 15 + ) + ) + } + func testFindMatchingDMGPrefersExactAssetOverRangeAsset() { let range = "oMLX-0.4.4-macos26-27.dmg" let exact = "oMLX-0.4.4-macos27-beta.dmg" diff --git a/apps/omlx-mac/oMLX.xcodeproj/project.pbxproj b/apps/omlx-mac/oMLX.xcodeproj/project.pbxproj index f7be6aa6c3..cb8f559dc1 100644 --- a/apps/omlx-mac/oMLX.xcodeproj/project.pbxproj +++ b/apps/omlx-mac/oMLX.xcodeproj/project.pbxproj @@ -883,7 +883,7 @@ GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; LOCALIZATION_PREFERS_STRING_CATALOGS = YES; - MACOSX_DEPLOYMENT_TARGET = 15.0; + MACOSX_DEPLOYMENT_TARGET = 26.0; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; ONLY_ACTIVE_ARCH = YES; @@ -940,7 +940,7 @@ GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; LOCALIZATION_PREFERS_STRING_CATALOGS = YES; - MACOSX_DEPLOYMENT_TARGET = 15.0; + MACOSX_DEPLOYMENT_TARGET = 26.0; MTL_ENABLE_DEBUG_INFO = NO; MTL_FAST_MATH = YES; SDKROOT = macosx; @@ -965,7 +965,7 @@ "$(inherited)", "@executable_path/../Frameworks", ); - MACOSX_DEPLOYMENT_TARGET = 15.0; + MACOSX_DEPLOYMENT_TARGET = 26.0; MARKETING_VERSION = 0.0.1; OMLX_RELEASE_REPOSITORY = jonathan308/omlx; PRODUCT_BUNDLE_IDENTIFIER = app.omlx; @@ -992,7 +992,7 @@ "$(inherited)", "@executable_path/../Frameworks", ); - MACOSX_DEPLOYMENT_TARGET = 15.0; + MACOSX_DEPLOYMENT_TARGET = 26.0; MARKETING_VERSION = 0.0.1; OMLX_RELEASE_REPOSITORY = jonathan308/omlx; PRODUCT_BUNDLE_IDENTIFIER = app.omlx; @@ -1009,7 +1009,7 @@ CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; GENERATE_INFOPLIST_FILE = YES; - MACOSX_DEPLOYMENT_TARGET = 15.0; + MACOSX_DEPLOYMENT_TARGET = 26.0; MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = app.omlx.tests; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -1026,7 +1026,7 @@ CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; GENERATE_INFOPLIST_FILE = YES; - MACOSX_DEPLOYMENT_TARGET = 15.0; + MACOSX_DEPLOYMENT_TARGET = 26.0; MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = app.omlx.tests; PRODUCT_NAME = "$(TARGET_NAME)"; diff --git a/docs/dev-prerelease-notes-template.md b/docs/dev-prerelease-notes-template.md index eb489fec9a..8789560260 100644 --- a/docs/dev-prerelease-notes-template.md +++ b/docs/dev-prerelease-notes-template.md @@ -2,15 +2,15 @@ # oMLX 0.6.4 Beta 1 release-candidate notes -> This is a readiness record, not a published release announcement. The -> recommended prerelease tag is `v0.6.4b1`, but the source version has not been -> changed to `0.6.4b1`, no tag has been created, and no signed artifact exists -> yet. +> This is a readiness record, not a published release announcement. Source +> version is `0.6.4b1`. Tag `v0.6.4b1` must **not** be created until the +> guarded cp311 mlx/mlx-metal wheels exist and have been reviewed. No signed +> artifact exists yet. Benchmark source commit: `6ebc22a822e3b50fe3c3d59acf1da62e8694f5dc` Expected artifact after the release gates pass: -`oMLX-0.6.4b1-macos15-26-arm64.dmg` +`oMLX-0.6.4b1-macos26-arm64.dmg` ## Readiness summary @@ -22,8 +22,9 @@ Expected artifact after the release gates pass: | In-flight prefill cancellation | Passed | Both Qwen ranks stopped at 26,624 processed tokens and returned to ready. | | Concurrent generation | Passed | Qwen completed four 512-token streams at 216.3 aggregate tok/s; DS4 completed its qualified B4 run at 69.3 aggregate tok/s. | | Functional greedy parity | Passed with limitation | With thinking disabled, four of six Qwen cases were token-exact and two of six were semantically equivalent. This is not a bit-exact or token-exact parity claim. | -| Version and tag | Pending | After review and green release CI, change `omlx/_version.py` from `0.6.4.dev1` to `0.6.4b1`, then create a matching protected tag `v0.6.4b1`. | -| Signed, notarized DMG | Blocked | Install the five missing Apple secrets, configure `APPLE_TEAM_ID`, and approve the protected `macos-release` environment. | +| Version and tag | Source bumped; tag blocked | `omlx/_version.py` is `0.6.4b1`. Do not create `v0.6.4b1` until the guarded cp311 wheels are reviewed. | +| Guarded MLX pin | Blocked on macOS wheel build | Pin `0.32.1.dev20260825+26421e953` from `26421e953`. Never 0.32.2. cp313 rollback wheels are not drop-in. | +| Signed, notarized DMG | Blocked | macOS 26-only artifact name `oMLX-0.6.4b1-macos26-arm64.dmg`. Add Apple secrets and `APPLE_TEAM_ID` to `macos-release`. | Do not advertise this candidate as signed, notarized, or downloadable until the gated workflow has produced and verified the DMG and checksum. Publishing the @@ -122,10 +123,10 @@ log, or paste the credential values into release notes. After downloading the DMG and its generated `.sha256` file: ```sh -shasum -a 256 -c oMLX-0.6.4b1-macos15-26-arm64.dmg.sha256 -xcrun stapler validate oMLX-0.6.4b1-macos15-26-arm64.dmg +shasum -a 256 -c oMLX-0.6.4b1-macos26-arm64.dmg.sha256 +xcrun stapler validate oMLX-0.6.4b1-macos26-arm64.dmg spctl --assess --type open --verbose=2 \ - --context context:primary-signature oMLX-0.6.4b1-macos15-26-arm64.dmg + --context context:primary-signature oMLX-0.6.4b1-macos26-arm64.dmg ``` The checksum proves byte identity. Gatekeeper and stapler checks confirm the diff --git a/docs/packaging-guarded-mlx.md b/docs/packaging-guarded-mlx.md new file mode 100644 index 0000000000..335ce3a593 --- /dev/null +++ b/docs/packaging-guarded-mlx.md @@ -0,0 +1,69 @@ +# Guarded MLX pin for the 0.6.4b1 Fusion DMG + +Path B of this first cluster beta ships a **macOS 26-only** signed DMG. The +bundled CPython 3.11 environment must not resolve stock PyPI MLX. + +## Pin + +- `mlx==0.32.1.dev20260825+26421e953` +- `mlx-metal==0.32.1.dev20260825+26421e953` +- Source: `jonathan308/mlx@26421e953` (`feat/jaccl-subgroups`) +- nanobind **2.13.0** (must match the wheel ABI or custom kernels reject + every `mlx.core.array`) +- Never `0.32.2`. TP2 rejected 0.32.2 (JACCL all-reduce loss, rank exit 75, + ~92-94 GiB wired on M5). + +`pyproject.toml` previously used `mlx>=0.32.0.dev0,<0.33`, which admits +0.32.2, and `[tool.uv] override-dependencies` forced stock `mlx==0.32.0`. +Both are now the exact local version above. A fresh resolve without the +local wheels must fail rather than land PyPI. + +## CPython 3.13 rollback wheels are not drop-in + +`minimax-m3-cluster/runtime_patches/variants/guard-0321-rollback/` holds +**cp313** wheels. The DMG embeds **CPython 3.11**. Those files cannot be +copied into `packaging/_wheels/guarded/`. Build a matched cp311 +frontend (`mlx`) + backend (`mlx-metal`) pair from `26421e953`. + +## Native kernels (same ABI) + +Rebuild with `OMLX_WITH_CUSTOM_KERNEL=1` against the guarded cp311 pair: + +| Extension | Notes | +| --- | --- | +| `glm_moe_dsa` / DS4 | Sparse MLA, DSA indexer, DSpark, DeepSeek V4 | +| `decode_fast` | Fused decode paths | +| `qwen35_prefill` | Prefill + NAX/ANE metallibs on SDK 26.2+ | +| `minimax_m3` | MiniMax M3 MSA | +| `bonsai` | 1-bit / 2-bit decode | + +`apps/omlx-mac/Scripts/build.sh release --with-custom-kernel` is the DMG +path. Default kernel deployment target is **26.0**. + +## Next local / macOS step + +This pin does **not** include wheel binaries. On a macOS 26 Apple Silicon +machine with Xcode 26.6 and CPython 3.11: + +```bash +scripts/build_guarded_mlx_cp311_wheels.sh +``` + +Confirm the produced version string is exactly +`0.32.1.dev20260825+26421e953` (mlx's setup stamps `.devYYYYMMDD` from the +build date plus `+` plus `git rev-parse --short HEAD`). Then rebuild the +venvstacks export and the app on that Mac. Do not create or push tag +`v0.6.4b1` until that pair exists and is reviewed. + +## Signing environment + +Create GitHub Environment `macos-release` (protected, required reviewers). +Do not commit Apple credentials. In the GitHub UI add secrets: + +- `APPLE_DEVELOPER_ID_APPLICATION_P12_BASE64` +- `APPLE_DEVELOPER_ID_APPLICATION_P12_PASSWORD` +- `APPLE_NOTARY_API_KEY_P8_BASE64` +- `APPLE_NOTARY_KEY_ID` +- `APPLE_NOTARY_ISSUER_ID` + +and environment variable `APPLE_TEAM_ID`. diff --git a/docs/release-public-checklist.md b/docs/release-public-checklist.md index 7a15f08f1b..fa045b7b39 100644 --- a/docs/release-public-checklist.md +++ b/docs/release-public-checklist.md @@ -28,12 +28,11 @@ and [common issue checks](https://developer.apple.com/documentation/security/res ## One-time GitHub setup -Create a GitHub Actions environment named `macos-release`. Restrict it to the -protected default branch and require at least one reviewer who can compare the -requested tag/commit before secrets are released. Environment protection rules +Create a GitHub Actions environment named `macos-release`. Restrict it with +required reviewers and a deployment-branch policy. Environment protection rules delay access to environment secrets until approval; see [GitHub's environment documentation](https://docs.github.com/en/actions/reference/workflows-and-actions/deployments-and-environments). -Add these environment secrets: +Jonathan must add these environment secrets in the GitHub UI (never commit them): | Name | Value | | --- | --- | @@ -64,20 +63,26 @@ before future Xcode pin changes instead of assuming `macos-latest` is stable. ## Cutting a release -1. Update `omlx/_version.py`, merge the reviewed release commit, and create and - push a matching tag such as `v0.6.3`. Prefer a protected, signed tag. -2. Run `Signed macOS DMG` manually and enter that existing tag. Approve the +1. Update `omlx/_version.py` (this beta is `0.6.4b1`), merge the reviewed + release commit, and only then create and push a matching tag such as + `v0.6.4b1`. Prefer a protected, signed tag. **Do not create `v0.6.4b1` + until the guarded cp311 wheels exist and have been reviewed.** +2. On macOS 26, build the matched mlx/mlx-metal pair with + `scripts/build_guarded_mlx_cp311_wheels.sh` and place them in + `packaging/_wheels/guarded/`. See [packaging-guarded-mlx.md](packaging-guarded-mlx.md). +3. Run `Signed macOS DMG` manually and enter that existing tag. Approve the `macos-release` environment only after confirming the tag resolves to the intended reviewed commit. -3. Inspect the workflow's signed DMG artifact and its SHA-256 file. Confirm the +4. Inspect the workflow's signed DMG artifact and its SHA-256 file. Confirm the draft release notes and assets. On a separate Mac, download through a quarantine-setting browser, verify the checksum, mount the image, and launch the app. Test a full in-app update from the preceding release as well. -4. Publish the draft manually only after those checks pass. Publishing remains +5. Publish the draft manually only after those checks pass. Publishing remains an explicit maintainer action. -The DMG is named `oMLX-VERSION-macos15-26-arm64.dmg`, matching the updater's -macOS-range selection. `OMLX_RELEASE_REPOSITORY` is embedded at build time and +The DMG is named `oMLX-VERSION-macos26-arm64.dmg`. This Fusion cluster beta is +**macOS 26-only** (Path B). Do not name or advertise it as macOS 15-26; the +updater matches `-macos26-` as an exact major. `OMLX_RELEASE_REPOSITORY` is embedded at build time and defaults to `jonathan308/omlx`; the workflow sets it to the repository running the workflow. The updater validates the DMG with Gatekeeper, mounts it with image verification enabled, and requires the app to be notarized Developer ID diff --git a/omlx/_version.py b/omlx/_version.py index 99836f8d61..5459fbd25b 100644 --- a/omlx/_version.py +++ b/omlx/_version.py @@ -1 +1 @@ -__version__ = "0.6.4.dev1" +__version__ = "0.6.4b1" diff --git a/omlx/custom_kernels/__init__.py b/omlx/custom_kernels/__init__.py index 33ae183b3c..dfcfb6b402 100644 --- a/omlx/custom_kernels/__init__.py +++ b/omlx/custom_kernels/__init__.py @@ -5,7 +5,13 @@ import importlib -NATIVE_KERNEL_PACKAGES = ("bonsai", "glm_moe_dsa", "minimax_m3", "qwen35_prefill") +NATIVE_KERNEL_PACKAGES = ( + "bonsai", + "decode_fast", + "glm_moe_dsa", + "minimax_m3", + "qwen35_prefill", +) def native_kernel_status() -> dict[str, dict[str, object]]: diff --git a/omlx/custom_kernels/decode_fast/fast.py b/omlx/custom_kernels/decode_fast/fast.py index 75f9535f9d..9c3d5c77a0 100644 --- a/omlx/custom_kernels/decode_fast/fast.py +++ b/omlx/custom_kernels/decode_fast/fast.py @@ -65,6 +65,14 @@ def _verify_abi(ext, import_error): NATIVE_AVAILABLE = _ext is not None +def is_native_available() -> bool: + return NATIVE_AVAILABLE + + +def import_error() -> Exception | None: + return _IMPORT_ERROR + + def _composed_rms_norm_residual( x: mx.array, weight: mx.array, residual: mx.array, eps: float ) -> Tuple[mx.array, mx.array]: @@ -213,6 +221,8 @@ def sparse_attn_decode( __all__ = [ "NATIVE_AVAILABLE", + "is_native_available", + "import_error", "rms_norm_residual", "sdpa_decode", "rope_kv_append", diff --git a/packaging/.gitignore b/packaging/.gitignore index 3263378cfb..6a82b55726 100644 --- a/packaging/.gitignore +++ b/packaging/.gitignore @@ -3,6 +3,8 @@ _build/ _export/ dist/ requirements/ +_wheels/*.whl +_wheels/guarded/*.whl # Python __pycache__/ diff --git a/packaging/README.md b/packaging/README.md index bbc8fa0a4c..59522456a3 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -12,9 +12,14 @@ this directory only hands it a `_export/` tree of Python layers. ## Requirements -- macOS 15.0+ (Sequoia) — required by MLX ≥ 0.29.2 -- Apple Silicon (M1/M2/M3/M4) -- Python 3.11+ on the host +- macOS 26 (Tahoe) for this Fusion cluster beta (Path B). Do not label the + bundled runtime as macOS 15-26. +- Apple Silicon +- Python 3.11 on the host (the embedded runtime is cpython-3.11) +- Guarded local `mlx` / `mlx-metal` **cp311** wheels at + `packaging/_wheels/guarded/` — see + [docs/packaging-guarded-mlx.md](../docs/packaging-guarded-mlx.md). + CPython 3.13 wheels from `guard-0321-rollback/` are not drop-in. - venvstacks (installed via `pip install -e ".[dev]"` from the repo root, or any of `uv`, `pipx run`) @@ -34,7 +39,7 @@ Then the Swift bundle: ```bash apps/omlx-mac/Scripts/build.sh release # full bundle apps/omlx-mac/Scripts/build.sh release --no-rebuild-donor # reuse _export/ -apps/omlx-mac/Scripts/build.sh release --with-custom-kernel # bundle GLM-5.2 / MiniMax M3 native kernels +apps/omlx-mac/Scripts/build.sh release --with-custom-kernel # glm_moe_dsa/DS4, decode_fast, qwen35_prefill (NAX/ANE), minimax_m3, bonsai ``` ## Output @@ -43,7 +48,7 @@ apps/omlx-mac/Scripts/build.sh release --with-custom-kernel # bundle GLM-5.2 / packaging/ ├── _build/ # venvstacks intermediate layers ├── _export/ # venvstacks export — embedded into the .app -└── _wheels/ # cached local wheels (e.g. mlx + mlx-metal pins) +└── _wheels/guarded/ # operator-supplied mlx + mlx-metal cp311 pair (not git) ``` ## Layer Configuration diff --git a/packaging/_wheels/guarded/README.md b/packaging/_wheels/guarded/README.md new file mode 100644 index 0000000000..4db03c341a --- /dev/null +++ b/packaging/_wheels/guarded/README.md @@ -0,0 +1,43 @@ +# Guarded CPython 3.11 MLX wheels + +This directory is the only allowed source of `mlx` / `mlx-metal` for the +bundled Fusion DMG runtime. Wheels are **not** committed; drop the matched +pair here before `packaging/build.py --venvstacks-only`. + +## Pin + +| Field | Value | +| --- | --- | +| Version | `0.32.1.dev20260825+26421e953` | +| Commit | `26421e953` on `jonathan308/mlx` branch `feat/jaccl-subgroups` | +| nanobind | `2.13.0` | +| Python ABI | **cp311** (the DMG embeds `cpython-3.11`) | +| Platform | `macosx_26_0_arm64` (Path B: macOS 26 only) | +| Forbidden | PyPI `mlx==0.32.0`, PyPI/stock `mlx==0.32.2` | + +TP2 rejected 0.32.2 (JACCL all-reduce loss, rank exit 75, ~92-94 GiB wired +on M5). Do not package it. + +## CPython 3.13 wheels are not drop-in + +Wheels under `minimax-m3-cluster/runtime_patches/variants/guard-0321-rollback/` +were built for **CPython 3.13**. They cannot be copied into this cp311 +venvstacks layer. A matched **frontend + backend** pair must be rebuilt from +`26421e953` with Python 3.11. + +## Build the pair (macOS 26 + Metal) + +This Linux cloud VM cannot compile Metal. On a Mac: + +```bash +scripts/build_guarded_mlx_cp311_wheels.sh +``` + +Expected filenames (do not invent bytes; produce them on macOS): + +``` +mlx-0.32.1.dev20260825+26421e953-cp311-cp311-macosx_26_0_arm64.whl +mlx_metal-0.32.1.dev20260825+26421e953-py3-none-macosx_26_0_arm64.whl +``` + +See `docs/packaging-guarded-mlx.md`. diff --git a/packaging/build.py b/packaging/build.py index 4eb73f1d37..03cc1d9960 100644 --- a/packaging/build.py +++ b/packaging/build.py @@ -25,8 +25,15 @@ EXPORT_DIR = SCRIPT_DIR / "_export" DIST_DIR = SCRIPT_DIR / "dist" WHEELS_DIR = SCRIPT_DIR / "_wheels" +GUARDED_WHEELS_DIR = WHEELS_DIR / "guarded" APP_NAME = "oMLX" +# Fusion Path B pin. NEVER package stock PyPI mlx 0.32.0 or 0.32.2. +GUARDED_MLX_VERSION = "0.32.1.dev20260825+26421e953" +GUARDED_MLX_COMMIT = "26421e953" +GUARDED_MLX_PYTHON_TAG = "cp311" +FORBIDDEN_MLX_VERSIONS = ("0.32.2",) + def _read_version() -> str: """Read version from omlx/_version.py (single source of truth).""" @@ -75,9 +82,17 @@ def _rm_onerror(func, path, exc_info): for d in dirs_to_clean: if preserve_venv and d in venv_dirs: continue - if d.exists(): - shutil.rmtree(d, onerror=_rm_onerror) - print(f" Removed {d.relative_to(SCRIPT_DIR)}/") + if d == WHEELS_DIR and d.exists(): + # Keep operator-supplied guarded mlx wheels; wipe the rest. + for child in d.iterdir(): + if child.name == "guarded": + continue + if child.is_dir(): + shutil.rmtree(child, onerror=_rm_onerror) + else: + child.unlink() + print(" Removed packaging/_wheels/ (kept guarded/)") + continue for f in files_to_clean: if preserve_venv and f.name == "_venvstacks_resolved.toml": @@ -99,12 +114,102 @@ def run_cmd(cmd: list, cwd: Path = None, check: bool = True): return result -def _resolve_mlx_version(toml_path: Path) -> str: - """Resolve the mlx version that venvstacks locked. +def _guarded_wheel_error(detail: str) -> None: + print( + " ✗ Guarded MLX wheels are required for the bundled CPython 3.11 " + "environment and must not be resolved from PyPI.\n" + f" pin: mlx/mlx-metal == {GUARDED_MLX_VERSION}\n" + f" commit: jonathan308/mlx@{GUARDED_MLX_COMMIT} " + "(feat/jaccl-subgroups), nanobind 2.13.0, ABI cp311\n" + f" {detail}\n" + " Do not use CPython 3.13 wheels from " + "minimax-m3-cluster/runtime_patches/variants/guard-0321-rollback/; " + "those are not drop-in for this runtime.\n" + " Next step (macOS 26 + Metal, not this Linux VM):\n" + " scripts/build_guarded_mlx_cp311_wheels.sh", + file=sys.stderr, + ) + sys.exit(1) + + +def _wheel_filename_version(name: str) -> str | None: + # mlx-0.32.1.dev20260825+26421e953-cp311-cp311-macosx_26_0_arm64.whl + # mlx_metal-0.32.1.dev20260825+26421e953-py3-none-macosx_26_0_arm64.whl + match = re.match( + r"^(?:mlx|mlx_metal)-(.+?)-(?:cp\d+|py3)\b", + name, + re.IGNORECASE, + ) + return match.group(1) if match else None + - Reads the locked requirements file to find the exact mlx version, - falling back to the latest version from PyPI if no lock file exists. +def _reject_forbidden_mlx(label: str) -> None: + lowered = label.lower() + for forbidden in FORBIDDEN_MLX_VERSIONS: + if forbidden in lowered: + _guarded_wheel_error( + f"refusing MLX {forbidden} in {label!r}; TP2 rejected " + "0.32.2 (JACCL all-reduce loss, rank exit 75)." + ) + + +def require_guarded_mlx_wheels() -> list[Path]: + """Require a matched local cp311 mlx + mlx-metal pair. + + Does not download from PyPI and does not invent wheel bytes. """ + if not GUARDED_WHEELS_DIR.is_dir(): + _guarded_wheel_error(f"missing directory {GUARDED_WHEELS_DIR}") + + wheels = sorted(GUARDED_WHEELS_DIR.glob("*.whl")) + if not wheels: + _guarded_wheel_error(f"no .whl files in {GUARDED_WHEELS_DIR}") + + mlx_wheels = [] + metal_wheels = [] + for whl in wheels: + _reject_forbidden_mlx(whl.name) + lowered = whl.name.lower() + if "cp313" in lowered or "cp3.13" in lowered: + _guarded_wheel_error( + f"{whl.name} is a CPython 3.13 wheel; the bundled runtime " + "is CPython 3.11 and needs a matched cp311 pair built from " + f"{GUARDED_MLX_COMMIT}." + ) + version = _wheel_filename_version(whl.name) + if version != GUARDED_MLX_VERSION: + _guarded_wheel_error( + f"{whl.name} version {version!r} does not match pin " + f"{GUARDED_MLX_VERSION}" + ) + if lowered.startswith("mlx-") and "cp311" in lowered: + mlx_wheels.append(whl) + elif lowered.startswith("mlx_metal-") or lowered.startswith("mlx-metal-"): + metal_wheels.append(whl) + + if not mlx_wheels: + _guarded_wheel_error( + f"no mlx-*{GUARDED_MLX_PYTHON_TAG}* frontend wheel in {GUARDED_WHEELS_DIR}" + ) + if not metal_wheels: + _guarded_wheel_error( + f"no mlx_metal backend wheel in {GUARDED_WHEELS_DIR}" + ) + return mlx_wheels + metal_wheels + + +def _copy_guarded_wheels_into(dest: Path) -> None: + dest.mkdir(parents=True, exist_ok=True) + for whl in require_guarded_mlx_wheels(): + target = dest / whl.name + if target.resolve() != whl.resolve(): + shutil.copy2(whl, target) + print(f" guarded {whl.name}") + + +def _resolve_mlx_version(toml_path: Path) -> str: + """Return the guarded mlx pin. Never query PyPI.""" + del toml_path req_file = ( SCRIPT_DIR / "requirements" @@ -112,35 +217,34 @@ def _resolve_mlx_version(toml_path: Path) -> str: / "requirements-framework-mlx-base-macosx_arm64.txt" ) if req_file.exists(): - import re as _re - content = req_file.read_text() - match = _re.search(r"^mlx==(\S+)", content, _re.MULTILINE) + match = re.search(r"^mlx==(\S+)", content, re.MULTILINE) if match: - return match.group(1) - - # No lock file yet — query PyPI for the latest version - import json - import urllib.request - - data = json.loads( - urllib.request.urlopen("https://pypi.org/pypi/mlx/json").read() - ) - return data["info"]["version"] + locked = match.group(1) + _reject_forbidden_mlx(locked) + if locked != GUARDED_MLX_VERSION: + _guarded_wheel_error( + f"lock file has mlx=={locked}, expected {GUARDED_MLX_VERSION}" + ) + return locked + return GUARDED_MLX_VERSION def swap_platform_wheels( export_dir: Path, macos_target: str, python_version: str = "3.11" ): - """Replace mlx and mlx-metal in exported venvstacks with platform-specific wheels. + """Install the guarded local mlx/mlx-metal wheels into the export. - Downloads the wheels for the given macOS target (e.g. "26.0") and replaces - the existing packages in the framework layer's site-packages. This allows - building on macOS 15 while targeting macOS 26 wheels that contain - M5 Neural Accelerator matmul kernels. + Never downloads from PyPI. Path B of this beta is macOS 26-only; the + local wheels must already be a macosx_26_* cp311 pair. """ import zipfile + if python_version != "3.11": + _guarded_wheel_error( + f"bundled runtime is CPython 3.11, not {python_version}" + ) + site_packages = ( export_dir / "framework-mlx-base" @@ -153,28 +257,28 @@ def swap_platform_wheels( sys.exit(1) platform_tag = f"macosx_{macos_target.replace('.', '_')}_arm64" - toml_path = SCRIPT_DIR / "venvstacks.toml" - mlx_version = _resolve_mlx_version(toml_path) - packages = ["mlx", "mlx-metal"] - - print(f"\n Swapping mlx/mlx-metal to {platform_tag} (v{mlx_version})...") + mlx_version = _resolve_mlx_version(SCRIPT_DIR / "venvstacks.toml") + _reject_forbidden_mlx(mlx_version) + if mlx_version != GUARDED_MLX_VERSION: + _guarded_wheel_error( + f"refusing to install mlx=={mlx_version}; pin is {GUARDED_MLX_VERSION}" + ) - # Download platform-specific wheels + print(f"\n Installing guarded mlx/mlx-metal {mlx_version} for {platform_tag}...") wheels_tmp = SCRIPT_DIR / "_platform_wheels" if wheels_tmp.exists(): shutil.rmtree(wheels_tmp) wheels_tmp.mkdir() - - for pkg in packages: - run_cmd([ - sys.executable, "-m", "pip", "download", - f"{pkg}=={mlx_version}", - "--platform", platform_tag, - f"--python-version={python_version}", - "--only-binary", ":all:", - "--no-deps", - "-d", str(wheels_tmp), - ]) + selected = [] + for whl in require_guarded_mlx_wheels(): + if "macosx_26" not in whl.name: + _guarded_wheel_error( + f"{whl.name} is not a macOS 26 arm64 wheel (expected {platform_tag})" + ) + shutil.copy2(whl, wheels_tmp / whl.name) + selected.append(whl.name) + if not selected: + _guarded_wheel_error("no guarded wheels to install") # Remove existing mlx/mlx-metal from site-packages for item in site_packages.iterdir(): @@ -186,7 +290,7 @@ def swap_platform_wheels( shutil.rmtree(item) print(f" Removed {item.name}") - # Install downloaded wheels into site-packages + # Install guarded wheels into site-packages for whl in wheels_tmp.glob("*.whl"): print(f" Installing {whl.name}") with zipfile.ZipFile(whl) as zf: @@ -294,10 +398,19 @@ def build_local_wheels(source_toml: Path | None = None): source_toml = SCRIPT_DIR / "venvstacks.toml" git_reqs = _parse_git_requirements(source_toml) - # Clean and recreate wheels dir for fresh builds + # Recreate the working wheels dir for git-pinned packages, then copy + # the guarded mlx pair back in so venvstacks never consults PyPI for mlx. + require_guarded_mlx_wheels() if WHEELS_DIR.exists(): - shutil.rmtree(WHEELS_DIR) - WHEELS_DIR.mkdir(parents=True) + for child in WHEELS_DIR.iterdir(): + if child.resolve() == GUARDED_WHEELS_DIR.resolve(): + continue + if child.is_dir(): + shutil.rmtree(child) + else: + child.unlink() + WHEELS_DIR.mkdir(parents=True, exist_ok=True) + _copy_guarded_wheels_into(WHEELS_DIR) # Build wheels from git-pinned packages for full_req, git_url in git_reqs: @@ -639,6 +752,11 @@ def _venvstacks_driver() -> list[str]: def build_venvstacks(): """Build venvstacks layers.""" print("\n[1/4] Building venvstacks layers...") + print( + f" Guarded MLX pin: {GUARDED_MLX_VERSION} " + f"(commit {GUARDED_MLX_COMMIT}, cp311, never PyPI 0.32.0/0.32.2)" + ) + require_guarded_mlx_wheels() venvstacks_cmd = _venvstacks_driver() print(f" Using venvstacks driver: {' '.join(venvstacks_cmd)}") @@ -1062,10 +1180,16 @@ def _compute_donor_fingerprint() -> str: SCRIPT_DIR.parent / "pyproject.toml", SCRIPT_DIR / "venvstacks.toml", SCRIPT_DIR.parent / "uv.lock", + SCRIPT_DIR / "_wheels" / "guarded" / "README.md", ] for path in inputs: if path.exists(): h.update(path.read_bytes()) + h.update(GUARDED_MLX_VERSION.encode()) + if GUARDED_WHEELS_DIR.is_dir(): + for whl in sorted(GUARDED_WHEELS_DIR.glob("*.whl")): + h.update(whl.name.encode()) + h.update(str(whl.stat().st_size).encode()) return h.hexdigest() @@ -1100,9 +1224,9 @@ def main(): "staged omlx package directory and exit. " "`build.sh` uses this after copying Resources/omlx.") parser.add_argument("--macos-target", - help="Target macOS version for mlx/mlx-metal wheels " - "(e.g. 26.0). Downloads platform-specific wheels " - "with M5 Neural Accelerator support.") + help="Install the guarded local mlx/mlx-metal wheels " + "for this macOS tag (Path B: 26.0). Never " + "downloads mlx from PyPI.") args = parser.parse_args() if args.print_fingerprint: diff --git a/packaging/venvstacks.toml b/packaging/venvstacks.toml index 792da817d8..19e87b8b66 100644 --- a/packaging/venvstacks.toml +++ b/packaging/venvstacks.toml @@ -41,6 +41,10 @@ dynlib_exclude = [ [tool.uv] exclude-newer = "2026-04-23T00:00:00Z" +# Local cp311 mlx/mlx-metal pair. Without this, uv would look on PyPI and +# could land stock 0.32.0 or 0.32.2. Wheels are not committed; see +# packaging/_wheels/guarded/README.md. +find-links = ["_wheels/guarded"] environments = [ "sys_platform == 'darwin' and platform_machine == 'arm64'", ] diff --git a/pyproject.toml b/pyproject.toml index b0d53780d4..5dd6dd3bf9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,18 +3,18 @@ requires = [ "setuptools>=61.0", "wheel", "cmake>=3.27", - # Pin nanobind to match the ABI version MLX 0.32.0 was built with - # (MLX pins nanobind v2.13.0 via FetchContent in its CMakeLists.txt). + # Pin nanobind to match the ABI version the guarded MLX pair was built + # with (MLX pins nanobind v2.13.0 via FetchContent in its CMakeLists.txt). # A mismatched nanobind isolates the `mlx` NB_DOMAIN, so custom kernel # extensions (omlx.custom_kernels.*) reject every mlx.core.array at the # type caster with `incompatible function arguments`. "nanobind==2.13.0", - # mlx floor admits the custom 0.32.0.dev build (JACCL guards/backports); - # an exact ==0.32.0 pin would make a fresh resolve replace it with the - # stock PyPI wheel and silently drop those fixes. When building with - # isolation, pre-install the dev wheel with --no-build-isolation or - # PIP_NO_BUILD_ISOLATION so this floor resolves to it. - "mlx>=0.32.0.dev0,<0.33", + # Exact local version of the JACCL-qualified pair from + # jonathan308/mlx@26421e953 (feat/jaccl-subgroups). This is not on + # PyPI. A range such as >=0.32.0.dev0,<0.33 admits stock 0.32.2, which + # TP2 rejected (JACCL all-reduce loss, rank exit 75). Isolated builds + # must use the local cp311 wheels via --no-build-isolation / find-links. + "mlx==0.32.1.dev20260825+26421e953", ] build-backend = "setuptools.build_meta" @@ -44,16 +44,16 @@ classifiers = [ ] dependencies = [ - # MLX 0.32.0 line; optional custom kernels live under - # omlx.custom_kernels.* when built for release. The bundled kernel - # binaries are ABI-coupled to this mlx version — bumping the pin - # requires rebuilding them (OMLX_WITH_CUSTOM_KERNEL=1 setup.py build_ext). - # The floor admits the custom 0.32.0.dev wheel (JACCL guards/backports): - # an exact ==0.32.0 pin made a fresh resolve reinstall the stock PyPI - # wheel over the dev build, silently dropping those fixes. Install the - # dev wheel first with --no-deps (ops/install_mlx_variant.sh) so the - # resolver keeps it. - "mlx>=0.32.0.dev0,<0.33", + # Guarded MLX 0.32.1.dev pair (jonathan308/mlx@26421e953, nanobind + # 2.13.0). Optional custom kernels live under omlx.custom_kernels.* + # when built for release. The bundled kernel binaries are ABI-coupled + # to this exact mlx version — bumping the pin requires rebuilding them + # (OMLX_WITH_CUSTOM_KERNEL=1 setup.py build_ext). NEVER resolve stock + # PyPI mlx 0.32.0 or 0.32.2 into the bundled CPython 3.11 environment. + # Install the matched cp311 frontend+backend wheels from + # packaging/_wheels/guarded (see docs/packaging-guarded-mlx.md). + "mlx==0.32.1.dev20260825+26421e953", + "mlx-metal==0.32.1.dev20260825+26421e953", # mlx-lm from commit (ab1806e, v0.31.3) - transformers 5.13+ # NewlineTokenizer registration compatibility, DeepSeek/GLM DSA indexer # RoPE fix, trust_remote_code load gate, transformers 5.7.0 @@ -281,12 +281,16 @@ include = ["omlx*"] "omlx.custom_kernels.minimax_m3" = ["*.metallib", "*.dylib", "*.so"] "omlx.custom_kernels.qwen35_prefill" = ["*.metallib", "*.dylib", "*.so"] "omlx.custom_kernels.decode_fast" = ["*.metallib", "*.dylib", "*.so"] +"omlx.custom_kernels.bonsai" = ["*.metallib", "*.dylib", "*.so"] [tool.uv] -# mlx and mlx-lm are git-pinned; override transitive pins -# (e.g. mlx-audio -> mlx-lm==0.31.1) so the resolver accepts them. +# Guarded mlx/mlx-metal are local cp311 wheels, not PyPI. find-links keeps +# a fresh resolve from landing stock 0.32.0 or 0.32.2. mlx-lm stays git-pinned +# so mlx-audio's transitive mlx-lm==0.31.1 pin is overridden. +find-links = ["packaging/_wheels/guarded"] override-dependencies = [ - "mlx==0.32.0", + "mlx==0.32.1.dev20260825+26421e953", + "mlx-metal==0.32.1.dev20260825+26421e953", "mlx-lm @ git+https://github.com/ml-explore/mlx-lm@ab1806e8f5d6aa035973af194a1b9198ab4754dc", ] diff --git a/scripts/build_guarded_mlx_cp311_wheels.sh b/scripts/build_guarded_mlx_cp311_wheels.sh new file mode 100755 index 0000000000..8ef9c8ca38 --- /dev/null +++ b/scripts/build_guarded_mlx_cp311_wheels.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# Build a matched CPython 3.11 mlx + mlx-metal wheel pair from +# jonathan308/mlx@26421e953 for the Fusion DMG. +# +# Metal-only. Do not run on Linux. Does not invent wheel bytes. + +set -euo pipefail + +PIN_VERSION="0.32.1.dev20260825+26421e953" +PIN_COMMIT="26421e953" +PIN_REPO="https://github.com/jonathan308/mlx.git" +NANOBIND_PIN="2.13.0" +MACOS_TARGET="26.0" + +die() { + echo "build_guarded_mlx_cp311_wheels.sh: error: $*" >&2 + exit 1 +} + +[ "$(uname -s)" = "Darwin" ] || die "Metal wheels must be built on macOS, not $(uname -s)" +[ "$(uname -m)" = "arm64" ] || die "expected Apple Silicon arm64, got $(uname -m)" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" +OUT_DIR="$REPO_ROOT/packaging/_wheels/guarded" +mkdir -p "$OUT_DIR" + +PYTHON_BIN="${PYTHON_BIN:-$(command -v python3.11 || command -v python3 || true)}" +[ -n "$PYTHON_BIN" ] || die "python3.11 not found" +"$PYTHON_BIN" - <<'PY' || die "interpreter is not CPython 3.11" +import sys +raise SystemExit(0 if sys.version_info[:2] == (3, 11) else 1) +PY + +"$PYTHON_BIN" -m pip install -q "nanobind==$NANOBIND_PIN" "cmake>=3.27" \ + "setuptools" "wheel" "pip" + +SRC_DIR="${OMLX_MLX_SRC:-}" +cleanup() { + if [ -n "${TMP_SRC:-}" ]; then + rm -rf "$TMP_SRC" + fi +} +trap cleanup EXIT + +if [ -z "$SRC_DIR" ]; then + TMP_SRC="$(mktemp -d "${TMPDIR:-/tmp}/omlx-mlx-src.XXXXXX")" + git clone --filter=blob:none "$PIN_REPO" "$TMP_SRC" + git -C "$TMP_SRC" checkout --detach "$PIN_COMMIT" + SRC_DIR="$TMP_SRC" +fi + +actual="$(git -C "$SRC_DIR" rev-parse HEAD)" +[[ "$actual" == "$PIN_COMMIT"* ]] \ + || die "mlx checkout $actual is not $PIN_COMMIT" + +export MACOSX_DEPLOYMENT_TARGET="$MACOS_TARGET" +export CMAKE_ARGS="${CMAKE_ARGS:+$CMAKE_ARGS }-DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOS_TARGET}" +export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL:-$(sysctl -n hw.ncpu)}" + +DIST="$SRC_DIR/dist" +rm -rf "$DIST" +mkdir -p "$DIST" + +( + cd "$SRC_DIR" + "$PYTHON_BIN" setup.py clean --all + MLX_BUILD_STAGE=1 "$PYTHON_BIN" setup.py bdist_wheel + "$PYTHON_BIN" setup.py clean --all + MLX_BUILD_STAGE=2 "$PYTHON_BIN" setup.py bdist_wheel +) + +shopt -s nullglob +wheels=("$DIST"/*.whl) +[ "${#wheels[@]}" -gt 0 ] || die "no wheels produced in $DIST" + +found_mlx=0 +found_metal=0 +for whl in "${wheels[@]}"; do + base="$(basename "$whl")" + case "$base" in + *0.32.2*) die "produced $base contains forbidden 0.32.2" ;; + *cp313*) die "produced $base is cp313; need cp311" ;; + esac + case "$base" in + mlx-*"$PIN_VERSION"*cp311*) found_mlx=1 ;; + mlx_metal-*"$PIN_VERSION"*) found_metal=1 ;; + esac + cp "$whl" "$OUT_DIR/$base" + echo "copied $base" +done + +if [ "$found_mlx" -ne 1 ] || [ "$found_metal" -ne 1 ]; then + echo "warning: expected version string $PIN_VERSION was not both frontend and backend." >&2 + echo "mlx setup.py stamps .devYYYYMMDD from the build date. The DMG pin is exact;" >&2 + echo "rebuild so metadata matches $PIN_VERSION or stop and inspect $OUT_DIR." >&2 + ls -1 "$OUT_DIR" + exit 1 +fi + +echo "Guarded cp311 pair ready in $OUT_DIR" +echo "Pin: $PIN_VERSION (commit $PIN_COMMIT, nanobind $NANOBIND_PIN, macOS $MACOS_TARGET)" diff --git a/scripts/release_macos_dmg.sh b/scripts/release_macos_dmg.sh index 0c0805427e..bdd2683dff 100755 --- a/scripts/release_macos_dmg.sh +++ b/scripts/release_macos_dmg.sh @@ -234,7 +234,7 @@ mkdir -p "$DMG_ROOT" ditto "$APP_PATH" "$DMG_ROOT/oMLX.app" ln -s /Applications "$DMG_ROOT/Applications" -DMG_NAME="oMLX-$APP_VERSION-macos15-26-arm64.dmg" +DMG_NAME="oMLX-$APP_VERSION-macos26-arm64.dmg" DMG_PATH="$OUTPUT_DIR/$DMG_NAME" hdiutil create -volname "oMLX $APP_VERSION" -srcfolder "$DMG_ROOT" \ -format UDZO -ov "$DMG_PATH" diff --git a/setup.py b/setup.py index 03a097331a..614f650d5a 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ CUSTOM_KERNEL_FLAG = "--with-custom-kernel" TRUTHY = {"1", "true", "yes", "on"} -DEFAULT_CUSTOM_KERNEL_DEPLOYMENT_TARGET = "15.0" +DEFAULT_CUSTOM_KERNEL_DEPLOYMENT_TARGET = "26.0" def _with_custom_kernel() -> bool: diff --git a/tests/test_custom_kernel_abi_probe.py b/tests/test_custom_kernel_abi_probe.py index d1d1d51529..3526e0f3bb 100644 --- a/tests/test_custom_kernel_abi_probe.py +++ b/tests/test_custom_kernel_abi_probe.py @@ -10,11 +10,12 @@ import pytest from omlx.custom_kernels.bonsai import fast as bonsai_fast +from omlx.custom_kernels.decode_fast import fast as decode_fast from omlx.custom_kernels.glm_moe_dsa import fast as glm_fast from omlx.custom_kernels.minimax_m3 import fast as minimax_fast from omlx.custom_kernels.qwen35_prefill import fast as qwen35_fast -ALL_FAST = (qwen35_fast, glm_fast, minimax_fast, bonsai_fast) +ALL_FAST = (qwen35_fast, glm_fast, minimax_fast, bonsai_fast, decode_fast) class _MismatchedExt: diff --git a/tests/test_packaging_guarded_mlx_pin.py b/tests/test_packaging_guarded_mlx_pin.py new file mode 100644 index 0000000000..12d592010e --- /dev/null +++ b/tests/test_packaging_guarded_mlx_pin.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The Fusion DMG must pin guarded MLX and must not admit 0.32.2.""" + +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +PIN = "0.32.1.dev20260825+26421e953" + + +def _read(rel: str) -> str: + return (ROOT / rel).read_text() + + +def test_pyproject_pins_exact_guarded_mlx_not_open_range(): + text = _read("pyproject.toml") + assert "mlx>=0.32.0.dev0,<0.33" not in text + assert PIN in text + assert f"mlx=={PIN}" in text + assert f"mlx-metal=={PIN}" in text + overrides = text.split("[tool.uv]", 1)[1] + assert f"mlx=={PIN}" in overrides + assert "mlx==0.32.0" not in overrides + + +def test_packaging_build_never_queries_pypi_for_mlx(): + text = _read("packaging/build.py") + assert "pypi.org/pypi/mlx" not in text + assert "GUARDED_MLX_VERSION" in text + assert PIN in text + assert "0.32.2" in text # forbidden-version guard + + +def test_dmg_name_is_macos26_only(): + script = _read("scripts/release_macos_dmg.sh") + assert "macos15-26" not in script + assert 'DMG_NAME="oMLX-$APP_VERSION-macos26-arm64.dmg"' in script + + +def test_checklist_does_not_label_runtime_macos_15_26(): + text = _read("docs/release-public-checklist.md") + assert "macos15-26" not in text + assert "macos26-arm64" in text + + +@pytest.mark.parametrize( + "rel", + [ + "packaging/_wheels/guarded/README.md", + "docs/packaging-guarded-mlx.md", + ], +) +def test_docs_reject_cp313_rollback_wheels(rel): + text = _read(rel) + assert "cp313" in text.lower() or "CPython 3.13" in text + assert "not drop-in" in text or "not drop-in" in text.lower() + assert "26421e953" in text + assert "0.32.2" in text From b46e4f2b00a2d7d3bdf04ad3c4eda200e5480344 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 25 Aug 2026 14:40:37 +0000 Subject: [PATCH 2/4] docs: keep Metal wheel builds off the live DS4 Studio Refuse self-hosted cluster-coordinator runners in the signed DMG workflow and require an explicit idle-Mac override before compiling guarded cp311 mlx-metal wheels. The Studio serving live DS4 stays up. Co-authored-by: Jonathan Spangler --- .github/workflows/release-macos-dmg.yml | 21 +++++++++++++++++++++ docs/dev-prerelease-notes-template.md | 2 +- docs/packaging-guarded-mlx.md | 14 +++++++++++--- docs/release-public-checklist.md | 9 ++++++--- packaging/README.md | 4 ++++ packaging/_wheels/guarded/README.md | 9 +++++++-- packaging/build.py | 6 ++++-- scripts/build_guarded_mlx_cp311_wheels.sh | 8 ++++++++ tests/test_packaging_guarded_mlx_pin.py | 18 ++++++++++++++++++ 9 files changed, 80 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release-macos-dmg.yml b/.github/workflows/release-macos-dmg.yml index d7f3b8b4d8..1fecf5de89 100644 --- a/.github/workflows/release-macos-dmg.yml +++ b/.github/workflows/release-macos-dmg.yml @@ -19,9 +19,20 @@ concurrency: jobs: build-unsigned-app: name: Build unsigned arm64 app + # GitHub-hosted macos-26 only. Do not retarget to a self-hosted Mac that + # is the live Fusion cluster coordinator (Studio serving DS4). runs-on: macos-26 timeout-minutes: 120 steps: + - name: Refuse self-hosted cluster coordinator + run: | + set -euo pipefail + if [ "${RUNNER_ENVIRONMENT:-}" != "github-hosted" ]; then + echo "this job must run on GitHub-hosted macos-26, not a self-hosted Mac" >&2 + echo "the live Fusion Studio / cluster coordinator serving DS4 is off-limits" >&2 + exit 1 + fi + - name: Check out the release tag uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: @@ -121,12 +132,22 @@ jobs: sign-notarize-release: name: Sign, notarize, and draft release needs: build-unsigned-app + # GitHub-hosted macos-26 only. Do not retarget to the live Fusion coordinator. runs-on: macos-26 timeout-minutes: 180 environment: macos-release permissions: contents: write steps: + - name: Refuse self-hosted cluster coordinator + run: | + set -euo pipefail + if [ "${RUNNER_ENVIRONMENT:-}" != "github-hosted" ]; then + echo "this job must run on GitHub-hosted macos-26, not a self-hosted Mac" >&2 + echo "the live Fusion Studio / cluster coordinator serving DS4 is off-limits" >&2 + exit 1 + fi + # Release tooling and entitlements come from the protected default # branch, while the untrusted/tagged app payload comes from the prior # secretless job. diff --git a/docs/dev-prerelease-notes-template.md b/docs/dev-prerelease-notes-template.md index 8789560260..680c96a921 100644 --- a/docs/dev-prerelease-notes-template.md +++ b/docs/dev-prerelease-notes-template.md @@ -23,7 +23,7 @@ Expected artifact after the release gates pass: | Concurrent generation | Passed | Qwen completed four 512-token streams at 216.3 aggregate tok/s; DS4 completed its qualified B4 run at 69.3 aggregate tok/s. | | Functional greedy parity | Passed with limitation | With thinking disabled, four of six Qwen cases were token-exact and two of six were semantically equivalent. This is not a bit-exact or token-exact parity claim. | | Version and tag | Source bumped; tag blocked | `omlx/_version.py` is `0.6.4b1`. Do not create `v0.6.4b1` until the guarded cp311 wheels are reviewed. | -| Guarded MLX pin | Blocked on macOS wheel build | Pin `0.32.1.dev20260825+26421e953` from `26421e953`. Never 0.32.2. cp313 rollback wheels are not drop-in. | +| Guarded MLX pin | Blocked: not on the live Studio | Pin `0.32.1.dev20260825+26421e953` from `26421e953`. Never 0.32.2. cp313 rollback wheels are not drop-in. Do **not** compile on the DS4-serving Studio / cluster coordinator. | | Signed, notarized DMG | Blocked | macOS 26-only artifact name `oMLX-0.6.4b1-macos26-arm64.dmg`. Add Apple secrets and `APPLE_TEAM_ID` to `macos-release`. | Do not advertise this candidate as signed, notarized, or downloadable until the diff --git a/docs/packaging-guarded-mlx.md b/docs/packaging-guarded-mlx.md index 335ce3a593..5e56c078d6 100644 --- a/docs/packaging-guarded-mlx.md +++ b/docs/packaging-guarded-mlx.md @@ -40,12 +40,20 @@ Rebuild with `OMLX_WITH_CUSTOM_KERNEL=1` against the guarded cp311 pair: `apps/omlx-mac/Scripts/build.sh release --with-custom-kernel` is the DMG path. Default kernel deployment target is **26.0**. -## Next local / macOS step +## Next local / macOS step (not the live Studio) -This pin does **not** include wheel binaries. On a macOS 26 Apple Silicon -machine with Xcode 26.6 and CPython 3.11: +This pin does **not** include wheel bytes. **Do not** run +`scripts/build_guarded_mlx_cp311_wheels.sh`, `setup.py build_ext`, or +`apps/omlx-mac/Scripts/build.sh` on the Studio that is serving live DS4, +and do not send this work to a self-hosted Mac worker / private machine +that is the in-use cluster coordinator. + +cp311 `mlx` / `mlx-metal` compile waits for a later window or a +**GitHub-hosted** `macos-26` runner that is not that coordinator. When +that idle Mac exists, with Xcode 26.6 and CPython 3.11: ```bash +export OMLX_I_AM_NOT_THE_LIVE_CLUSTER_COORDINATOR=1 scripts/build_guarded_mlx_cp311_wheels.sh ``` diff --git a/docs/release-public-checklist.md b/docs/release-public-checklist.md index fa045b7b39..3effe766f2 100644 --- a/docs/release-public-checklist.md +++ b/docs/release-public-checklist.md @@ -67,9 +67,12 @@ before future Xcode pin changes instead of assuming `macos-latest` is stable. release commit, and only then create and push a matching tag such as `v0.6.4b1`. Prefer a protected, signed tag. **Do not create `v0.6.4b1` until the guarded cp311 wheels exist and have been reviewed.** -2. On macOS 26, build the matched mlx/mlx-metal pair with - `scripts/build_guarded_mlx_cp311_wheels.sh` and place them in - `packaging/_wheels/guarded/`. See [packaging-guarded-mlx.md](packaging-guarded-mlx.md). +2. **Do not compile on the live Fusion Studio / cluster coordinator + while it is serving DS4.** cp311 mlx-metal builds wait for a later + window or a GitHub-hosted `macos-26` runner that is not that machine. + Never run `scripts/build_guarded_mlx_cp311_wheels.sh`, + `setup.py build_ext`, or `apps/omlx-mac/Scripts/build.sh` there. + When an idle Mac exists, see [packaging-guarded-mlx.md](packaging-guarded-mlx.md). 3. Run `Signed macOS DMG` manually and enter that existing tag. Approve the `macos-release` environment only after confirming the tag resolves to the intended reviewed commit. diff --git a/packaging/README.md b/packaging/README.md index 59522456a3..924372e114 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -25,6 +25,10 @@ this directory only hands it a `_export/` tree of Python layers. ## Build +Do **not** run the commands below on the live Fusion Studio / cluster +coordinator while it is serving DS4. Wait for a later window or a +GitHub-hosted macos-26 runner that is not that machine. + ```bash # Re-export the venvstacks layers (cold ~10-20 min, warm ~4 min) python packaging/build.py --venvstacks-only diff --git a/packaging/_wheels/guarded/README.md b/packaging/_wheels/guarded/README.md index 4db03c341a..be5bd4b9d3 100644 --- a/packaging/_wheels/guarded/README.md +++ b/packaging/_wheels/guarded/README.md @@ -25,9 +25,14 @@ were built for **CPython 3.13**. They cannot be copied into this cp311 venvstacks layer. A matched **frontend + backend** pair must be rebuilt from `26421e953` with Python 3.11. -## Build the pair (macOS 26 + Metal) +## Build the pair later (not the live Studio) -This Linux cloud VM cannot compile Metal. On a Mac: +Do **not** compile Metal, run `setup.py build_ext`, or run +`apps/omlx-mac/Scripts/build.sh` on the Fusion Studio that is serving +live DS4, and do not send this to a self-hosted Mac that is the cluster +coordinator. This Linux cloud VM also cannot compile Metal. + +When an idle GitHub-hosted macos-26 runner or later window exists: ```bash scripts/build_guarded_mlx_cp311_wheels.sh diff --git a/packaging/build.py b/packaging/build.py index 03cc1d9960..230dba79a7 100644 --- a/packaging/build.py +++ b/packaging/build.py @@ -125,8 +125,10 @@ def _guarded_wheel_error(detail: str) -> None: " Do not use CPython 3.13 wheels from " "minimax-m3-cluster/runtime_patches/variants/guard-0321-rollback/; " "those are not drop-in for this runtime.\n" - " Next step (macOS 26 + Metal, not this Linux VM):\n" - " scripts/build_guarded_mlx_cp311_wheels.sh", + " Next step: wait for a later window or a GitHub-hosted macos-26 " + "runner that is NOT the live Fusion Studio / DS4 cluster coordinator.\n" + " Do not run scripts/build_guarded_mlx_cp311_wheels.sh, " + "setup.py build_ext, or apps/omlx-mac/Scripts/build.sh on that machine.", file=sys.stderr, ) sys.exit(1) diff --git a/scripts/build_guarded_mlx_cp311_wheels.sh b/scripts/build_guarded_mlx_cp311_wheels.sh index 8ef9c8ca38..870831f227 100755 --- a/scripts/build_guarded_mlx_cp311_wheels.sh +++ b/scripts/build_guarded_mlx_cp311_wheels.sh @@ -3,6 +3,11 @@ # jonathan308/mlx@26421e953 for the Fusion DMG. # # Metal-only. Do not run on Linux. Does not invent wheel bytes. +# +# HARD STOP: do not run this on the live Fusion Studio / cluster +# coordinator while it is serving DS4. Do not send it to a self-hosted +# Mac worker that is that machine. Wait for a later window or a +# GitHub-hosted macos-26 runner that is not the coordinator. set -euo pipefail @@ -17,6 +22,9 @@ die() { exit 1 } +[ "${OMLX_I_AM_NOT_THE_LIVE_CLUSTER_COORDINATOR:-}" = "1" ] || die \ + "refusing to compile Metal on a machine that might be the live DS4 Studio. Wait for a later window or a GitHub-hosted macos-26 runner that is not the cluster coordinator. To proceed on an idle Mac, export OMLX_I_AM_NOT_THE_LIVE_CLUSTER_COORDINATOR=1" + [ "$(uname -s)" = "Darwin" ] || die "Metal wheels must be built on macOS, not $(uname -s)" [ "$(uname -m)" = "arm64" ] || die "expected Apple Silicon arm64, got $(uname -m)" diff --git a/tests/test_packaging_guarded_mlx_pin.py b/tests/test_packaging_guarded_mlx_pin.py index 12d592010e..57c7f8e02c 100644 --- a/tests/test_packaging_guarded_mlx_pin.py +++ b/tests/test_packaging_guarded_mlx_pin.py @@ -57,3 +57,21 @@ def test_docs_reject_cp313_rollback_wheels(rel): assert "not drop-in" in text or "not drop-in" in text.lower() assert "26421e953" in text assert "0.32.2" in text + + +def test_live_studio_coordinator_is_off_limits_for_metal_compile(): + docs = " ".join( + _read(rel) + for rel in ( + "docs/packaging-guarded-mlx.md", + "docs/release-public-checklist.md", + "scripts/build_guarded_mlx_cp311_wheels.sh", + ".github/workflows/release-macos-dmg.yml", + ) + ) + assert "serving live DS4" in docs or "serving DS4" in docs + assert "cluster coordinator" in docs + assert "OMLX_I_AM_NOT_THE_LIVE_CLUSTER_COORDINATOR" in _read( + "scripts/build_guarded_mlx_cp311_wheels.sh" + ) + assert "github-hosted" in _read(".github/workflows/release-macos-dmg.yml") From fa58617f0225f49ea4024f1b13b67bf3784d781e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 25 Aug 2026 14:46:23 +0000 Subject: [PATCH 3/4] fix: restore ops/install_mlx_variant.sh and keep uv off stock MLX Fusion 3a82bd27 referenced this --no-deps install path but the file was missing, and [tool.uv] override-dependencies still pinned mlx==0.32.0. Install only the guarded cp311 pair from packaging/_wheels/guarded with --no-index so a fresh resolve cannot land stock 0.32.0 or 0.32.2. Co-authored-by: Jonathan Spangler --- docs/packaging-guarded-mlx.md | 19 +++- docs/release-public-checklist.md | 13 ++- ops/install_mlx_variant.sh | 143 ++++++++++++++++++++++++ packaging/_wheels/guarded/README.md | 4 +- packaging/build.py | 10 +- pyproject.toml | 14 ++- tests/test_packaging_guarded_mlx_pin.py | 47 +++++++- 7 files changed, 227 insertions(+), 23 deletions(-) create mode 100755 ops/install_mlx_variant.sh diff --git a/docs/packaging-guarded-mlx.md b/docs/packaging-guarded-mlx.md index 5e56c078d6..2efb019819 100644 --- a/docs/packaging-guarded-mlx.md +++ b/docs/packaging-guarded-mlx.md @@ -15,8 +15,15 @@ bundled CPython 3.11 environment must not resolve stock PyPI MLX. `pyproject.toml` previously used `mlx>=0.32.0.dev0,<0.33`, which admits 0.32.2, and `[tool.uv] override-dependencies` forced stock `mlx==0.32.0`. -Both are now the exact local version above. A fresh resolve without the -local wheels must fail rather than land PyPI. +Both are now the exact local version above. Fusion 3a82bd27 also referenced +`ops/install_mlx_variant.sh` (install the custom wheel first with `--no-deps`) +but that file was missing; it is restored here and refuses 0.32.0 / 0.32.2. +A fresh resolve without the local wheels must fail rather than land PyPI. + +```bash +ops/install_mlx_variant.sh --check +ops/install_mlx_variant.sh --python /path/to/cpython-3.11 +``` ## CPython 3.13 rollback wheels are not drop-in @@ -65,8 +72,12 @@ venvstacks export and the app on that Mac. Do not create or push tag ## Signing environment -Create GitHub Environment `macos-release` (protected, required reviewers). -Do not commit Apple credentials. In the GitHub UI add secrets: +Create GitHub Environment `macos-release` on **jonathan308/omlx** (empty, +protected, required reviewer **jonathan308**). Do **not** upload secrets +when creating it, and do not copy secrets from `jonathan308/omlx-fusion`. +Do not approve or revive omlx-fusion Signed macOS DMG run 32676126995. + +Later, in the GitHub UI only, add secrets: - `APPLE_DEVELOPER_ID_APPLICATION_P12_BASE64` - `APPLE_DEVELOPER_ID_APPLICATION_P12_PASSWORD` diff --git a/docs/release-public-checklist.md b/docs/release-public-checklist.md index 3effe766f2..833761d7ac 100644 --- a/docs/release-public-checklist.md +++ b/docs/release-public-checklist.md @@ -28,11 +28,14 @@ and [common issue checks](https://developer.apple.com/documentation/security/res ## One-time GitHub setup -Create a GitHub Actions environment named `macos-release`. Restrict it with -required reviewers and a deployment-branch policy. Environment protection rules -delay access to environment secrets until approval; see [GitHub's environment documentation](https://docs.github.com/en/actions/reference/workflows-and-actions/deployments-and-environments). - -Jonathan must add these environment secrets in the GitHub UI (never commit them): +Create a GitHub Actions environment named `macos-release` on jonathan308/omlx: +empty, protected, required reviewer **jonathan308**. Do **not** upload secrets +at create time and do not copy secrets from `jonathan308/omlx-fusion`. Do not +approve or revive omlx-fusion Signed macOS DMG run 32676126995. A deployment-branch +policy can be added later. Environment protection rules delay access to +environment secrets until approval; see [GitHub's environment documentation](https://docs.github.com/en/actions/reference/workflows-and-actions/deployments-and-environments). + +Later, Jonathan must add these environment secrets in the GitHub UI (never commit them): | Name | Value | | --- | --- | diff --git a/ops/install_mlx_variant.sh b/ops/install_mlx_variant.sh new file mode 100755 index 0000000000..e444f13b96 --- /dev/null +++ b/ops/install_mlx_variant.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# Install the guarded MLX cp311 pair with --no-deps / --no-index. +# +# Fusion 3a82bd27 referenced this path from pyproject.toml but the file was +# missing, so a fresh uv/pip resolve could still land stock PyPI mlx 0.32.0 +# (the old [tool.uv] override) or 0.32.2 (the open 0.32.x range). This script +# is the operator install path: local wheels only, exact pin, never PyPI. +# +# This does NOT compile Metal, does NOT run setup.py build_ext, and does NOT +# SSH to / mutate the live Fusion cluster ranks while the Studio is serving +# DS4. If packaging/_wheels/guarded is empty, fail and wait — do not invent +# wheel bytes and do not fall back to stock. + +set -euo pipefail + +PIN_VERSION="0.32.1.dev20260825+26421e953" +PIN_COMMIT="26421e953" +PYTHON_BIN="${PYTHON_BIN:-}" +CHECK_ONLY=0 + +die() { + echo "ops/install_mlx_variant.sh: error: $*" >&2 + exit 1 +} + +usage() { + cat </dev/null 2>&1 || true + +# --no-index: a missing local wheel must fail, not silently pull PyPI 0.32.0/0.32.2. +"$PYTHON_BIN" -m pip install \ + --no-deps \ + --no-index \ + --find-links "$WHEEL_DIR" \ + "mlx==${PIN_VERSION}" \ + "mlx-metal==${PIN_VERSION}" + +"$PYTHON_BIN" - < str: @@ -149,9 +149,13 @@ def _reject_forbidden_mlx(label: str) -> None: lowered = label.lower() for forbidden in FORBIDDEN_MLX_VERSIONS: if forbidden in lowered: + extra = ( + "TP2 rejected 0.32.2 (JACCL all-reduce loss, rank exit 75)." + if forbidden == "0.32.2" + else "Fusion 3a82bd27's uv override used stock 0.32.0; do not restore it." + ) _guarded_wheel_error( - f"refusing MLX {forbidden} in {label!r}; TP2 rejected " - "0.32.2 (JACCL all-reduce loss, rank exit 75)." + f"refusing MLX {forbidden} in {label!r}. {extra}" ) diff --git a/pyproject.toml b/pyproject.toml index 5dd6dd3bf9..d9417ca3d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,8 +50,10 @@ dependencies = [ # to this exact mlx version — bumping the pin requires rebuilding them # (OMLX_WITH_CUSTOM_KERNEL=1 setup.py build_ext). NEVER resolve stock # PyPI mlx 0.32.0 or 0.32.2 into the bundled CPython 3.11 environment. - # Install the matched cp311 frontend+backend wheels from - # packaging/_wheels/guarded (see docs/packaging-guarded-mlx.md). + # Install the matched cp311 frontend+backend wheels first with --no-deps + # (ops/install_mlx_variant.sh) from packaging/_wheels/guarded so a fresh + # resolve cannot replace them with stock 0.32.0 or 0.32.2. See + # docs/packaging-guarded-mlx.md. "mlx==0.32.1.dev20260825+26421e953", "mlx-metal==0.32.1.dev20260825+26421e953", # mlx-lm from commit (ab1806e, v0.31.3) - transformers 5.13+ @@ -284,9 +286,11 @@ include = ["omlx*"] "omlx.custom_kernels.bonsai" = ["*.metallib", "*.dylib", "*.so"] [tool.uv] -# Guarded mlx/mlx-metal are local cp311 wheels, not PyPI. find-links keeps -# a fresh resolve from landing stock 0.32.0 or 0.32.2. mlx-lm stays git-pinned -# so mlx-audio's transitive mlx-lm==0.31.1 pin is overridden. +# Fusion 3a82bd27 overrode stock 0.32.0, which made a fresh resolve replace a +# custom wheel with PyPI. Override the exact guarded pair instead, and +# install it first via ops/install_mlx_variant.sh (--no-deps --no-index). +# find-links keeps uv from landing stock 0.32.0 or 0.32.2. mlx-lm stays +# git-pinned so mlx-audio's transitive mlx-lm==0.31.1 pin is overridden. find-links = ["packaging/_wheels/guarded"] override-dependencies = [ "mlx==0.32.1.dev20260825+26421e953", diff --git a/tests/test_packaging_guarded_mlx_pin.py b/tests/test_packaging_guarded_mlx_pin.py index 57c7f8e02c..703d7bd17e 100644 --- a/tests/test_packaging_guarded_mlx_pin.py +++ b/tests/test_packaging_guarded_mlx_pin.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 -"""The Fusion DMG must pin guarded MLX and must not admit 0.32.2.""" +"""The Fusion DMG must pin guarded MLX and must not admit 0.32.0 or 0.32.2.""" +import re +import subprocess from pathlib import Path import pytest @@ -19,9 +21,12 @@ def test_pyproject_pins_exact_guarded_mlx_not_open_range(): assert PIN in text assert f"mlx=={PIN}" in text assert f"mlx-metal=={PIN}" in text - overrides = text.split("[tool.uv]", 1)[1] - assert f"mlx=={PIN}" in overrides - assert "mlx==0.32.0" not in overrides + overrides = text.split("[tool.uv]", 1)[1].split("[tool.", 1)[0] + quoted = re.findall(r'"([^"]+)"', overrides) + assert f"mlx=={PIN}" in quoted + assert f"mlx-metal=={PIN}" in quoted + assert not any(q in {"mlx==0.32.0", "mlx==0.32.2"} for q in quoted) + assert "ops/install_mlx_variant.sh" in text def test_packaging_build_never_queries_pypi_for_mlx(): @@ -29,7 +34,7 @@ def test_packaging_build_never_queries_pypi_for_mlx(): assert "pypi.org/pypi/mlx" not in text assert "GUARDED_MLX_VERSION" in text assert PIN in text - assert "0.32.2" in text # forbidden-version guard + assert 'FORBIDDEN_MLX_VERSIONS = ("0.32.0", "0.32.2")' in text def test_dmg_name_is_macos26_only(): @@ -75,3 +80,35 @@ def test_live_studio_coordinator_is_off_limits_for_metal_compile(): "scripts/build_guarded_mlx_cp311_wheels.sh" ) assert "github-hosted" in _read(".github/workflows/release-macos-dmg.yml") + + +def test_install_mlx_variant_script_closes_fusion_hole(): + path = ROOT / "ops" / "install_mlx_variant.sh" + assert path.is_file() + text = path.read_text() + assert PIN in text + assert "--no-deps" in text + assert "--no-index" in text + assert "0.32.0" in text + assert "0.32.2" in text + assert "ssh " not in text + assert "does NOT compile Metal" in text + subprocess.run(["bash", "-n", str(path)], check=True) + # Wheels are not committed; --check must fail closed instead of PyPI. + result = subprocess.run( + ["bash", str(path), "--check"], + cwd=ROOT, + capture_output=True, + text=True, + ) + assert result.returncode != 0 + err = result.stderr.lower() + assert "0.32.0" in err or "no .whl" in err or "missing" in err + stock = subprocess.run( + ["bash", str(path), "stock"], + cwd=ROOT, + capture_output=True, + text=True, + ) + assert stock.returncode != 0 + assert "0.32.0" in stock.stderr or "stock" in stock.stderr.lower() From 0ef97400c3c139966110bcd759657a5ec01cd778 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 25 Aug 2026 14:49:55 +0000 Subject: [PATCH 4/4] fix: reject macos15-26 DMG on 26 and bypass PyPI mlx swap Path B updater must pick oMLX-*-macos26-arm64.dmg, never a 15-26 range asset. --macos-target other than 26.0 is refused so Fusion's pip-download swap cannot land stock mlx from PyPI. Co-authored-by: Jonathan Spangler --- .../Sources/Updater/ReleasesChecker.swift | 30 +++++++++++---- .../oMLXTests/ReleasesCheckerTests.swift | 6 +++ packaging/build.py | 23 +++++++++-- tests/test_packaging_guarded_mlx_pin.py | 38 +++++++++++++++++++ 4 files changed, 86 insertions(+), 11 deletions(-) diff --git a/apps/omlx-mac/Sources/Updater/ReleasesChecker.swift b/apps/omlx-mac/Sources/Updater/ReleasesChecker.swift index 7512226681..d231154143 100644 --- a/apps/omlx-mac/Sources/Updater/ReleasesChecker.swift +++ b/apps/omlx-mac/Sources/Updater/ReleasesChecker.swift @@ -4,8 +4,8 @@ // single source of truth for distribution. No appcast XML, no EdDSA key // management on the maintainer side. Pulls a page of releases, picks the // latest stable PEP 440 tag, and selects the DMG asset whose filename -// embeds the current macOS major version (e.g. `-macos26-` for this -// Fusion Path B beta). Historical `-macos15-` assets remain parseable. +// embeds the current macOS major version. This Fusion Path B beta ships +// `-macos26-` only; a `macos15-26` range asset must not be chosen on 26. // // Channel handling: Stable accepts final tags only, Release Candidate also // accepts rc tags, and Dev accepts dev/pre-release tags. GitHub's @@ -311,9 +311,9 @@ enum ReleasesChecker { } /// Pick the DMG asset whose filename embeds the current macOS major - /// version (e.g. `-macos15-` / `-macos26-` / `-macos15_`). Version - /// ranges such as `-macos26-27.` are accepted after exact matches. - /// Falls back to the single DMG when there's only one. + /// version (e.g. `-macos26-` / `-macos15-`). Exact majors win. + /// Forward ranges such as `-macos26-27.` are accepted after exact + /// matches. A `macos15-26` span is not a macOS 26 asset for this beta. static func findMatchingDMG(assets: [GitHubRelease.Asset]) -> GitHubRelease.Asset? { findMatchingDMG(assets: assets, macOSMajor: currentMacOSMajor()) } @@ -336,11 +336,27 @@ enum ReleasesChecker { return exact } if let ranged = candidates.first(where: { candidate in - candidate.ranges.contains { $0.contains(macOSMajor) } + candidate.ranges.contains { range in + guard range.contains(macOSMajor) else { return false } + // Path B: never treat macos15-26 as macOS 26. + if macOSMajor == 26 && range.lowerBound < 26 { + return false + } + return true + } })?.asset { return ranged } - return dmgs.count == 1 ? dmgs[0] : nil + if dmgs.count == 1 { + let only = dmgs[0] + let ranges = macOSMajorRanges(in: only.name) + if macOSMajor == 26, + ranges.contains(where: { $0.lowerBound < 26 && $0.contains(26) }) { + return nil + } + return only + } + return nil } private static func macOSMajorRanges(in assetName: String) -> [ClosedRange] { diff --git a/apps/omlx-mac/Tests/oMLXTests/ReleasesCheckerTests.swift b/apps/omlx-mac/Tests/oMLXTests/ReleasesCheckerTests.swift index c229be8a6c..8f20311d3c 100644 --- a/apps/omlx-mac/Tests/oMLXTests/ReleasesCheckerTests.swift +++ b/apps/omlx-mac/Tests/oMLXTests/ReleasesCheckerTests.swift @@ -145,6 +145,12 @@ final class ReleasesCheckerTests: XCTestCase { macOSMajor: 15 ) ) + XCTAssertNil( + ReleasesChecker.findMatchingDMG( + assets: [asset(mislabeledRange)], + macOSMajor: 26 + ) + ) } func testFindMatchingDMGPrefersExactAssetOverRangeAsset() { diff --git a/packaging/build.py b/packaging/build.py index 76dc30521e..b0398f61bc 100644 --- a/packaging/build.py +++ b/packaging/build.py @@ -242,10 +242,18 @@ def swap_platform_wheels( """Install the guarded local mlx/mlx-metal wheels into the export. Never downloads from PyPI. Path B of this beta is macOS 26-only; the - local wheels must already be a macosx_26_* cp311 pair. + local wheels must already be a macosx_26_* cp311 pair. Passing + --macos-target 15.0 (or any non-26.0 tag) is refused so this cannot + fall back to Fusion's old pip-download swap. """ import zipfile + if macos_target != "26.0": + _guarded_wheel_error( + f"--macos-target {macos_target} is not Path B 26.0; " + "refusing PyPI platform-wheel swap" + ) + if python_version != "3.11": _guarded_wheel_error( f"bundled runtime is CPython 3.11, not {python_version}" @@ -1230,9 +1238,10 @@ def main(): "staged omlx package directory and exit. " "`build.sh` uses this after copying Resources/omlx.") parser.add_argument("--macos-target", - help="Install the guarded local mlx/mlx-metal wheels " - "for this macOS tag (Path B: 26.0). Never " - "downloads mlx from PyPI.") + help="Path B: only 26.0. Bypasses the old PyPI " + "mlx/mlx-metal platform-wheel swap and installs " + "guarded 0.32.1.dev20260825+26421e953 cp311 " + "wheels from packaging/_wheels/guarded.") args = parser.parse_args() if args.print_fingerprint: @@ -1255,6 +1264,12 @@ def main(): build_venvstacks() if args.macos_target: + if args.macos_target != "26.0": + _guarded_wheel_error( + f"--macos-target {args.macos_target} is not Path B 26.0; " + "PyPI mlx/mlx-metal swap is disabled for this beta" + ) + # Bypass Fusion's pip-download swap. Stage/install local guarded wheels. swap_platform_wheels(EXPORT_DIR, args.macos_target) _write_export_fingerprint() print("\n" + "=" * 50) diff --git a/tests/test_packaging_guarded_mlx_pin.py b/tests/test_packaging_guarded_mlx_pin.py index 703d7bd17e..009d46095f 100644 --- a/tests/test_packaging_guarded_mlx_pin.py +++ b/tests/test_packaging_guarded_mlx_pin.py @@ -112,3 +112,41 @@ def test_install_mlx_variant_script_closes_fusion_hole(): ) assert stock.returncode != 0 assert "0.32.0" in stock.stderr or "stock" in stock.stderr.lower() + + +def test_version_is_064b1(): + assert _read("omlx/_version.py").strip() == '__version__ = "0.6.4b1"' + + +def test_deployment_targets_are_macos26_nax_stays_262(): + setup = _read("setup.py") + assert 'DEFAULT_CUSTOM_KERNEL_DEPLOYMENT_TARGET = "26.0"' in setup + assert 'DEFAULT_CUSTOM_KERNEL_DEPLOYMENT_TARGET = "15.0"' not in setup + build = _read("apps/omlx-mac/Scripts/build.sh") + assert "OMLX_CUSTOM_KERNEL_DEPLOYMENT_TARGET:-${MACOSX_DEPLOYMENT_TARGET:-26.0}" in build + assert "--minimum-deployment-target 26.0" in build + pbx = _read("apps/omlx-mac/oMLX.xcodeproj/project.pbxproj") + assert "MACOSX_DEPLOYMENT_TARGET = 26.0;" in pbx + assert "MACOSX_DEPLOYMENT_TARGET = 15.0;" not in pbx + nax = _read("omlx/custom_kernels/qwen35_prefill/csrc/CMakeLists.txt") + assert "-mmacosx-version-min=26.2" in nax + + +def test_build_sh_includes_bonsai_and_decode_fast(): + text = _read("apps/omlx-mac/Scripts/build.sh") + assert "omlx/custom_kernels/decode_fast" in text + assert "omlx/custom_kernels/bonsai" in text + assert "omlx_decode_fast_kernels.metallib" in text + assert "omlx_bonsai_kernels.metallib" in text + + +def test_updater_requires_exact_macos26_not_15_26_range(): + src = _read("apps/omlx-mac/Sources/Updater/ReleasesChecker.swift") + tests = _read("apps/omlx-mac/Tests/oMLXTests/ReleasesCheckerTests.swift") + assert "macos15-26" in tests + assert 'tahoeOnly = "oMLX-0.6.4b1-macos26-arm64.dmg"' in tests + assert "macOSMajor == 26 && range.lowerBound < 26" in src + assert "pypi.org/pypi/mlx" not in _read("packaging/build.py") + build = _read("packaging/build.py") + assert "pip\", \"download\"" not in build.split("def swap_platform_wheels", 1)[1].split("def _parse_git_requirements", 1)[0] + assert "https://pypi.org/pypi/mlx/json" not in build