feat(vllm-cpp): enable and vendor the MLX GEMM provider on darwin/metal

backend/go/vllm-cpp/Makefile

@@ -11,7 +11,30 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e
 
 # vllm.cpp version
 VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp
-VLLM_CPP_VERSION?=a42b8187caff02c570c28e19e4dc2b1d7f55ed14
+VLLM_CPP_VERSION?=9d1fad3cde0acb95eb0bb0a1025f40a0eb614147
+
+# MLX GEMM provider (darwin/metal only; see the metal branch below for why).
+# Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun
+# metal`, i.e. a full Xcode the macOS runners do not have, while the wheel ships
+# include/, lib/libmlx.dylib and the compiled mlx.metallib ready to link.
+#
+# DEFAULT ON, but ONLY because VLLM_CPP_VERSION above is pinned at or past
+# vllm.cpp 89c46aeb, which SHAPE-GATES the provider to prefill. The ordering is
+# load-bearing, not incidental:
+#
+#   pin >= 89c46aeb, MLX on   -> 99.1% of MLX-LM  (gated: prefill only)
+#   pin <  89c46aeb, MLX on   -> ~51%             (ungated: it also takes decode)
+#
+# MLX's steel GEMM wins prefill (537 ms TTFT against 602) and loses decode badly,
+# because the provider pays an mx::eval sync plus an output memcpy per call and
+# decode makes ~112 calls per TOKEN. Ungated it does both; gated it does only the
+# good half. So if this pin is ever moved BACKWARDS, this default must go with it.
+VLLM_CPP_MLX?=on
+MLX_VERSION?=0.29.4
+MLX_VENV?=$(abspath ./mlx-venv)
+# Resolved lazily (recursive `=`, not `:=`): the glob only matches once the venv
+# target has run, and the interpreter version in the path varies per runner.
+MLX_ROOT=$(shell echo $(MLX_VENV)/lib/python*/site-packages/mlx)
 
 # The backend consumes only the stable C ABI (libvllm + include/vllm.h), so the
 # server, examples and tests of the engine are never built here.
@@ -49,6 +72,23 @@ else ifeq ($(BUILD_TYPE),vulkan)
 	CMAKE_ARGS+=-DVLLM_CPP_VULKAN=ON -DVLLM_CPP_CUDA=OFF
 else ifeq ($(BUILD_TYPE),metal)
 	CMAKE_ARGS+=-DVLLM_CPP_METAL=ON
+	# The optional MLX GEMM provider. vllm.cpp keeps it OFF by default because it
+	# is a ~19 MB libmlx.dylib plus a ~105 MB mlx.metallib, and upstream's
+	# position is that it must earn that cost by measurement. It does, on the
+	# only hardware this build targets: measured on an Apple M4 against the
+	# native MSL GEMM in the SAME binary (arms toggled by
+	# VT_OP_PROVIDER_DISABLE=mlx), Qwen3-1.7B-bf16 p=512 g=128, it is 1.5x to
+	# 2.2x aggregate throughput and 2x to 3x faster TTFT, at equal peak memory
+	# and bit-identical output on every parity shape. See vllm.cpp
+	# docs/BENCHMARKS.md "MLX GEMM provider A/B on Apple M4".
+	#
+	# MLX delegates the dense GEMM ONLY: kPagedAttention stays vllm.cpp's own
+	# kernel, because MLX has no paged-KV primitive at all.
+	#
+	# Set VLLM_CPP_MLX=off for a Metal build without it (smaller image, slower).
+	ifeq ($(VLLM_CPP_MLX),on)
+		MLX_ENABLED=1
+	endif
 else
 	CMAKE_ARGS+=-DVLLM_CPP_CUDA=OFF
 endif
@@ -68,23 +108,48 @@ sources/vllm.cpp:
 	git fetch --depth 1 origin $(VLLM_CPP_VERSION) && \
 	git checkout FETCH_HEAD
 
-$(LIB): sources/vllm.cpp
+ifeq ($(MLX_ENABLED),1)
+# A stamp FILE, not a phony target: a phony prerequisite is always "newer" than
+# $(LIB) and would re-link libvllm on every invocation. Keyed on the version so
+# a MLX_VERSION bump reinstalls instead of silently reusing the old wheel.
+MLX_STAMP=$(MLX_VENV)/.mlx-$(MLX_VERSION).stamp
+MLX_CMAKE_ARGS=-DVLLM_CPP_MLX=ON -DMLX_ROOT=$(MLX_ROOT)
+
+$(MLX_STAMP):
+	@if [ ! -x "$(MLX_VENV)/bin/pip" ]; then \
+		python3 -m venv "$(MLX_VENV)" || { echo "vllm-cpp: python3 with venv is required to build the MLX provider; pass VLLM_CPP_MLX=off to build Metal without it" >&2; exit 1; }; \
+	fi
+	"$(MLX_VENV)"/bin/pip install --quiet --disable-pip-version-check "mlx==$(MLX_VERSION)"
+	@# Resolved in the SHELL, not by $(MLX_ROOT): make expands a whole recipe
+	@# before running its first line, so the glob would still be unmatched here.
+	@# Every later use (the cmake args, package.sh) expands after this target has
+	@# completed, where $(MLX_ROOT) does resolve.
+	@root=$$(echo "$(MLX_VENV)"/lib/python*/site-packages/mlx); \
+	test -f "$$root/lib/libmlx.dylib" -a -f "$$root/include/mlx/array.h" || \
+		{ echo "vllm-cpp: mlx==$(MLX_VERSION) did not provide lib/libmlx.dylib + include/mlx/array.h under $$root" >&2; exit 1; }
+	touch $@
+else
+MLX_STAMP=
+MLX_CMAKE_ARGS=
+endif
+
+$(LIB): sources/vllm.cpp $(MLX_STAMP)
 	mkdir -p build && \
 	cd build && \
-	cmake ../sources/vllm.cpp $(CMAKE_ARGS) && \
+	cmake ../sources/vllm.cpp $(CMAKE_ARGS) $(MLX_CMAKE_ARGS) && \
 	cmake --build . --config Release -j$(JOBS) --target vllm_shared
 	cp -fL build/$(LIB) ./$(LIB)
 
 vllm-cpp: main.go govllmcpp.go backend.go options.go $(LIB)
 	CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o vllm-cpp ./
 
 package: vllm-cpp
-	bash package.sh
+	MLX_ROOT="$(MLX_ROOT)" bash package.sh
 
 build: package
 
 clean: purge
-	rm -rf libvllm.so libvllm.dylib package sources/vllm.cpp vllm-cpp
+	rm -rf libvllm.so libvllm.dylib package sources/vllm.cpp vllm-cpp "$(MLX_VENV)"
 
 purge:
 	rm -rf build

backend/go/vllm-cpp/README.md

@@ -41,5 +41,50 @@ options:
 - max_num_seqs:16
 ```
 
+## Apple Silicon: the MLX GEMM provider (ON by default, gated to prefill)
+
+`BUILD_TYPE=metal` builds vllm.cpp's MLX provider for the dense GEMM
+(`VLLM_CPP_MLX=on`, the default here). It is on because upstream now SHAPE-GATES
+it to prefill; it was briefly off in this branch's history, and that was correct
+at the time for an ungated provider.
+
+The gate matters more than the flag. MLX's steel GEMM wins prefill but loses
+decode, because the provider pays an `mx::eval` synchronisation plus an output
+memcpy on every call and decode makes ~112 calls *per token*. Measured on an
+Apple M4, Qwen3-1.7B-bf16 warm at p=512 g=128:
+
+| configuration | prefill TTFT | warm throughput |
+|---|--:|--:|
+| MLX **gated to prefill** (pin >= 89c46aeb) | **524.5 ms** | **24.37 tok/s, 97.6% of MLX-LM** |
+| MLX ungated (older pins) | 537 ms | 12.7 tok/s |
+| MLX off | 602 ms | 23.9 tok/s, 95.9% |
+
+Ratios are against an MLX-LM baseline measured INTERLEAVED with ours over four
+ABBA blocks (its spread 0.34%, ours 0.12%). An earlier revision of this file
+claimed 99.1%; that used a two-run MLX-LM baseline containing an outlier and
+overstated us by about 1.5 points.
+
+**`VLLM_CPP_VERSION` and this flag are coupled.** Moving the pin back before
+`89c46aeb` while leaving `VLLM_CPP_MLX=on` would take the middle row — roughly
+half throughput. If you roll the pin back, roll the default back with it.
+
+One caveat: MLX's GEMM is not bit-identical to the native kernel, so an MLX build
+produces a different greedy sequence than a non-MLX one. That is a property of the
+provider, not of the gate, and it predates this packaging. Full disposition in
+vllm.cpp `docs/BENCHMARKS.md`.
+
+Build knobs:
+
+- `VLLM_CPP_MLX=off` builds Metal without the provider: ~124 MB smaller, and
+  96.4% of MLX-LM instead of 99.1%.
+- `MLX_VERSION` pins the wheel (default `0.29.4`). MLX is consumed as the
+  prebuilt pip wheel because building it from source needs `xcrun metal`, i.e. a
+  full Xcode the macOS runners do not have.
+
+Packaging vendors `libmlx.dylib`, `mlx.metallib` and MLX's MIT license into
+`package/lib/`, and rewrites `libvllm.dylib`'s rpath to `@loader_path/lib`
+(re-signing it, since `install_name_tool` invalidates the signature). The
+metallib must stay beside `libmlx.dylib`: MLX looks for it there.
+
 Testing: `make test` runs the unit specs; export `VLLM_CPP_MODEL=<model>` (and
 optionally `VLLM_CPP_LIBRARY=<libvllm path>`) to enable the e2e specs.

backend/go/vllm-cpp/package.sh

@@ -43,6 +43,50 @@ elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then
     cp -arfLv /lib/aarch64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
 elif [ $(uname -s) = "Darwin" ]; then
     echo "Detected Darwin"
+    # Vendor the optional MLX GEMM provider, when libvllm was built against it.
+    # Three facts drive every line below, each verified on an Apple M4 before it
+    # was written:
+    #   1. libvllm.dylib carries an LC_LOAD_DYLIB on @rpath/libmlx.dylib, and its
+    #      build-time LC_RPATH points inside the build venv. That path does not
+    #      exist on a user's machine, so it must become @loader_path/lib.
+    #   2. MLX finds its ~100 MB mlx.metallib beside its OWN dylib, so the two
+    #      files have to land in the same directory or every Metal op dies with
+    #      "Failed to load the default metallib".
+    #   3. install_name_tool invalidates the code signature, and macOS refuses to
+    #      load an arm64 image whose signature does not match, so the patched
+    #      library must be re-signed ad-hoc afterwards.
+    if otool -L "$CURDIR/package/libvllm.dylib" 2>/dev/null | grep -q "libmlx.dylib"; then
+        MLX_LIB_DIR="${MLX_ROOT}/lib"
+        if [ ! -f "$MLX_LIB_DIR/libmlx.dylib" ] || [ ! -f "$MLX_LIB_DIR/mlx.metallib" ]; then
+            echo "Error: libvllm.dylib links libmlx.dylib but $MLX_LIB_DIR is missing libmlx.dylib/mlx.metallib" >&2
+            exit 1
+        fi
+        echo "Vendoring the MLX GEMM provider from $MLX_LIB_DIR"
+        cp -fLv "$MLX_LIB_DIR/libmlx.dylib" "$CURDIR/package/lib/"
+        cp -fLv "$MLX_LIB_DIR/mlx.metallib" "$CURDIR/package/lib/"
+        # MLX is MIT and we redistribute its binaries, so its license ships with
+        # them. mlx-metal is the wheel carrying the dylib and the metallib.
+        MLX_LICENSE=$(ls "${MLX_ROOT}"/../mlx_metal-*.dist-info/licenses/LICENSE 2>/dev/null | head -1)
+        if [ -z "$MLX_LICENSE" ]; then
+            MLX_LICENSE=$(ls "${MLX_ROOT}"/../mlx-*.dist-info/licenses/LICENSE 2>/dev/null | head -1)
+        fi
+        if [ -z "$MLX_LICENSE" ]; then
+            echo "Error: could not find the MLX LICENSE to redistribute alongside libmlx.dylib" >&2
+            exit 1
+        fi
+        cp -fLv "$MLX_LICENSE" "$CURDIR/package/lib/LICENSE.mlx"
+        # Drop every build-tree rpath, then point at the packaged copy.
+        otool -l "$CURDIR/package/libvllm.dylib" | awk '/LC_RPATH/{f=1;next} f&&/ path /{print $2;f=0}' | while read -r rp; do
+            install_name_tool -delete_rpath "$rp" "$CURDIR/package/libvllm.dylib" 2>/dev/null || true
+        done
+        install_name_tool -add_rpath "@loader_path/lib" "$CURDIR/package/libvllm.dylib"
+        codesign -f -s - "$CURDIR/package/libvllm.dylib"
+        # A broken rpath must fail the BUILD, not the user's first inference.
+        if ! otool -l "$CURDIR/package/libvllm.dylib" | grep -q "@loader_path/lib"; then
+            echo "Error: libvllm.dylib did not get the @loader_path/lib rpath" >&2
+            exit 1
+        fi
+    fi
 else
     echo "Error: Could not detect architecture"
     exit 1