Skip to content

Commit 16440ff

Browse files
feat(mac-bridge): pin workload interpreter (Layer B) + import self-check gate (Layer C)
Reboots can repoint the runner's default python3 to one without mlx_lm, which broke every full-engine preset with a deep ModuleNotFoundError. Make the workload interpreter explicit and verified: - inference_engine/bridge/runner_python.py (NEW, pure + 100% unit-tested): workload_python_candidates (pin KAKEYA_MAC_PYTHON -> venvs -> PATH), resolve_workload_python (first interpreter that can import mlx_lm; else fallback), preset_requires_gate (mlx-/k3- engine presets, minus env-probe/ upgrade), substitute_python, gate_error_message. - scripts/mac_bridge/run_preset.py: resolve the pinned interpreter, rewrite bare python3 argv0 to it, export KAKEYA_MAC_PYTHON to the subprocess, and FAIL FAST (exit 90 + ::error::) when a gated preset has no mlx_lm-capable interpreter. - scripts/run_kakeya_mac.sh: honor KAKEYA_MAC_PYTHON; preflight asserts mlx+mlx_lm. CI enforcement: the resolution/gate logic lives in the unit-tested, 100%-coverage library (runner_python.py), so every PR exercises it on the Linux gate. See docs/skills/pin-selfhosted-runner-python-env-skill.md. Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
1 parent 51ff901 commit 16440ff

4 files changed

Lines changed: 283 additions & 5 deletions

File tree

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
"""Pin the Mac-bridge workload interpreter (Layer B) + import self-check (Layer C).
2+
3+
A self-hosted runner's default ``python3`` can silently change across reboots /
4+
OS upgrades (observed 2026-06-18: it flipped to a Python 3.14 without ``mlx_lm``,
5+
breaking every full-engine preset with a deep ``ModuleNotFoundError``). The
6+
mac-bridge executor used to invoke a bare ``python3`` for the workload, so it
7+
inherited whatever interpreter happened to be first on ``PATH``.
8+
9+
This module makes the workload interpreter **explicit and verified**:
10+
11+
* **Layer B — resolution.** Build an ordered candidate list (a pinned
12+
``KAKEYA_MAC_PYTHON``, common venv paths, then ``PATH`` pythons) and pick the
13+
first one that can import the gate module (``mlx_lm``); fall back to the first
14+
existing candidate otherwise.
15+
* **Layer C — gate.** For presets whose workload needs ``mlx_lm`` (the ``mlx-`` /
16+
``k3-`` engine families, minus the env-probe / upgrade tools that exist to
17+
diagnose/repair the env), fail fast with a clear message instead of a deep
18+
import error when no capable interpreter exists.
19+
20+
All functions here are pure / dependency-injected so they are unit-tested on the
21+
Linux gate (the CLI ``scripts/mac_bridge/run_preset.py`` is a thin caller). See
22+
``docs/skills/pin-selfhosted-runner-python-env-skill.md``.
23+
"""
24+
25+
from __future__ import annotations
26+
27+
import os
28+
import shutil
29+
from dataclasses import dataclass
30+
from typing import Callable, List, Mapping, Optional, Sequence
31+
32+
# The single module whose absence broke the runner; importing it implies the
33+
# full MLX-LM stack is wired for the interpreter.
34+
GATE_MODULE = "mlx_lm"
35+
36+
# ``mlx-``/``k3-`` presets that must NOT be import-gated: these exist precisely
37+
# to probe or repair the environment, so they must run even when mlx_lm is gone.
38+
_IMPORT_GATE_SKIP = frozenset({"mlx-env-probe", "mlx-upgrade"})
39+
40+
SKILL_DOC = "docs/skills/pin-selfhosted-runner-python-env-skill.md"
41+
42+
43+
def workload_python_candidates(
44+
environ: Mapping[str, str],
45+
*,
46+
which: Callable[[str], Optional[str]] = shutil.which,
47+
expanduser: Callable[[str], str] = os.path.expanduser,
48+
) -> List[str]:
49+
"""Ordered, de-duplicated interpreter candidates for the heavy workload.
50+
51+
Priority: the explicit pin (``KAKEYA_MAC_PYTHON``), then conventional venv
52+
locations, then ``PATH`` pythons (a pinned minor version before the bare
53+
``python3`` that a reboot may have repointed)."""
54+
raw = [
55+
environ.get("KAKEYA_MAC_PYTHON"),
56+
expanduser("~/kakeya-venv/bin/python"),
57+
expanduser("~/.venv/bin/python"),
58+
which("python3.13"),
59+
which("python3"),
60+
]
61+
out: List[str] = []
62+
for c in raw:
63+
if c and c not in out:
64+
out.append(c)
65+
return out
66+
67+
68+
@dataclass(frozen=True)
69+
class ResolvedPython:
70+
"""The interpreter chosen for the workload."""
71+
72+
path: str
73+
gate_module_ok: bool # whether ``path`` can import GATE_MODULE
74+
from_pin: bool # whether it came from ``KAKEYA_MAC_PYTHON``
75+
76+
77+
def resolve_workload_python(
78+
candidates: Sequence[str],
79+
can_import: Callable[[str], bool],
80+
*,
81+
pinned: Optional[str] = None,
82+
) -> Optional[ResolvedPython]:
83+
"""Pick the first candidate that can import :data:`GATE_MODULE`; otherwise
84+
the first candidate (a fallback whose ``gate_module_ok`` is ``False``).
85+
Returns ``None`` only when there are no candidates at all."""
86+
first: Optional[str] = None
87+
for c in candidates:
88+
if first is None:
89+
first = c
90+
if can_import(c):
91+
return ResolvedPython(c, True, c == pinned)
92+
if first is None:
93+
return None
94+
return ResolvedPython(first, False, first == pinned)
95+
96+
97+
def preset_requires_gate(preset_name: str) -> bool:
98+
"""True iff a preset's workload needs :data:`GATE_MODULE` (so a missing
99+
import must fail fast). The ``mlx-`` / ``k3-`` engine presets do; the
100+
env-probe and upgrade tools (which diagnose/repair the env) are exempt."""
101+
if preset_name in _IMPORT_GATE_SKIP:
102+
return False
103+
return preset_name.startswith(("mlx-", "k3-"))
104+
105+
106+
def substitute_python(argv: Sequence[str], pybin: str) -> List[str]:
107+
"""Rewrite a leading bare ``python3`` to the resolved interpreter ``pybin``.
108+
Non-``python3`` argv (e.g. ``bash run_kakeya_mac.sh``, which reads
109+
``KAKEYA_MAC_PYTHON`` itself) is returned unchanged."""
110+
a = list(argv)
111+
if a and a[0] == "python3":
112+
a[0] = pybin
113+
return a
114+
115+
116+
def gate_error_message(preset_name: str, pybin: str) -> str:
117+
"""The fail-fast message when a gated preset has no mlx_lm-capable python."""
118+
return (
119+
f"runner python '{pybin}' cannot import {GATE_MODULE!r}, which preset "
120+
f"'{preset_name}' requires. The runner's default python likely changed "
121+
f"(e.g. after a reboot). Pin the venv via KAKEYA_MAC_PYTHON or the runner "
122+
f"agent PATH and reinstall the ML stack — see {SKILL_DOC}."
123+
)

scripts/mac_bridge/run_preset.py

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,29 @@
3232
build_commands,
3333
parse_manifest_text,
3434
)
35+
from inference_engine.bridge.runner_python import (
36+
GATE_MODULE,
37+
gate_error_message,
38+
preset_requires_gate,
39+
resolve_workload_python,
40+
substitute_python,
41+
workload_python_candidates,
42+
)
3543

3644
LOG_DIR = Path(".mac-bridge/logs")
3745

3846

47+
def _can_import_gate_module(pybin: str) -> bool:
48+
"""True iff interpreter ``pybin`` can import the gate module (mlx_lm)."""
49+
try:
50+
return subprocess.run(
51+
[pybin, "-c", f"import {GATE_MODULE}"],
52+
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
53+
).returncode == 0
54+
except OSError:
55+
return False
56+
57+
3958
def main() -> int:
4059
ap = argparse.ArgumentParser(description=__doc__)
4160
ap.add_argument("--manifest", default=".mac-bridge/request.json")
@@ -59,20 +78,43 @@ def main() -> int:
5978
print(json.dumps(argv))
6079
return 0
6180

81+
# Layer B — resolve a PINNED workload interpreter instead of trusting the
82+
# bare ``python3`` on PATH (which a reboot can repoint to a python without
83+
# mlx_lm). Layer C — gate: mlx-/k3- engine presets fail fast with a clear
84+
# message when no mlx_lm-capable interpreter exists.
85+
pinned = os.environ.get("KAKEYA_MAC_PYTHON")
86+
candidates = workload_python_candidates(os.environ)
87+
resolved = resolve_workload_python(
88+
candidates, _can_import_gate_module, pinned=pinned)
89+
pybin = resolved.path if resolved else "python3"
90+
gate_ok = bool(resolved and resolved.gate_module_ok)
91+
print(f"[mac-bridge] workload python={pybin} {GATE_MODULE}_ok={gate_ok} "
92+
f"pinned={pinned!r} candidates={candidates}", file=sys.stderr)
93+
if preset_requires_gate(request.preset.name) and not gate_ok:
94+
print(f"::error::{gate_error_message(request.preset.name, pybin)}",
95+
file=sys.stderr)
96+
return 90
97+
6298
LOG_DIR.mkdir(parents=True, exist_ok=True)
6399
summary = {
64100
"preset": request.preset.name,
65101
"params": dict(request.params),
66102
"nonce": request.nonce,
67103
"commands": [],
68104
}
105+
# Make the resolved interpreter authoritative for BOTH bare-``python3``
106+
# commands (rewritten here) and the launcher (which reads KAKEYA_MAC_PYTHON).
107+
sub_env = dict(os.environ)
108+
sub_env["KAKEYA_MAC_PYTHON"] = pybin
69109
rc = 0
70110
for idx, argv in enumerate(commands):
111+
argv = substitute_python(argv, pybin)
71112
log_path = LOG_DIR / f"{request.preset.name}-{idx}.log"
72113
print(f"[mac-bridge] exec[{idx}]: {argv}", file=sys.stderr)
73114
t0 = time.perf_counter()
74115
with log_path.open("wb") as log:
75-
proc = subprocess.run(argv, stdout=log, stderr=subprocess.STDOUT)
116+
proc = subprocess.run(argv, stdout=log, stderr=subprocess.STDOUT,
117+
env=sub_env)
76118
elapsed = time.perf_counter() - t0
77119
summary["commands"].append({
78120
"argv": argv,

scripts/run_kakeya_mac.sh

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,11 @@ done
5555

5656
log() { echo "[run-kakeya-mac] $*" >&2; }
5757

58+
# Pinned interpreter (Layer B): prefer KAKEYA_MAC_PYTHON (the venv python with
59+
# mlx_lm/torch/transformers) over a bare python3 that a host reboot may have
60+
# repointed. See docs/skills/pin-selfhosted-runner-python-env-skill.md.
61+
PYBIN="${KAKEYA_MAC_PYTHON:-python3}"
62+
5863
# ---- argv for the full-engine harness chat ----
5964
args=(
6065
--verifier-path "$VERIFIER"
@@ -80,17 +85,17 @@ log "drafter : $DRAFTER"
8085
log "f_theta : $FTHETA"
8186
log "params : sink=$SINK window=$WINDOW block=$BLOCK max_new=$MAX_NEW"
8287

83-
cmd=( python3 scripts/research/k3_integrated_niah_eval_mac.py "${args[@]}" "${EXTRA[@]}" )
88+
cmd=( "$PYBIN" scripts/research/k3_integrated_niah_eval_mac.py "${args[@]}" "${EXTRA[@]}" )
8489

8590
if [[ "$DRY_RUN" == "1" ]]; then
8691
echo "PYTHONPATH=.:sdks/python ${cmd[*]}"
8792
exit 0
8893
fi
8994

9095
# ---- preflight (Apple Silicon + MLX + model) ----
91-
command -v python3 >/dev/null || { log "python3 not found"; exit 1; }
92-
python3 -c "import mlx.core" 2>/dev/null \
93-
|| { log "MLX not importable — this needs Apple Silicon + 'pip install mlx mlx-lm'"; exit 2; }
96+
command -v "$PYBIN" >/dev/null 2>&1 || { log "interpreter not found: $PYBIN (set KAKEYA_MAC_PYTHON)"; exit 1; }
97+
"$PYBIN" -c "import mlx.core, mlx_lm" 2>/dev/null \
98+
|| { log "mlx/mlx_lm not importable by $PYBIN Apple Silicon + a venv with 'mlx mlx-lm'; set KAKEYA_MAC_PYTHON. See docs/skills/pin-selfhosted-runner-python-env-skill.md"; exit 2; }
9499
[[ -d "$VERIFIER" ]] \
95100
|| { log "verifier model dir not found: $VERIFIER (set KAKEYA_MAC_VERIFIER_PATH)"; exit 3; }
96101
if [[ "$FAST" != "1" && ! -e "$FTHETA" ]]; then
Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
"""Unit tests for the mac-bridge workload interpreter pinning (Layers B/C).
2+
3+
Pure / dependency-injected logic from ``inference_engine.bridge.runner_python``;
4+
the CLI ``scripts/mac_bridge/run_preset.py`` is a thin caller (coverage-exempt).
5+
"""
6+
7+
from __future__ import annotations
8+
9+
from inference_engine.bridge.runner_python import (
10+
GATE_MODULE,
11+
SKILL_DOC,
12+
ResolvedPython,
13+
gate_error_message,
14+
preset_requires_gate,
15+
resolve_workload_python,
16+
substitute_python,
17+
workload_python_candidates,
18+
)
19+
20+
21+
# --------------------------------------------------------------------------- #
22+
# workload_python_candidates
23+
# --------------------------------------------------------------------------- #
24+
def test_candidates_prioritise_pin_then_venvs_then_path():
25+
env = {"KAKEYA_MAC_PYTHON": "/pin/bin/python"}
26+
which = {"python3.13": "/usr/bin/python3.13", "python3": "/usr/bin/python3"}.get
27+
cands = workload_python_candidates(
28+
env, which=which, expanduser=lambda p: p.replace("~", "/home/me"))
29+
assert cands == [
30+
"/pin/bin/python",
31+
"/home/me/kakeya-venv/bin/python",
32+
"/home/me/.venv/bin/python",
33+
"/usr/bin/python3.13",
34+
"/usr/bin/python3",
35+
]
36+
37+
38+
def test_candidates_drop_empty_and_dedupe():
39+
# no pin, python3.13 missing, and python3 == an expanded venv path (dedupe).
40+
env: dict = {}
41+
which = {"python3.13": None, "python3": "/home/me/.venv/bin/python"}.get
42+
cands = workload_python_candidates(
43+
env, which=which, expanduser=lambda p: p.replace("~", "/home/me"))
44+
assert cands == [
45+
"/home/me/kakeya-venv/bin/python",
46+
"/home/me/.venv/bin/python",
47+
]
48+
assert None not in cands
49+
50+
51+
# --------------------------------------------------------------------------- #
52+
# resolve_workload_python
53+
# --------------------------------------------------------------------------- #
54+
def test_resolve_picks_first_importable():
55+
cands = ["/a/py", "/b/py", "/c/py"]
56+
r = resolve_workload_python(cands, lambda p: p == "/b/py", pinned="/a/py")
57+
assert r == ResolvedPython(path="/b/py", gate_module_ok=True, from_pin=False)
58+
59+
60+
def test_resolve_marks_from_pin_when_pinned_is_importable():
61+
r = resolve_workload_python(["/pin/py", "/x/py"], lambda p: True,
62+
pinned="/pin/py")
63+
assert r.path == "/pin/py" and r.gate_module_ok is True and r.from_pin is True
64+
65+
66+
def test_resolve_falls_back_to_first_when_none_importable():
67+
r = resolve_workload_python(["/a/py", "/b/py"], lambda p: False,
68+
pinned="/a/py")
69+
assert r == ResolvedPython(path="/a/py", gate_module_ok=False, from_pin=True)
70+
71+
72+
def test_resolve_returns_none_without_candidates():
73+
assert resolve_workload_python([], lambda p: True) is None
74+
75+
76+
# --------------------------------------------------------------------------- #
77+
# preset_requires_gate
78+
# --------------------------------------------------------------------------- #
79+
def test_gate_required_for_mlx_and_k3_engine_presets():
80+
assert preset_requires_gate("mlx-kakeya-launcher-full") is True
81+
assert preset_requires_gate("k3-step2-fused") is True
82+
83+
84+
def test_gate_skips_diagnostic_and_installer_and_non_engine():
85+
assert preset_requires_gate("mlx-env-probe") is False # diagnostic
86+
assert preset_requires_gate("mlx-upgrade") is False # installer
87+
assert preset_requires_gate("integration-tests") is False
88+
assert preset_requires_gate("agent-capacity-stress") is False
89+
90+
91+
# --------------------------------------------------------------------------- #
92+
# substitute_python / gate_error_message
93+
# --------------------------------------------------------------------------- #
94+
def test_substitute_rewrites_only_leading_bare_python3():
95+
assert substitute_python(["python3", "a.py", "--x"], "/v/py") == [
96+
"/v/py", "a.py", "--x"]
97+
# non-python3 argv0 (e.g. the launcher) is untouched.
98+
assert substitute_python(["bash", "run.sh"], "/v/py") == ["bash", "run.sh"]
99+
# empty argv is safe.
100+
assert substitute_python([], "/v/py") == []
101+
102+
103+
def test_gate_error_message_names_module_preset_and_skill():
104+
msg = gate_error_message("mlx-kakeya-launcher-full", "/usr/bin/python3")
105+
assert GATE_MODULE in msg
106+
assert "mlx-kakeya-launcher-full" in msg
107+
assert "/usr/bin/python3" in msg
108+
assert SKILL_DOC in msg

0 commit comments

Comments
 (0)