From 78f890b581e4cc5c74a1adad245c4c1d16711a07 Mon Sep 17 00:00:00 2001 From: dp-web4 Date: Sat, 24 Oct 2026 00:36:12 -0700 Subject: [PATCH] ear: transcribe out of process (whispercpp); MLX in-process starved the capture with music McNugget, 2026-10-24: the ear never chimed for five days. With music in the room, the MLX backend was decoding in 20 of 20 py-spy samples and the capture reader thread in none: it could not drain ffmpeg's pipe, samples were lost, and an addressed "hey McNugget" reached whisper as fragments ("Okay." x28, dropped as silence). SAGE_LISTEN_BACKEND=whispercpp decodes in whisper-server's own process: the same recorded audio replayed through it woke and heard correctly, and the first live try after the switch chimed and reached the room. Documentation only: the reason, at the two places a seat picks a backend (ear.py, listening.py's MLX branch). No behaviour change. Co-Authored-By: Claude Opus 5.5 --- sage/embodiment/ear.py | 8 ++++++++ sage/embodiment/listening.py | 2 ++ 2 files changed, 10 insertions(+) diff --git a/sage/embodiment/ear.py b/sage/embodiment/ear.py index 3162008f7..c90e0e34f 100644 --- a/sage/embodiment/ear.py +++ b/sage/embodiment/ear.py @@ -13,6 +13,14 @@ the room: "only being directly addressed should wake it" -- run with SAGE_LISTEN=wake and SAGE_WAKE_NAME, so words become heard lines only after the being's name (listening.py). + * TRANSCRIBE OUT OF PROCESS: SAGE_LISTEN_BACKEND=whispercpp. The capture is read by a Python thread, so + a transcriber that decodes in this process competes with it for the GIL. On McNugget (2026-10-24) the + MLX backend, with music in the room, was decoding in 20 of 20 py-spy samples and the reader in none: + samples were lost, "hey McNugget" reached whisper as fragments ("Okay." x28, dropped as silence), and + five days of tries never chimed. whisper-server (sage/embodiment/whispercpp.py) decodes in its own + process; the same audio replayed through it woke and heard correctly, and the first live try chimed. + MLX is fine for a quiet room or a test; a room with music needs the out-of-process backend. + Run as a service: python -m sage.embodiment.ear (SAGE_INSTANCE / SAGE_MEMBER as for presence.py) """ from __future__ import annotations diff --git a/sage/embodiment/listening.py b/sage/embodiment/listening.py index f30c4d3bf..620bc88ff 100644 --- a/sage/embodiment/listening.py +++ b/sage/embodiment/listening.py @@ -294,6 +294,8 @@ def _load(self): if BACKEND == "mlx": # Apple silicon (McNugget, 2026-10-17): the same whisper model through MLX on the GPU. Measured there: # base.en ~0.1 s per short phrase warm. Its segments carry no_speech_prob / avg_logprob like torch's. + # IT DECODES IN THIS PROCESS, holding the GIL the capture reader needs: in a room with music it starved + # the reader on McNugget (2026-10-24, see ear.py). For a live ear prefer whispercpp (its own process). try: import numpy as np import mlx_whisper