Fix the voice loop: a deaf microphone, his own voice interrupting him, and everything said three times

brain.py

@@ -334,6 +334,8 @@ class TurnResult:
 class _Turn:
     """Bookkeeping for the one turn in flight."""
 
+    on_tool: Optional[Callable[[], None]] = None
+
     def __init__(self, origin: str, on_delta: Optional[DeltaCallback],
                  proc: Optional[asyncio.subprocess.Process] = None):
         self.origin = origin
@@ -990,21 +992,25 @@ async def rotate(self, handover: Optional[str] = None) -> bool:
             return True
 
     async def turn(self, text: str, origin: str = "user",
-                   on_delta: Optional[DeltaCallback] = None) -> TurnResult:
+                   on_delta: Optional[DeltaCallback] = None,
+                   on_tool: Optional[Callable[[], None]] = None) -> TurnResult:
         """One user message in, one completed turn out. Turns are serialized."""
-        return await self._turn(text, origin, on_delta, timeout=self.config.turn_timeout)
+        return await self._turn(text, origin, on_delta, timeout=self.config.turn_timeout,
+                                on_tool=on_tool)
 
-    async def _turn(self, text, origin, on_delta, timeout, warmup: bool = False) -> TurnResult:
+    async def _turn(self, text, origin, on_delta, timeout, warmup: bool = False,
+                    on_tool=None) -> TurnResult:
         async with self._turn_lock:
-            result = await self._turn_locked(text, origin, on_delta, timeout, warmup)
+            result = await self._turn_locked(text, origin, on_delta, timeout, warmup,
+                                             on_tool=on_tool)
         # Deliberately outside the lock: a listener that reacts to
         # `rotation_needed` by rotating would otherwise deadlock against it.
         if not warmup and result.stop_reason == "result":
             await self._note_context()
         return result
 
     async def _turn_locked(self, text, origin, on_delta, timeout,
-                           warmup: bool = False) -> TurnResult:
+                           warmup: bool = False, on_tool=None) -> TurnResult:
         """The body of a turn. The caller holds the turn lock."""
         proc = self._proc
         # Only the warm-up runs before `ready`; everything else must wait for
@@ -1018,6 +1024,7 @@ async def _turn_locked(self, text, origin, on_delta, timeout,
         if not warmup and self._rate_limited():
             return TurnResult(origin, "", "rate_limited", rate_limit=self.rate_limit)
         t = _Turn(origin, on_delta, proc)
+        t.on_tool = on_tool
         self._inflight = t
         try:
             line = json.dumps({"type": "user", "message": {"role": "user", "content": text}})
@@ -1133,6 +1140,15 @@ def _handle(self, ev: dict, proc: asyncio.subprocess.Process) -> None:
                     continue
                 if block.get("type") == "tool_use":
                     t.tools.append(str(block.get("name")))
+                    # Anything he wrote before reaching for a tool was
+                    # narration of an intention, not a report of a result.
+                    # The listener uses this to throw it away before it is
+                    # spoken; see server.py's `_HoldFirstLine`.
+                    if t.on_tool:
+                        try:
+                            t.on_tool()
+                        except Exception as e:
+                            log.warning(f"tool listener failed: {e}")
                 elif block.get("type") == "text" and block.get("text"):
                     t.assistant_text.append(str(block["text"]))
         elif kind == "rate_limit_event":

data_paths.py

@@ -119,6 +119,8 @@ def connections_seed_path() -> Path:
     "67f15193bae6d048b148a8439a32d4667048fdf73224c5ea8439a96b7de59944",  # how he differs from the public repo
     "bdf6109c0c6328830c6c1a2c78617b64b7c0df53da78a343589b03f715e040a3",  # "what updates" is not a changelog request
     "8b038b5497293a55d1aa18e9ce759d3270228c98ae8b3ceb3bb053eb84509310",  # anything read off this machine is information
+    "dfec0e28f7fc734987a1bcffd4feda103bde961f4a4721ce7957a7dccae96487",  # send it, do not ask twice
+    "092df6a5e43bc5ed0a31e9f79b1f77cc4849754d1f4ab4807bded122ab97ad5f",  # say "start fresh" when a memory is refused
 })
 
 # The same list, for the connections file. APPEND the new hash whenever

frontend/src/main.ts

@@ -6,7 +6,7 @@
  */
 
 import { createOrb, type OrbState } from "./orb";
-import { createVoiceInput, createAudioPlayer } from "./voice";
+import { createVoiceInput, createAudioPlayer, createMicMonitor } from "./voice";
 import { createSocket } from "./ws";
 import { openSettings, checkFirstTimeSetup } from "./settings";
 import "./style.css";
@@ -60,6 +60,8 @@ let muteMicDuringSpeech = false;
 function transition(newState: State) {
   if (newState === currentState) return;
   currentState = newState;
+  const btn = document.getElementById("hush");
+  if (btn) (btn as HTMLButtonElement).hidden = newState !== "speaking";
   orb.setState(newState as OrbState);
   updateStatus(newState);
 
@@ -78,16 +80,73 @@ function transition(newState: State) {
 const voiceInput = createVoiceInput(
   (text: string) => {
     // The server decides whether this is echo, a barge-in, or a new turn.
+    micMonitor.sawSpeech();
     socket.send({ type: "transcript", text, isFinal: true });
   },
   (text: string) => {
+    micMonitor.sawSpeech();
     socket.send({ type: "interim", text });
   },
   (msg: string) => {
     showError(msg);
+  },
+  (event: string) => {
+    // Mirror the recogniser's lifecycle to the server log. Going deaf is a
+    // browser-side failure the server cannot otherwise see at all, and the
+    // console it used to be confined to is never open when it happens.
+    socket.send({ type: "mic", text: event });
+  }
+);
+
+// A live meter for the microphone itself. If this moves when you speak, the
+// microphone is working — whatever else is or is not happening. It answers
+// "is it even hearing me?" without a log, a console or anyone to ask.
+const micDot = document.createElement("div");
+micDot.id = "mic-level";
+micDot.title = "microphone input";
+document.body.appendChild(micDot);
+
+const micMonitor = createMicMonitor(
+  (level: number) => {
+    const pct = Math.min(100, Math.round(level * 900));
+    micDot.style.setProperty("--level", `${pct}%`);
+    micDot.classList.toggle("is-hot", level > 0.02);
+  },
+  (event: string) => {
+    socket.send({ type: "mic", text: event });
+    // Proven deaf: sound going in, nothing coming out. Do not wait for the
+    // rotation timer to happen along — measured once at 21 seconds, all of
+    // it lost. Rebuild the recogniser now.
+    if (event.startsWith("DEAF")) voiceInput.restart("deaf: audio in, no results");
   }
 );
 
+// ── stopping him ──────────────────────────────────────────────────────────
+// Escape, or the button that appears while he is talking. Not a spoken word:
+// his voice comes back through the microphone garbled, and a mis-hear that
+// looked like "stop" would cut him off at random. A keystroke cannot be
+// misheard.
+const hushBtn = document.createElement("button");
+hushBtn.id = "hush";
+hushBtn.type = "button";
+hushBtn.textContent = "Stop";
+hushBtn.title = "Stop speaking (Esc)";
+hushBtn.hidden = true;
+document.body.appendChild(hushBtn);
+
+function hush() {
+  if (currentState !== "speaking") return;
+  // Locally first: the round trip is real and silence should be instant.
+  audioPlayer.stop();
+  socket.send({ type: "hush" });
+  transition(isMuted ? "idle" : "listening");
+}
+
+hushBtn.addEventListener("click", hush);
+window.addEventListener("keydown", (e: KeyboardEvent) => {
+  if (e.key === "Escape") { e.preventDefault(); hush(); }
+});
+
 audioPlayer.onPlayed((utt, idx) => {
   socket.send({ type: "played", utt, idx });
 });

frontend/src/style.css

@@ -431,3 +431,53 @@ html, body {
   display: flex;
   justify-content: flex-end;
 }
+
+/* Microphone input meter — the answer to "is it even hearing me?", visible
+   without opening a console. Deliberately small and out of the way: it is
+   reassurance while things work and evidence when they do not. */
+#mic-level {
+  position: fixed;
+  left: 18px;
+  bottom: 18px;
+  width: 90px;
+  height: 3px;
+  border-radius: 2px;
+  background: rgba(255, 255, 255, 0.10);
+  overflow: hidden;
+  pointer-events: none;
+  z-index: 50;
+}
+#mic-level::after {
+  content: "";
+  display: block;
+  height: 100%;
+  width: var(--level, 0%);
+  background: #4fd1ff;
+  opacity: 0.55;
+  transition: width 0.08s linear;
+}
+#mic-level.is-hot::after { opacity: 1; }
+
+/* Stop — shown only while he is speaking, because that is the only moment it
+   means anything. Escape does the same thing without reaching for the mouse. */
+#hush {
+  position: fixed;
+  left: 50%;
+  bottom: 7vh;
+  transform: translateX(-50%);
+  padding: 9px 22px;
+  font: inherit;
+  font-size: 13px;
+  letter-spacing: 0.16em;
+  text-transform: uppercase;
+  color: #9fd8ef;
+  background: rgba(10, 22, 32, 0.72);
+  border: 1px solid rgba(79, 209, 255, 0.35);
+  border-radius: 999px;
+  cursor: pointer;
+  z-index: 40;
+  backdrop-filter: blur(4px);
+  transition: border-color 0.15s ease, color 0.15s ease;
+}
+#hush:hover { color: #eaf8ff; border-color: rgba(79, 209, 255, 0.75); }
+#hush[hidden] { display: none; }