From 56ed8cb3a38f865baa64337c15242a29a1037d31 Mon Sep 17 00:00:00 2001 From: Ikra Siddiqui Date: Wed, 22 Apr 2026 15:48:12 +0000 Subject: [PATCH] BZ-2517: feat(breeze-buddy): emit voice-to-chat-redirect RTVI event for restricted tools When the LLM attempts to call a voice-restricted tool (e.g. initiateAgenticLoop, runSRAnalysis) during a Daily.co voice session, intercept it via the LLM's on_function_calls_started event, speak a short TTS redirect message, and emit a custom 'voice-to-chat-redirect' RTVI server message to the Lighthouse frontend. --- .../agents/breeze_buddy/agent/__init__.py | 46 +++++++++++++++++++ .../breeze_buddy/utils/voice_restrictions.py | 27 +++++++++++ 2 files changed, 73 insertions(+) create mode 100644 app/ai/voice/agents/breeze_buddy/utils/voice_restrictions.py diff --git a/app/ai/voice/agents/breeze_buddy/agent/__init__.py b/app/ai/voice/agents/breeze_buddy/agent/__init__.py index 887f39b19..11bd7ea9c 100644 --- a/app/ai/voice/agents/breeze_buddy/agent/__init__.py +++ b/app/ai/voice/agents/breeze_buddy/agent/__init__.py @@ -81,6 +81,10 @@ create_background_sound_mixer, track_error, ) +from app.ai.voice.agents.breeze_buddy.utils.voice_restrictions import ( + VOICE_REDIRECT_TTS_MESSAGE, + VOICE_RESTRICTED_TOOLS, +) from app.ai.voice.agents.breeze_buddy.utils.transport.websockets import ( close_websocket_safely, ) @@ -164,6 +168,9 @@ def __init__( # RTVI processor for daily mode real-time events self._rtvi_processor: Any = None + # LLM service (set after create_services; None in stream mode) + self._llm: Any = None + # Stream mode transcript collector (replaces LLMContext for transcription) self._transcript_collector: Optional[TranscriptCollectorProcessor] = None @@ -644,6 +651,44 @@ async def on_client_ready(rtvi): RTVIServerMessageFrame(data={"type": "bot-ready"}) ) + # Voice-to-chat redirect: intercept restricted tool calls in daily mode + if self.is_daily_mode and self._rtvi_processor and not self.is_stream_mode and self._llm: + + @self._llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + """Intercept voice-restricted tool calls and redirect to chat.""" + for fc in function_calls: + if fc.function_name not in VOICE_RESTRICTED_TOOLS: + continue + + logger.info( + f"[VOICE_REDIRECT] Intercepted restricted tool '{fc.function_name}' " + f"in voice mode, emitting redirect event. " + f"conversation_id={self.conversation_id}" + ) + + # Speak the redirect message via TTS + if self.task: + try: + await self.task.queue_frame( + TTSSpeakFrame(text=VOICE_REDIRECT_TTS_MESSAGE) + ) + except Exception as e: + logger.warning( + f"[VOICE_REDIRECT] Failed to queue TTS redirect message: {e}" + ) + + # Emit the RTVI event to Lighthouse frontend + await self._emit_rtvi_event( + "voice-to-chat-redirect", + { + "tool": fc.function_name, + "reason": "voice_restricted", + "conversation_id": self.conversation_id or "", + }, + ) + break # Handle only the first restricted tool per batch + # Stream mode: accept tts-speak via RTVI client-message (PipecatClient SDK) if self.is_stream_mode and self._rtvi_processor: @@ -808,6 +853,7 @@ async def run(self, runner_args: Optional[RunnerArguments] = None) -> None: stt, llm, tts = await create_services( self.configurations, include_llm=not is_stream ) + self._llm = llm # May be None in stream mode if not is_stream: assert llm is not None, "LLM is required in agent mode" diff --git a/app/ai/voice/agents/breeze_buddy/utils/voice_restrictions.py b/app/ai/voice/agents/breeze_buddy/utils/voice_restrictions.py new file mode 100644 index 000000000..2cc485e92 --- /dev/null +++ b/app/ai/voice/agents/breeze_buddy/utils/voice_restrictions.py @@ -0,0 +1,27 @@ +"""Voice-restricted tool definitions for Breeze Buddy. + +Tools listed here are available in chat mode but not in voice mode +because they are long-running agentic loops that would exhaust the +LLM context window during a real-time voice conversation. +""" + +from typing import FrozenSet + +# Tools that are only available in chat mode. +# When the LLM invokes one of these during a voice session, the agent +# should speak a redirect message and emit a 'voice-to-chat-redirect' +# RTVI event to the Lighthouse frontend. +VOICE_RESTRICTED_TOOLS: FrozenSet[str] = frozenset( + [ + "initiateAgenticLoop", + "generateAdsPerformanceReport", + "generateShopifyBreezeGoLiveReport", + "runSRAnalysis", + "generateReport", + ] +) + +VOICE_REDIRECT_TTS_MESSAGE = ( + "This feature is only available in chat mode. " + "I'll take you there now so you can get the full analysis." +)