refactor: align message handling with Claude Code patterns and enhance MCP tool search

- Add Claude-style tool lifecycle message ordering (tool_use → hooks → tool_result)
- Implement sticky MCP tool search mode with cache invalidation strategy
- Support new content block types: server_tool_use, tool_search_tool_result, tool_reference
- Add hook_additional_context message handling with system-reminder tag parsing
- Refactor query loop with improved iteration planning and tool discovery extraction
- Add LLM debug proxy script for development debugging
- Enhance message normalization for Anthropic/OpenAI protocol compatibility

Generated with Ripperdoc

Co-Authored-By: Ripperdoc

Author: jstzwj

ripperdoc/cli/runtime_cli.py

@@ -37,7 +37,11 @@
     shutdown_mcp_runtime,
 )
 from ripperdoc.utils.memory import build_memory_instructions
-from ripperdoc.utils.messaging.messages import create_user_message
+from ripperdoc.utils.messaging.messages import (
+    UserMessage,
+    create_hook_additional_context_message,
+    create_user_message,
+)
 from ripperdoc.utils.sessions.session_history import SessionHistory
 from ripperdoc.utils.collaboration.tasks import set_runtime_task_scope
 from ripperdoc.utils.collaboration.worktree import (
@@ -133,13 +137,21 @@ async def _run_prompt_submission_hooks(
     prompt: str,
     query_context: QueryContext,
     additional_instructions: List[str],
-) -> bool:
+) -> tuple[bool, List[UserMessage]]:
+    hook_context_messages: List[UserMessage] = []
     with bind_pending_message_queue(query_context.pending_message_queue):
         session_start_result = await hook_manager.run_session_start_async("startup")
         _print_hook_system_message(session_start_result, "SessionStart")
         session_hook_contexts = _collect_hook_contexts(session_start_result)
         if session_hook_contexts:
-            additional_instructions.extend(session_hook_contexts)
+            for text in session_hook_contexts:
+                msg = create_hook_additional_context_message(
+                    text,
+                    hook_name="SessionStart",
+                    hook_event="SessionStart",
+                )
+                if msg:
+                    hook_context_messages.append(msg)
 
         prompt_hook_result = await hook_manager.run_user_prompt_submit_async(prompt)
         if prompt_hook_result.should_block or not prompt_hook_result.should_continue:
@@ -149,12 +161,19 @@ async def _run_prompt_submission_hooks(
                 or "Prompt blocked by hook."
             )
             console.print(f"[red]{escape(str(reason))}[/red]")
-            return False
+            return False, hook_context_messages
         _print_hook_system_message(prompt_hook_result, "UserPromptSubmit")
         prompt_hook_contexts = _collect_hook_contexts(prompt_hook_result)
         if prompt_hook_contexts:
-            additional_instructions.extend(prompt_hook_contexts)
-    return True
+            for text in prompt_hook_contexts:
+                msg = create_hook_additional_context_message(
+                    text,
+                    hook_name="UserPromptSubmit",
+                    hook_event="UserPromptSubmit",
+                )
+                if msg:
+                    hook_context_messages.append(msg)
+    return True, hook_context_messages
 
 
 def _build_effective_system_prompt(
@@ -379,7 +398,6 @@ async def run_query(
     )
     hook_manager.set_transcript_path(str(session_history.path))
     messages: List[Any] = [create_user_message(prompt)]
-    session_history.append(messages[0])
 
     resolved_model = _resolve_model_pointer_with_fallback(
         model,
@@ -409,13 +427,19 @@ async def run_query(
             query_context=query_context,
             disable_skills=disable_skills,
         )
-        should_continue = await _run_prompt_submission_hooks(
+        should_continue, hook_context_messages = await _run_prompt_submission_hooks(
             prompt=prompt,
             query_context=query_context,
             additional_instructions=additional_instructions,
         )
         if not should_continue:
+            for message in messages:
+                session_history.append(message)
             return
+        if hook_context_messages:
+            messages = [*hook_context_messages, *messages]
+        for message in messages:
+            session_history.append(message)
 
         system_prompt = _build_effective_system_prompt(
             custom_system_prompt=custom_system_prompt,

ripperdoc/cli/ui/rich_ui/session.py

@@ -78,6 +78,7 @@
     UserMessage,
     AssistantMessage,
     ProgressMessage,
+    create_hook_additional_context_message,
     create_user_message,
 )
 from ripperdoc.utils.log import enable_session_file_logging, get_logger
@@ -250,7 +251,7 @@ def __init__(
             },
         )
         self._session_history = SessionHistory(self.project_path, self.session_id)
-        self._session_hook_contexts: List[str] = []
+        self._session_hook_messages: List[UserMessage] = []
         self._session_start_time = time.time()
         self._session_end_sent = False
         self._exit_reason: Optional[str] = None
@@ -686,6 +687,19 @@ def _collect_hook_contexts(self, hook_result: Any) -> List[str]:
             contexts.append(str(additional_context))
         return contexts
 
+    def _collect_hook_context_messages(self, hook_result: Any, hook_event: str) -> List[UserMessage]:
+        messages: List[UserMessage] = []
+        additional_context = getattr(hook_result, "additional_context", None)
+        if additional_context:
+            message = create_hook_additional_context_message(
+                str(additional_context),
+                hook_name=hook_event,
+                hook_event=hook_event,
+            )
+            if message is not None:
+                messages.append(message)
+        return messages
+
     def _display_hook_system_message(
         self, hook_result: Any, event: str, tool_name: Optional[str] = None
     ) -> None:
@@ -698,7 +712,13 @@ def _display_hook_system_message(
         )
 
     def _set_session_hook_contexts(self, hook_result: Any) -> None:
-        self._session_hook_contexts = self._collect_hook_contexts(hook_result)
+        self._session_hook_messages = self._collect_hook_context_messages(
+            hook_result,
+            "SessionStart",
+        )
+        for message in self._session_hook_messages:
+            self.conversation_messages.append(message)
+            self._log_message(message)
         self._session_start_time = time.time()
         self._session_end_sent = False
 
@@ -1083,8 +1103,21 @@ def replay_conversation(
         """Render a conversation history in the console and seed prompt history."""
         if not messages:
             return
-        replay_messages = list(messages)
-        total_messages = len(replay_messages)
+
+        # Pre-filter messages: exclude hook_additional_context messages from UI display
+        # These messages are for AI context only, not for user display
+        def _should_display(msg: ConversationMessage) -> bool:
+            message_payload = getattr(msg, "message", None) or getattr(msg, "content", None)
+            metadata = getattr(message_payload, "metadata", None) or {}
+            return not metadata.get("hook_additional_context")
+
+        replay_messages = [msg for msg in messages if _should_display(msg)]
+        total_messages = len(messages)
+        displayable_count = len(replay_messages)
+
+        # Don't print anything if all messages were filtered out
+        if not replay_messages:
+            return
 
         if isinstance(max_messages, int):
             if max_messages == 0:
@@ -1093,12 +1126,12 @@ def replay_conversation(
                     "(history replay skipped).[/dim]"
                 )
                 return
-            if max_messages > 0 and total_messages > max_messages:
+            if max_messages > 0 and displayable_count > max_messages:
                 replay_messages = replay_messages[-max_messages:]
                 shown = len(replay_messages)
-                skipped = total_messages - shown
+                skipped = displayable_count - shown
                 self.console.print(
-                    f"\n[dim]Restored recent conversation ({shown}/{total_messages} messages; "
+                    f"\n[dim]Restored recent conversation ({shown}/{displayable_count} messages; "
                     f"skipped {skipped}).[/dim]"
                 )
             else:
@@ -1211,9 +1244,7 @@ def _stringify_message_content(self, content: Any) -> str:
     def _print_reasoning(self, reasoning: Any) -> None:
         self._message_display.print_reasoning(reasoning)
 
-    async def _prepare_query_context(
-        self, user_input: str, hook_instructions: Optional[List[str]] = None
-    ) -> tuple[str, Dict[str, str]]:
+    async def _prepare_query_context(self, user_input: str) -> tuple[str, Dict[str, str]]:
         """Load MCP servers, skills, and build system prompt.
 
         Returns:
@@ -1226,7 +1257,10 @@ async def _prepare_query_context(
         )
 
         if dynamic_tools and self.query_context:
-            merged_tools = merge_tools_with_dynamic(self.query_context.tools, dynamic_tools)
+            merged_tools = merge_tools_with_dynamic(
+                self.query_context.all_tools(),
+                dynamic_tools,
+            )
             if self.allowed_tools is not None:
                 merged_tools = filter_tools_by_names(merged_tools, self.allowed_tools)
             self.query_context.tools = merged_tools
@@ -1264,11 +1298,6 @@ async def _prepare_query_context(
         memory_instructions = build_memory_instructions()
         if memory_instructions:
             additional_instructions.append(memory_instructions)
-        if self._session_hook_contexts:
-            additional_instructions.extend(self._session_hook_contexts)
-        if hook_instructions:
-            additional_instructions.extend([text for text in hook_instructions if text])
-
         # Build system prompt based on options:
         # - custom_system_prompt: replaces the default entirely
         # - append_system_prompt: appends to the default system prompt
@@ -1822,16 +1851,24 @@ async def process_query(
                     self.console.print(f"[red]{escape(str(reason))}[/red]")
                     return
                 self._display_hook_system_message(hook_result, "UserPromptSubmit")
-                hook_instructions = self._collect_hook_contexts(hook_result)
-
-                system_prompt, context = await self._prepare_query_context(
-                    user_input, hook_instructions
+                prompt_hook_messages = self._collect_hook_context_messages(
+                    hook_result,
+                    "UserPromptSubmit",
                 )
+
+                system_prompt, context = await self._prepare_query_context(user_input)
                 processed_input, user_message = self._build_user_message_from_input(user_input)
                 if user_message_metadata:
                     user_message.message.metadata.update(dict(user_message_metadata))
 
-                messages: List[ConversationMessage] = self.conversation_messages + [user_message]
+                for message in prompt_hook_messages:
+                    self._log_message(message)
+
+                messages: List[ConversationMessage] = [
+                    *self.conversation_messages,
+                    *prompt_hook_messages,
+                    user_message,
+                ]
                 self._log_message(user_message)
                 if append_prompt_history:
                     self._append_prompt_history(processed_input)

ripperdoc/core/message_utils.py

@@ -449,14 +449,34 @@ def normalize_tool_args(raw_args: Any) -> Dict[str, Any]:
     return {}
 
 
+def format_context_as_system_reminder(context: Dict[str, str]) -> str:
+    """Format context dict as a system-reminder wrapped string."""
+    if not context:
+        return ""
+
+    # Format each context entry as "# key\nvalue"
+    context_lines = []
+    for key, value in context.items():
+        context_lines.append(f"# {key}\n{value}")
+    context_content = "\n".join(context_lines)
+
+    return f"""<system-reminder>
+As you answer the user's questions, you can use the following context:
+{context_content}
+
+      IMPORTANT: this context may or may not be relevant to your tasks. You should not respond to this context unless it is highly relevant to your task.
+</system-reminder>"""
+
+
 def build_full_system_prompt(
     system_prompt: str, context: Dict[str, str], tool_mode: str, tools: List[Tool[Any, Any]]
 ) -> str:
     """Compose the final system prompt including context and tool hints."""
     full_prompt = system_prompt
     if context:
-        context_str = "\n".join(f"{k}: {v}" for k, v in context.items())
-        full_prompt = f"{system_prompt}\n\nContext:\n{context_str}"
+        context_reminder = format_context_as_system_reminder(context)
+        if context_reminder:
+            full_prompt = f"{system_prompt}\n\n{context_reminder}"
     if tool_mode == "text":
         tool_hint = _tool_prompt_for_text_mode(tools)
         if tool_hint:
@@ -520,6 +540,24 @@ async def build_openai_tool_schemas(tools: List[Tool[Any, Any]]) -> List[Dict[st
 
 def content_blocks_from_anthropic_response(response: Any, tool_mode: str) -> List[Dict[str, Any]]:
     """Normalize Anthropic response content to our internal block format."""
+    def _to_plain_json(value: Any) -> Any:
+        if value is None:
+            return None
+        if hasattr(value, "model_dump"):
+            try:
+                value = value.model_dump(mode="json")
+            except (TypeError, ValueError):
+                value = value.model_dump()
+        elif hasattr(value, "dict"):
+            value = value.dict()
+        if isinstance(value, list):
+            return [_to_plain_json(item) for item in value]
+        if isinstance(value, tuple):
+            return [_to_plain_json(item) for item in value]
+        if isinstance(value, dict):
+            return {str(key): _to_plain_json(item) for key, item in value.items()}
+        return value
+
     blocks: List[Dict[str, Any]] = []
     for block in getattr(response, "content", []) or []:
         btype = getattr(block, "type", None)
@@ -552,6 +590,33 @@ def content_blocks_from_anthropic_response(response: Any, tool_mode: str) -> Lis
                     "input": normalize_tool_args(raw_input),
                 }
             )
+        elif btype == "server_tool_use":
+            raw_input = getattr(block, "input", {}) or {}
+            blocks.append(
+                {
+                    "type": "server_tool_use",
+                    "id": getattr(block, "id", None) or str(uuid4()),
+                    "name": getattr(block, "name", None),
+                    "input": normalize_tool_args(raw_input),
+                }
+            )
+        elif btype == "tool_search_tool_result":
+            blocks.append(
+                {
+                    "type": "tool_search_tool_result",
+                    "tool_use_id": getattr(block, "tool_use_id", None)
+                    or getattr(block, "id", None)
+                    or "",
+                    "content": _to_plain_json(getattr(block, "content", None)) or {},
+                }
+            )
+        elif btype == "tool_reference":
+            blocks.append(
+                {
+                    "type": "tool_reference",
+                    "tool_name": getattr(block, "tool_name", None),
+                }
+            )
 
     if tool_mode == "text":
         blocks = _maybe_convert_json_block_to_tool_use(blocks)