feat(agent): let read_file return images and transcribe audio

core/agent/agent_executor.py

@@ -14,6 +14,7 @@
 from core.chat.message_utils import KiraExceptionEvent
 from core.plugin.plugin_handlers import event_handler_reg, EventType
 from core.prompt_manager import Prompt
+from core.utils.media_refs import MEDIA_REF_TYPE
 
 if TYPE_CHECKING:
     from core.chat.message_utils import KiraMessageBatchEvent
@@ -48,6 +49,62 @@ def __init__(self, tool_manager: FuncToolManager, tool_set: Optional[ToolSet] =
         self.tool_manager = tool_manager
         self.tool_set = tool_set
 
+    @staticmethod
+    def _build_tool_messages(tool_results: list[dict]) -> list[OpenAIMessage]:
+        """Build the tool messages of one agent step from raw tool results.
+
+        Media references embedded by tools (``kira_image_ref`` parts) are
+        relocated into a single user message appended after the tool results:
+        some providers reject image parts inside tool-role messages, while a
+        user message carrying images is supported by every vision-capable
+        provider. Both the request and the persisted history receive the same
+        relocated shape, so replays stay consistent with what the model saw.
+        """
+        messages: list[OpenAIMessage] = []
+        media_parts: list[dict] = []
+        media_tool_names: list[str] = []
+        for result in tool_results:
+            content = result.get("content")
+            if isinstance(content, list):
+                text_parts = [
+                    part for part in content
+                    if not (isinstance(part, dict) and part.get("type") == MEDIA_REF_TYPE)
+                ]
+                refs = [
+                    part for part in content
+                    if isinstance(part, dict) and part.get("type") == MEDIA_REF_TYPE
+                ]
+                if refs:
+                    media_parts.extend(refs)
+                    tool_name = result.get("name") or "unknown_tool"
+                    if tool_name not in media_tool_names:
+                        media_tool_names.append(tool_name)
+                    # Refs were the only non-text parts, so joining the rest
+                    # always restores the original plain-text tool output.
+                    tool_content = "".join(
+                        part.get("text", "") if isinstance(part, dict) else str(part)
+                        for part in text_parts
+                    )
+                    messages.append(OpenAIMessage(
+                        role="tool",
+                        tool_call_id=result.get("tool_call_id"),
+                        name=result.get("name"),
+                        content=tool_content,
+                    ))
+                    continue
+            messages.append(OpenAIMessage(**result))
+
+        if media_parts:
+            joined_names = ", ".join(media_tool_names)
+            messages.append(OpenAIMessage(
+                role="user",
+                content=[
+                    {"type": "text", "text": f"Media returned by tool call(s): {joined_names}"},
+                    *media_parts,
+                ],
+            ))
+        return messages
+
     async def run(
         self,
         ctx: AgentExecutionContext,
@@ -225,7 +282,7 @@ async def run(
             )
             request.messages.append(msg)
             ctx.new_messages.append(msg)
-            tool_msgs = [OpenAIMessage(**r) for r in llm_resp.tool_results]
+            tool_msgs = self._build_tool_messages(llm_resp.tool_results)
             request.messages.extend(tool_msgs)
             ctx.new_messages.extend(tool_msgs)
 

core/agent/func_tool_manager.py

@@ -137,6 +137,7 @@ async def execute_tool(self, event: KiraMessageBatchEvent, resp: LLMResponse, to
 
             # Save tool results
             content = await tool_result_obj.assemble_result()
+            content = tool_result_obj.build_content(content)
             tool_logger.info(f"tool_result: {content}")
             resp.tool_results.append({
                 "role": "tool",

core/agent/tool.py

@@ -60,6 +60,13 @@ class ToolResult:
 
     attachments: list[Union[Image, Record, File]] = field(default_factory=list)
 
+    # Provider-independent image references (``kira_image_ref`` parts) attached
+    # to the tool result. The agent executor relocates them into a user message
+    # following the tool results (some providers reject image parts in
+    # tool-role messages); refs stay compact paths in persisted history and are
+    # resolved to image parts per LLM request.
+    media_refs: list[dict] = field(default_factory=list)
+
     result_str: str = field(default="", init=False, repr=False)
 
     async def assemble_result(self):
@@ -101,3 +108,14 @@ async def assemble_result(self):
         )
         self.result_str = "".join(res_text)
         return self.result_str
+
+    def build_content(self, text: str) -> Union[str, list]:
+        """Return the tool result content, embedding media refs when present.
+
+        Without media refs the content stays a plain string; with them it becomes
+        a content part list the agent executor splits into the tool message text
+        and a following user message carrying the media.
+        """
+        if not self.media_refs:
+            return text
+        return [{"type": "text", "text": text}, *self.media_refs]

core/plugin/builtin_plugins/agent/main.py

@@ -14,9 +14,13 @@
 
 from core.plugin import BasePlugin, logger, on, Priority, register
 from core.chat import KiraMessageBatchEvent, MessageChain
-from core.chat.message_elements import Text
+from core.chat.message_elements import Image, Record, Text
 from core.provider import LLMRequest
+from core.agent.tool import ToolResult
 
+from core.utils.common_utils import desc_img, speech_to_text
+from core.utils.image_compression import compress_image_file
+from core.utils.media_refs import store_session_media
 from core.utils.path_utils import get_config_path, get_data_path, get_root_path
 
 PLUGIN_ID = "agent"
@@ -51,6 +55,17 @@
                       '.dll', '.so', '.dylib', '.pdf', '.doc', '.docx', '.xls', '.xlsx',
                       '.ppt', '.pptx', '.iso', '.img', '.dmg'}
 
+# Raster image formats read_file can serve to the LLM (compression and VLM
+# both handle these). SVG stays blocked: it is text-based and neither the
+# image compressor nor vision models treat it as a raster image.
+readable_image_extensions = {'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.webp'}
+
+# Audio formats read_file transcribes via the default STT model, mirroring the
+# voice-recognition set of the chat adapters (which also accepts .m4a, so it is
+# routed here even though it is not in blocked_extensions).
+readable_audio_extensions = {'.mp3', '.wav', '.ogg', '.flac', '.aac', '.m4a',
+                             '.amr', '.silk', '.slk', '.slac'}
+
 ALL_TOOL_NAMES = [
     "read_file", "write_file", "edit_file", "list_files", "grep", "search_files",
     "exec", "manage_background_exec",
@@ -551,13 +566,27 @@ def _on_background_exec_done(self, task_id: str, session: str, task: asyncio.Tas
         notice_task.add_done_callback(self._background_notice_tasks.discard)
 
     def _is_path_allowed(self, path: str, allowed_prefixes: tuple) -> bool:
-        """Check if path starts with an allowed prefix directory."""
+        """Check a normalized path against the allowed roots with symlinks resolved.
+
+        Both the candidate and each allowed root are resolved to their real
+        location before the containment check, so a symlink planted inside an
+        allowed directory cannot alias a read or write out of the configured
+        roots. Not-yet-existing write targets resolve as far as their existing
+        ancestors, which is sufficient here.
+        """
+        try:
+            candidate = self._resolve_path(path).resolve()
+        except (OSError, RuntimeError, ValueError):
+            return False
         for prefix in allowed_prefixes:
-            prefix = self._normalize_path(prefix)
-            if prefix is None:
+            normalized = self._normalize_path(prefix)
+            if normalized is None:
                 continue
-            prefix = prefix.rstrip('/')
-            if path == prefix or path.startswith(prefix + '/'):
+            try:
+                root = self._resolve_path(normalized).resolve()
+            except (OSError, RuntimeError, ValueError):
+                continue
+            if candidate == root or root in candidate.parents:
                 return True
         return False
 
@@ -603,18 +632,18 @@ def _find_files(path: Path, pattern: str) -> list[Path]:
 
     @register.tool(
         "read_file",
-        "Read a plain text file (txt, html, py, etc..) in allowed read paths",
+        "Read a plain text file (txt, html, py, etc..), an image file (jpg, png, gif, etc..) or an audio file (mp3, wav, etc..) in allowed read paths. Images follow the configured image processing mode: returned as raw image data attached to this result in native multimodal mode, or as a text description in VLM description mode. Audio files are transcribed to text via speech recognition.",
         {
             "type": "object",
             "properties": {
                 "path": {"type": "string", "description": "File path, must start with an allowed path prefix"},
-                "offset": {"type": "integer", "description": "Which line to start reading, defaults to 1"},
-                "limit": {"type": "integer", "description": "Maximum lines to read, defaults to 200"},
+                "offset": {"type": "integer", "description": "Which line to start reading, defaults to 1. Ignored for image and audio files."},
+                "limit": {"type": "integer", "description": "Maximum lines to read, defaults to 200. Ignored for image and audio files."},
             },
             "required": ["path"]
         }
     )
-    async def read_file(self, event: KiraMessageBatchEvent, path: str, offset: int = 1, limit: int = 200) -> str:
+    async def read_file(self, event: KiraMessageBatchEvent, path: str, offset: int = 1, limit: int = 200) -> str | ToolResult:
         if not self._is_file_session_allowed(event.sid):
             return "Permission denied: current session not allowed to access local files"
 
@@ -630,6 +659,10 @@ async def read_file(self, event: KiraMessageBatchEvent, path: str, offset: int =
             return f"Permission denied: Path must start with one of: {', '.join(self.allowed_read_paths)}"
 
         ext = Path(path).suffix.lower()
+        if ext in readable_image_extensions:
+            return await self._read_image_file(event, path)
+        if ext in readable_audio_extensions:
+            return await self._read_audio_file(event, path)
         if ext in blocked_extensions:
             return "Multimedia and binary files are not allowed"
 
@@ -653,6 +686,131 @@ async def read_file(self, event: KiraMessageBatchEvent, path: str, offset: int =
         except Exception as e:
             return f"[Failed to read file: {e}]"
 
+    async def _read_image_file(self, event: KiraMessageBatchEvent, path: str) -> str | ToolResult:
+        """Read an image file according to the image processing mode.
+
+        Native multimodal mode attaches the (possibly compressed) image data as
+        a media reference so the main LLM sees it directly; VLM description mode
+        returns the transcribed description instead. Both paths honor the global
+        image compression settings, mirroring how incoming chat images are
+        bounded before reaching a model.
+        """
+        abs_path = self._resolve_path(path)
+        if not abs_path.is_file():
+            return f"[Failed to read file: file not found: {path}]"
+
+        capabilities = self.ctx.get_session_capabilities(event.sid) if self.ctx else {}
+        image_recognition = capabilities.get("image_recognition") if isinstance(capabilities, dict) else None
+        if not isinstance(image_recognition, dict):
+            image_recognition = {}
+        mode = image_recognition.get("mode", "vlm_description")
+
+        compression_config = (
+            self.ctx.config.get_config("bot_config.image_compression", {}) if self.ctx else {}
+        )
+        try:
+            image_path, mime = await compress_image_file(abs_path, compression_config)
+        except Exception as e:
+            return f"[Failed to read file: {e}]"
+
+        try:
+            if mode == "native":
+                message_id = event.messages[-1].message_id if event.messages else "read_file"
+                media_ref = await store_session_media(
+                    Image(image=str(image_path), mime=mime), event.sid, message_id
+                )
+                return ToolResult(
+                    text=f"Image file: {path} ({mime}). The raw image data is attached in the message below.",
+                    media_refs=[media_ref],
+                )
+
+            if not image_recognition.get("enabled", True):
+                return f"[Image (recognition disabled), file_path: {path}]"
+
+            desc = await self._describe_image_file(image_path, mime, image_recognition)
+            if not desc:
+                return f"[Image (description unavailable), file_path: {path}]"
+            return f"[Image {desc}, file_path: {path}]"
+        finally:
+            if image_path != abs_path:
+                # The compressed copy in data/temp is only ever consumed inside
+                # this call (the native branch copies it into session media, the
+                # VLM branch turns it into a description), so it can go now.
+                try:
+                    await asyncio.to_thread(image_path.unlink, missing_ok=True)
+                except OSError as e:
+                    logger.warning(f"Failed to remove compressed temp image {image_path.name}: {e}")
+
+    async def _describe_image_file(self, image_path: Path, mime: str, image_recognition: dict) -> str:
+        """Transcribe an image file with the default VLM, reusing the shared description cache.
+
+        An empty return means the VLM produced no description; the caller tells
+        recognition-disabled and VLM-failure apart on its own.
+        """
+        desc_cache = None
+        md5 = None
+        try:
+            from core.message_manager import ImageDescCache
+
+            desc_cache = ImageDescCache(self.ctx.db)
+            md5 = await Image(image=str(image_path), mime=mime).hash_image()
+            cached_desc = await desc_cache.get(md5)
+            if cached_desc:
+                return cached_desc
+        except Exception as e:
+            logger.warning(f"Failed to read image desc cache for read_file: {e}")
+            desc_cache = None
+
+        desc = ""
+        try:
+            vlm_client = self.ctx.provider_mgr.get_default_vlm()
+            desc = await desc_img(
+                client=vlm_client,
+                image=Image(image=str(image_path), mime=mime),
+                prompt=str(image_recognition.get("desc_prompt", "") or "").strip() or None,
+                lang=self.ctx.get_lang(),
+            )
+        except Exception as e:
+            logger.error(f"Failed to describe image file for read_file: {e}")
+
+        if desc and desc_cache and md5:
+            try:
+                await desc_cache.set(md5, desc)
+            except Exception as e:
+                logger.warning(f"Failed to cache read_file image desc: {e}")
+        return desc
+
+    async def _read_audio_file(self, event: KiraMessageBatchEvent, path: str) -> str:
+        """Transcribe an audio file with the default STT model.
+
+        Mirrors how incoming voice records are recognized: the session's
+        ``stt`` capability toggle is honored, and an unavailable STT model or a
+        failed transcription degrades to returning the file path only, so the
+        turn can still forward the file to the user.
+        """
+        abs_path = self._resolve_path(path)
+        if not abs_path.is_file():
+            return f"[Failed to read file: file not found: {path}]"
+
+        capabilities = self.ctx.get_session_capabilities(event.sid) if self.ctx else {}
+        stt_caps = capabilities.get("stt") if isinstance(capabilities, dict) else None
+        if not isinstance(stt_caps, dict):
+            stt_caps = {}
+        if not stt_caps.get("enabled", True):
+            return f"[Record (speech recognition disabled), file_path: {path}]"
+
+        try:
+            stt_client = self.ctx.provider_mgr.get_default_stt()
+            transcript = await speech_to_text(
+                client=stt_client, record=Record(record=str(abs_path))
+            )
+        except Exception as e:
+            logger.error(f"Failed to transcribe audio file for read_file: {e}")
+            transcript = ""
+        if not transcript:
+            return f"[Record (speech recognition unavailable), file_path: {path}]"
+        return f"[Record {transcript}, file_path: {path}]"
+
     @register.tool(
         "write_file",
         "Write content to a plain text file in allowed write paths. Creates the file if it doesn't exist, overwrites if it does.",

core/prompts/agent_tmpl.py

@@ -122,7 +122,7 @@
 [At all] # at全体成员消息
 [Reply message_id/message_content] # message_id 为用户回复的消息的ID 或者 message_content 为引用的消息内容
 [Poke 用户xxx戳了戳/捏了捏你的xxx] # 不要认为这是冒犯或真实的对话，这是社交平台的戳一戳互动提示，表示对方在轻轻提醒或调侃你。请理解为轻松、友好的互动
-[Image image_description file_path: xxx] # 用户发送的图片消息，通常情况下你无需使用工具如list_files等来获取图片路径的相关信息
+[Image image_description file_path: xxx] # 用户发送的图片消息，描述即完整的识图结果：通常情况下你无需使用list_files等工具获取图片路径的相关信息，也无需通过read_file等工具查看该路径
 
 """
 
@@ -257,7 +257,7 @@
 [At all] # message mentioning all members
 [Reply message_id/message_content] # ID or quoted message content
 [Poke User xxx poked/nudged you] # a friendly social-platform interaction, not an insult or literal dialogue
-[Image image_description file_path: xxx] # an image message; normally no tool is needed to obtain the image path
+[Image image_description file_path: xxx] # an image message; the description is the complete recognition result: normally no tool such as list_files is needed for image path information, and there is no need to view the path with read_file or similar tools
 
 """,
     "accounts": """\