From df1bbb655ab21f2521be637baed640feaf7414d9 Mon Sep 17 00:00:00 2001 From: guerler Date: Mon, 19 Jan 2026 13:02:54 +0300 Subject: [PATCH] Add viz guidance to chatgxy --- lib/galaxy/agents/base.py | 16 ++- lib/galaxy/agents/orchestrator.py | 139 ++++++++++++++++++++- lib/galaxy/agents/prompts/orchestrator.md | 13 ++ lib/galaxy/agents/prompts/router.md | 3 +- lib/galaxy/agents/router.py | 25 ++-- lib/galaxy/agents/visualization_context.py | 131 +++++++++++++++++++ lib/galaxy/managers/agents.py | 6 + lib/galaxy/schema/agents.py | 1 + 8 files changed, 323 insertions(+), 11 deletions(-) create mode 100644 lib/galaxy/agents/visualization_context.py diff --git a/lib/galaxy/agents/base.py b/lib/galaxy/agents/base.py index 6c2133968476..ac2231c8606f 100644 --- a/lib/galaxy/agents/base.py +++ b/lib/galaxy/agents/base.py @@ -293,12 +293,26 @@ def _prepare_prompt(self, query: str, context: dict[str, Any]) -> str: """Prepare the full prompt including context.""" prompt_parts = [query] - # Add context if available if context: + # Work with a copy to avoid mutating the caller's context + context = context.copy() + + # Handle visualizations specially + visualizations = context.pop("visualizations", None) + + # Add remaining context context_str = "\n".join([f"{k}: {v}" for k, v in context.items() if v]) if context_str: prompt_parts.insert(0, f"Context:\n{context_str}\n") + # Add visualization context + if visualizations: + from galaxy.agents.visualization_context import format_visualization_context + + viz_context = format_visualization_context(visualizations) + if viz_context: + prompt_parts.insert(0, viz_context) + return "\n".join(prompt_parts) def _format_response(self, result: Any, query: str, context: dict[str, Any]) -> AgentResponse: diff --git a/lib/galaxy/agents/orchestrator.py b/lib/galaxy/agents/orchestrator.py index 7691b79f1d52..f2982319d11d 100644 --- a/lib/galaxy/agents/orchestrator.py +++ b/lib/galaxy/agents/orchestrator.py @@ -14,13 +14,18 @@ from pydantic import BaseModel from pydantic_ai import Agent -from galaxy.schema.agents import ConfidenceLevel +from galaxy.schema.agents import ( + ActionSuggestion, + ActionType, + ConfidenceLevel, +) from .base import ( AgentResponse, AgentType, BaseGalaxyAgent, GalaxyAgentDependencies, ) +from .visualization_context import is_visualization_query log = logging.getLogger(__name__) @@ -93,6 +98,11 @@ async def process(self, query: str, context: Optional[dict[str, Any]] = None) -> Comprehensive response from multiple coordinated agents """ try: + # Check if this is a visualization query - handle directly without sub-agents + visualizations = (context or {}).get("visualizations", []) + if visualizations and is_visualization_query(query, visualizations): + return await self._handle_visualization_query(query, context or {}) + # Get agent plan from LLM plan = await self._get_agent_plan(query) @@ -105,11 +115,14 @@ async def process(self, query: str, context: Optional[dict[str, Any]] = None) -> # Combine responses combined_content = self._combine_responses(responses, plan.reasoning) + # Extract visualization suggestions from content + suggestions = self._extract_visualization_suggestions(combined_content, context or {}) + return AgentResponse( content=combined_content, confidence=ConfidenceLevel.HIGH, agent_type=self.agent_type, - suggestions=[], + suggestions=suggestions, metadata={ "agents_used": plan.agents, "execution_type": "sequential" if plan.sequential else "parallel", @@ -127,6 +140,87 @@ async def process(self, query: str, context: Optional[dict[str, Any]] = None) -> log.error(f"Unexpected error during orchestration: {e}") return self._get_fallback_response(query, str(e)) + async def _handle_visualization_query( + self, query: str, context: dict[str, Any] + ) -> AgentResponse: + """Handle visualization queries directly using available plugins.""" + visualizations = context.get("visualizations", []) + + # Use LLM to generate a helpful response about visualizations + viz_prompt = self._build_visualization_prompt(query, visualizations) + + try: + result = await self._run_with_retry(viz_prompt) + response_text = str(result.data) if hasattr(result, "data") else str(result) + except Exception as e: + log.warning(f"LLM call failed for visualization query, using fallback: {e}") + response_text = self._build_visualization_fallback(query, visualizations) + + # Extract visualization suggestions from the response + suggestions = self._extract_visualization_suggestions(response_text, context) + + return AgentResponse( + content=response_text, + confidence=ConfidenceLevel.HIGH, + agent_type=self.agent_type, + suggestions=suggestions, + metadata={ + "handled_directly": True, + "query_type": "visualization", + "available_visualizations": len(visualizations), + }, + ) + + def _build_visualization_prompt(self, query: str, visualizations: list[dict]) -> str: + """Build a prompt for answering visualization questions.""" + viz_list = "\n".join( + f"- **{v['title']}** (`{v['name']}`): {v.get('description', 'No description')} " + f"- URL: /visualizations/create/{v['name']}" + for v in visualizations + ) + + return f"""You are a Galaxy assistant helping users visualize their data. + +IMPORTANT: Only recommend visualizations from this list. Do NOT make up or hallucinate information about plugins not listed here. If a plugin the user asks about is not in this list, say "I don't have information about that visualization plugin" or suggest alternatives from the list. + +Available visualization plugins in this Galaxy instance: +{viz_list} + +User question: {query} + +Instructions: +- Only describe plugins that are in the list above +- If the user asks about a specific plugin, look for it in the list and provide its actual description +- If the plugin is not in the list, say so honestly +- Include visualization URLs in markdown link format: [Plugin Name](/visualizations/create/plugin_name) +- Be concise and accurate.""" + + def _build_visualization_fallback(self, query: str, visualizations: list[dict]) -> str: + """Build a fallback response when LLM is unavailable.""" + query_lower = query.lower() + + # Try to match relevant visualizations based on keywords + relevant = [] + for viz in visualizations: + viz_text = f"{viz.get('title', '')} {viz.get('description', '')} {' '.join(viz.get('keywords', []))}".lower() + if any(word in viz_text for word in query_lower.split()): + relevant.append(viz) + + if not relevant: + relevant = visualizations[:5] # Show first 5 if no matches + + if not relevant: + return "No visualization plugins are currently available. Please contact your Galaxy administrator." + + response = "Here are some visualization options that might help:\n\n" + for viz in relevant[:5]: + response += f"- [{viz['title']}](/visualizations/create/{viz['name']})" + if viz.get("description"): + response += f": {viz['description']}" + response += "\n" + + return response + async def _get_agent_plan(self, query: str) -> AgentPlan: """Get plan for which agents to call.""" try: @@ -287,3 +381,44 @@ def _get_simple_system_prompt(self) -> str: SEQUENTIAL: true REASONING: Analyze error first, then suggest creating a tool """ + + def _extract_visualization_suggestions( + self, content: str, context: dict[str, Any] + ) -> list[ActionSuggestion]: + """ + Extract visualization suggestions from response content. + + Finds visualization links in the format /visualizations/create/PLUGINNAME + and creates ActionSuggestion objects for each valid visualization. + + Args: + content: Response content to search for visualization links + context: Context dictionary containing visualizations metadata + + Returns: + List of ActionSuggestion objects for found visualizations + """ + suggestions = [] + visualizations = context.get("visualizations", []) + + # Build lookup map from visualization name to metadata + viz_map = {v["name"]: v for v in visualizations} + + # Find visualization links in response + seen = set() + for match in re.finditer(r"/visualizations/create/(\w+)", content): + viz_name = match.group(1) + if viz_name in viz_map and viz_name not in seen: + seen.add(viz_name) + viz = viz_map[viz_name] + suggestions.append( + ActionSuggestion( + action_type=ActionType.VIEW_VISUALIZATION, + description=f"Open {viz['title']} visualization", + parameters={"url": viz["url"], "plugin_name": viz_name}, + confidence=ConfidenceLevel.HIGH, + priority=1, + ) + ) + + return suggestions diff --git a/lib/galaxy/agents/prompts/orchestrator.md b/lib/galaxy/agents/prompts/orchestrator.md index 4a94f60c3e4a..c5d69be188fb 100644 --- a/lib/galaxy/agents/prompts/orchestrator.md +++ b/lib/galaxy/agents/prompts/orchestrator.md @@ -27,3 +27,16 @@ You coordinate multiple Galaxy agents for complex queries. Determine which agent - **Sequential**: When later agents need results from earlier ones - **Parallel**: When agents work on independent aspects of the query - **Hybrid**: Mix of sequential and parallel when appropriate + +## Visualization Suggestions + +When users ask about visualizing data, viewing results, or creating charts/plots, +suggest relevant plugins from the Available Visualizations section in the context. + +Format visualization links as: `[Plugin Title](/visualizations/create/plugin_name)` + +Examples: +- "How can I view my BAM file?" → Suggest IGV or similar genome browsers +- "I want to make a scatter plot" → Suggest Plotly or Charts +- "Show me my phylogenetic tree" → Suggest Phylocanvas +- "Visualize my multiple sequence alignment" → Suggest MSA viewer diff --git a/lib/galaxy/agents/prompts/router.md b/lib/galaxy/agents/prompts/router.md index 0a3669c2201b..24fba89d55cb 100644 --- a/lib/galaxy/agents/prompts/router.md +++ b/lib/galaxy/agents/prompts/router.md @@ -9,7 +9,8 @@ Focus on the user's **intent**. ## Routing Rules - For errors, failures, or debugging, route to: **error_analysis**. -- For creating new tools or tool wrappers, route to: **custom_tool**. +- For creating new Galaxy tools or tool wrappers (XML/YAML tool definitions), route to: **custom_tool**. +- For visualization questions (viewing data, charts, plots, graphs, genome browsers), route to: **orchestrator**. - For complex, multi-part queries (e.g., "fix my error AND create a tool"), route to: **orchestrator**. - For general questions or tasks that don't fit the above categories, route to: **orchestrator**. diff --git a/lib/galaxy/agents/router.py b/lib/galaxy/agents/router.py index b117b8594189..c8a7972cb204 100644 --- a/lib/galaxy/agents/router.py +++ b/lib/galaxy/agents/router.py @@ -22,6 +22,7 @@ BaseGalaxyAgent, GalaxyAgentDependencies, ) +from .visualization_context import is_visualization_query log = logging.getLogger(__name__) @@ -133,6 +134,17 @@ def _fallback_routing(self, query: str, context: Optional[dict[str, Any]] = None """Fallback routing when AI router fails - uses intent-based heuristics.""" query_lower = query.lower() + # Check for visualization keywords first - these should go to orchestrator + visualizations = (context or {}).get("visualizations") + if is_visualization_query(query, visualizations): + return RoutingDecision( + primary_agent=AgentType.ORCHESTRATOR, + secondary_agents=[], + complexity="simple", + confidence="high", + reasoning="Query relates to data visualization, routing to orchestrator for visualization suggestions.", + ) + # Define keyword sets for different intents intent_keywords = { AgentType.ERROR_ANALYSIS: ( @@ -151,14 +163,12 @@ def _fallback_routing(self, query: str, context: Optional[dict[str, Any]] = None ), AgentType.CUSTOM_TOOL: ( [ - "create", - "build", - "make", - "wrap", "custom tool", "new tool", - "yaml", + "tool wrapper", "xml definition", + "tool yaml", + "galaxy tool", ], 1.0, ), @@ -290,9 +300,10 @@ def _get_simple_system_prompt(self) -> str: Available agents: - error_analysis: For debugging, troubleshooting, job failures - - custom_tool: For creating new tools, tool development - - orchestrator: For general queries, multi-step tasks + - custom_tool: For creating new Galaxy tools (XML/YAML tool definitions) + - orchestrator: For visualizations, charts, plots, viewing data, and general queries + IMPORTANT: For visualization questions (charts, plots, graphs, viewing data), always route to orchestrator. Respond in this exact format: ROUTE_TO: [agent_name] diff --git a/lib/galaxy/agents/visualization_context.py b/lib/galaxy/agents/visualization_context.py new file mode 100644 index 000000000000..c9c7b4a44272 --- /dev/null +++ b/lib/galaxy/agents/visualization_context.py @@ -0,0 +1,131 @@ +""" +Helper functions for providing visualization plugin context to AI agents. +""" + +from typing import ( + Any, + TYPE_CHECKING, +) + +if TYPE_CHECKING: + from galaxy.managers.context import ProvidesUserContext + +# Shared keywords for identifying visualization-related queries +VISUALIZATION_KEYWORDS = [ + "visualiz", + "chart", + "plot", + "graph", + "view", + "display", + "genome browser", + "igv", + "phylo", + "tree", + "heatmap", + "scatter", + "histogram", +] + + +def is_visualization_query(query: str, visualizations: list[dict[str, Any]] | None = None) -> bool: + """ + Check if a query is about visualization. + + Args: + query: User query string + visualizations: Optional list of available visualization plugins + + Returns: + True if query appears to be visualization-related + """ + query_lower = query.lower() + + # Check for general visualization keywords + if any(keyword in query_lower for keyword in VISUALIZATION_KEYWORDS): + return True + + # Check if query mentions any known visualization plugin by name or title + if visualizations: + for viz in visualizations: + plugin_name = viz.get("name", "").lower() + plugin_title = viz.get("title", "").lower() + if plugin_name and plugin_name in query_lower: + return True + if plugin_title and plugin_title in query_lower: + return True + + return False + + +def get_visualization_summaries(trans: "ProvidesUserContext") -> list[dict[str, Any]]: + """ + Get condensed visualization plugin info for AI agent context. + + Returns a list of dictionaries containing: + - name: Plugin identifier + - title: Display name + - description: Plugin description + - keywords: Tags/keywords for the plugin + - url: URL path to create visualization + + Args: + trans: Galaxy transaction context + + Returns: + List of visualization plugin summaries, sorted by title + """ + summaries = [] + + if not hasattr(trans, "app") or not hasattr(trans.app, "visualizations_registry"): + return summaries + + registry = trans.app.visualizations_registry + if not hasattr(registry, "plugins"): + return summaries + + for name, plugin in registry.plugins.items(): + if plugin.config.get("hidden"): + continue + summaries.append({ + "name": name, + "title": plugin.config.get("name") or name, + "description": plugin.config.get("description") or "", + "keywords": plugin.config.get("tags") or [], + "url": f"/visualizations/create/{name}", + }) + + return sorted(summaries, key=lambda x: x["title"]) + + +def format_visualization_context(summaries: list[dict[str, Any]]) -> str: + """ + Format visualization summaries for inclusion in LLM prompts. + + Args: + summaries: List of visualization plugin summaries + + Returns: + Formatted string for prompt injection + """ + if not summaries: + return "" + + lines = ["## Available Visualizations", ""] + + for viz in summaries: + title = viz.get("title", viz.get("name", "Unknown")) + name = viz.get("name", "") + description = viz.get("description", "") + keywords = viz.get("keywords", []) + + line = f"- **{title}** (`{name}`)" + if description: + line += f": {description}" + if keywords: + line += f" [Keywords: {', '.join(keywords)}]" + + lines.append(line) + + lines.append("") + return "\n".join(lines) diff --git a/lib/galaxy/managers/agents.py b/lib/galaxy/managers/agents.py index 8a08a96193e0..f6a16c6f64b9 100644 --- a/lib/galaxy/managers/agents.py +++ b/lib/galaxy/managers/agents.py @@ -122,6 +122,12 @@ async def route_and_execute( if context is None: context = {} + # Inject visualization context if not already present + if "visualizations" not in context: + from galaxy.agents.visualization_context import get_visualization_summaries + + context["visualizations"] = get_visualization_summaries(trans) + # Route to appropriate agent actual_agent_type = agent_type routing_reasoning = None diff --git a/lib/galaxy/schema/agents.py b/lib/galaxy/schema/agents.py index 7175176f5a10..b0a29281de8f 100644 --- a/lib/galaxy/schema/agents.py +++ b/lib/galaxy/schema/agents.py @@ -29,6 +29,7 @@ class ActionType(str, Enum): DOCUMENTATION = "documentation" CONTACT_SUPPORT = "contact_support" VIEW_EXTERNAL = "view_external" + VIEW_VISUALIZATION = "view_visualization" SAVE_TOOL = "save_tool" REFINE_QUERY = "refine_query"