diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c87628f..192d4c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,8 +27,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r requirements-dev.txt + pip install -e ".[dev]" - name: Run pytest run: pytest tests/ -v --timeout=30 --cov=core --cov-fail-under=90 diff --git a/.gitignore b/.gitignore index d9a1253..d3ea078 100644 --- a/.gitignore +++ b/.gitignore @@ -1,213 +1,67 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[codz] -*$py.class +# Logs and temporary files +*.log +*.tmp -# C extensions -*.so +# Environment files +.env +.env.local +*.env.* -# Distribution / packaging +# Python specific +__pycache__/ +*.pyc +*.pyo +*.pyd .Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ +*.so .coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py.cover -.hypothesis/ +coverage/ +htmlcov/ .pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# UV -# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -#uv.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock -#poetry.toml - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. -# https://pdm-project.org/en/latest/usage/project/#working-with-version-control -#pdm.lock -#pdm.toml -.pdm-python -.pdm-build/ - -# pixi -# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. -#pixi.lock -# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one -# in the .venv directory. It is recommended not to include this directory in version control. -.pixi - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.envrc -.venv -.venv-wsl/ -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy .mypy_cache/ -.dmypy.json -dmypy.json -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +# Build and distribution artifacts +dist/ +build/ +*.egg-info/ +.eggs/ -# Abstra -# Abstra is an AI-powered process automation framework. -# Ignore directories containing user credentials, local state, and settings. -# Learn more at https://abstra.io/docs -.abstra/ +# Dependencies +.venv/ +venv/ +.env/ +node_modules/ -# Visual Studio Code -# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore -# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore -# and can be added to the global gitignore or merged into this file. However, if you prefer, -# you could uncomment the following to ignore the entire vscode folder +# Editors and IDEs .vscode/ - -# Ruff stuff: -.ruff_cache/ - -# PyPI configuration file -.pypirc - -# Cursor -# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to -# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data -# refer to https://docs.cursor.com/context/ignore-files -.cursorignore -.cursorindexingignore - -# Marimo -marimo/_static/ -marimo/_lsp/ -__marimo__/ - -# Orchestrator runtime files -metrics.jsonl -pending_requirements.txt -plugins_store/ +.idea/ +*.swp +*.swo + +# OS generated files +.DS_Store +Thumbs.db + +# Compression and archives +*.zip +*.gz +*.tar +*.tgz +*.bz2 +*.xz +*.7z +*.rar +*.zst +*.lz4 +*.lzh +*.cab +*.arj +*.rpm +*.deb +*.Z +*.lz +*.lzo +*.tar.gz +*.tar.bz2 +*.tar.xz +*.tar.zst diff --git a/HOLOBIONT_ROADMAP.md b/HOLOBIONT_ROADMAP.md new file mode 100644 index 0000000..7038806 --- /dev/null +++ b/HOLOBIONT_ROADMAP.md @@ -0,0 +1,513 @@ +# πŸ“‹ RawLLM to HolobiontLLM Development Roadmap + +This document describes the roadmap for transforming RawLLM into a multi-agent system with self-learning capabilities, aligned with the HolobiontLLM concept. + +## 🎯 Strategic Goal + +Transform RawLLM from a monolithic orchestrator system into an **evolving multi-agent ecosystem** where: +- Agents form dynamic communication graphs +- The system strategically plans actions via MCTS +- Accumulated experience is used for self-learning (self-play) + +--- + +## 🧱 Phase 1: Core Functionality Extension + +**Goal:** Create a foundation for multi-agent interactions with basic intelligent functions. + +### 1.1 Enhanced Tool Usage + +#### Task 1.1.1: Tool Reranking Implementation +**Files:** `core/tool_executor.py`, `core/llm/protocol.py` +**Description:** +- Add post-processing stage for LLM-selected tools +- Implement scoring mechanism for ranking tools by relevance +- Add confidence threshold for filtering low-priority calls + +**API Changes:** +```python +class ToolExecutor: + def rerank_tools( + self, + tool_calls: List[ToolCall], + context: dict + ) -> List[ToolCall]: + """Rerank tools based on context and history.""" +``` + +**Priority:** πŸ”΄ High +**Complexity:** Medium +**Dependencies:** None +**Status:** βœ… Completed + +--- + +#### Task 1.1.2: Tool Reject Option +**Files:** `core/tool_executor.py`, `core/taor_loop.py` +**Description:** +- Give the system the right to refuse tool execution +- Implement "soft rejection" mechanism with explanation +- Log rejection cases for subsequent analysis + +**API Changes:** +```python +class ToolExecutor: + def reject_tool_call( + self, + tool_call: ToolCall, + reason: str + ) -> RejectionResult: + """Reject tool execution with justification.""" +``` + +**Priority:** πŸ”΄ High +**Complexity:** Low +**Dependencies:** 1.1.1 +**Status:** βœ… Completed + +--- + +### 1.2 Self-Reflection and Self-Correction Tools + +#### Task 1.2.1: ToolReflection Cycle +**Files:** `core/tool_executor.py`, `core/metrics.py`, new: `core/reflection.py` +**Description:** +- Track tool execution errors +- Automatic generation of corrected requests +- Feedback from API/sandbox for error analysis + +**Components:** +``` +core/reflection.py +β”œβ”€β”€ ErrorAnalyzer +β”‚ β”œβ”€β”€ analyze_error(tool_call, result, traceback) +β”‚ └── categorize_error(error_type) +β”œβ”€β”€ CorrectionGenerator +β”‚ β”œβ”€β”€ generate_correction(error_analysis, original_call) +β”‚ └── validate_correction(proposed_call) +└── ReflectionLoop + β”œβ”€β”€ run_reflection_cycle(history, error_context) + └── log_reflection_event(reflection_data) +``` + +**Priority:** 🟑 Medium +**Complexity:** High +**Dependencies:** 1.1 +**Status:** ⏳ In Progress + +--- + +### 1.3 Context Prompt Repository + +#### Task 1.3.1: ContextPromptRepository Subsystem +**Files:** new: `core/context_repository.py`, modification: `core/prompt_builder.py` +**Description:** +- Store prompt templates for various task types +- Extract relevant context via semantic search +- Integrate with ProConSuL-like logic + +**Components:** +``` +core/context_repository.py +β”œβ”€β”€ ContextPromptRepository +β”‚ β”œβ”€β”€ store_prompt(template_id, prompt_template, metadata) +β”‚ β”œβ”€β”€ retrieve_prompts(query, top_k=5) +β”‚ β”œβ”€β”€ search_by_semantics(embedding_query) +β”‚ └── get_context_for_task(task_type, context_hints) +β”œβ”€β”€ PromptTemplate +β”‚ β”œβ”€β”€ template: str +β”‚ β”œβ”€β”€ variables: List[str] +β”‚ └── render(**kwargs) -> str +└── SemanticIndex + β”œβ”€β”€ build_index(prompts) + └── similarity_search(query_vector) +``` + +**Integration with prompt_builder.py:** +```python +def build_startup_prompt( + available_resources: dict | None = None, + user_task: str | None = None, + context_repository: ContextPromptRepository | None = None, +) -> str: +``` + +**Priority:** 🟑 Medium +**Complexity:** Medium +**Dependencies:** None +**Status:** ⏳ Planned + +--- + +### 1.4 Metrics and Evaluation System + +#### Task 1.4.1: Extended Event Logging +**Files:** `core/metrics.py`, `core/tool_executor.py` +**Description:** +- Add `success_score` field (0-1) for each event +- Log execution trajectories (sequence of tool calls) +- Evaluate success of multi-step operations + +**API Changes:** +```python +def log_execution(...): + # Add parameters: + success_score: float, # 0.0 - 1.0 + trajectory_id: str, # ID of action sequence + step_number: int, # Step number in trajectory +``` + +**Priority:** πŸ”΄ High +**Complexity:** Low +**Dependencies:** None +**Status:** βœ… Completed + +--- + +#### Task 1.4.2: RecVAE for Agent Recommendation +**Files:** new: `core/agent_recommender.py`, modification: `core/metrics.py` +**Description:** +- Use success history to select agent committee composition +- Simple VAE architecture for encoding trajectories +- Recommend optimal agents for new tasks + +**Components:** +``` +core/agent_recommender.py +β”œβ”€β”€ TrajectoryEncoder +β”‚ β”œβ”€β”€ encode_trajectory(events_list) -> latent_vector +β”‚ └── decode_vector(latent_vector) -> trajectory_pattern +β”œβ”€β”€ AgentRecommender +β”‚ β”œβ”€β”€ train_on_history(metrics_events) +β”‚ β”œβ”€β”€ recommend_agents(task_description, context) +β”‚ └── get_agent_success_rate(agent_id, task_type) +└── CommitteeBuilder + β”œβ”€β”€ build_committee(recommendations, constraints) + └── optimize_committee_composition(candidate_agents) +``` + +**Priority:** 🟒 Low (for Phase 1) +**Complexity:** Very High +**Dependencies:** 1.4.1 +**Status:** ⏳ Planned + +--- + +## 🌿 Phase 2: Advanced Methods (HolobiontLLM) + +**Goal:** Implement key principles of the HolobiontLLM concept β€” strategic planning and self-learning. + +### 2.1 Agent Communication Graph + +#### Task 2.1.1: Dynamic Routing System +**Files:** new: `core/agent_graph.py`, modification: `core/taor_loop.py` +**Description:** +- Create dynamic agent graph (Planner, Coder, Critic, Executor) +- Route calls on-the-fly depending on task +- Support cyclic dependencies and feedback loops + +**Components:** +``` +core/agent_graph.py +β”œβ”€β”€ AgentNode +β”‚ β”œβ”€β”€ agent_id: str +β”‚ β”œβ”€β”€ role: str # planner, coder, critic, executor +β”‚ β”œβ”€β”€ capabilities: List[str] +β”‚ └── current_load: float +β”œβ”€β”€ CommunicationGraph +β”‚ β”œβ”€β”€ add_agent(agent_node) +β”‚ β”œβ”€β”€ remove_agent(agent_id) +β”‚ β”œβ”€β”€ route_request(source_id, target_ids, message) +β”‚ β”œβ”€β”€ build_dynamic_graph(task_requirements) +β”‚ └── get_optimal_path(start_agent, end_goal) +└── GraphRouter + β”œβ”€β”€ dispatch_to_agents(request, graph_config) + β”œβ”€β”€ collect_responses(timeout) + └── aggregate_results(responses) +``` + +**Integration with taor_loop.py:** +```python +class TAORLoop: + def __init__(self, ..., agent_graph: CommunicationGraph | None = None): + self._agent_graph = agent_graph + + async def process_request_async(self, ...): + if self._agent_graph: + return await self._process_with_graph(...) + else: + return await self._process_single(...) +``` + +**Priority:** πŸ”΄ High (key for Holobiont) +**Complexity:** Very High +**Dependencies:** Phase 1 complete +**Status:** ⏳ Planned + +--- + +#### Task 2.1.2: Specialized Agents +**Files:** new: `core/agents/` package +**Description:** +- Planner: complex task decomposition +- Coder: code generation and refactoring +- Critic: result validation, error detection +- Executor: tool execution +- Meta-Agent: past error analysis, coordination + +**Structure:** +``` +core/agents/ +β”œβ”€β”€ __init__.py +β”œβ”€β”€ base_agent.py # AbstractAgent base class +β”œβ”€β”€ planner.py # PlannerAgent +β”œβ”€β”€ coder.py # CoderAgent +β”œβ”€β”€ critic.py # CriticAgent +β”œβ”€β”€ executor.py # ExecutorAgent +└── meta_agent.py # MetaAgent (coordination + learning) +``` + +**Priority:** πŸ”΄ High +**Complexity:** High +**Dependencies:** 2.1.1 +**Status:** ⏳ Planned + +--- + +### 2.2 MCTS as Action Planner + +#### Task 2.2.1: MCTS Module +**Files:** new: `core/mcts_planner.py` +**Description:** +- Monte Carlo Tree Search for strategic planning +- Build tree of possible agent call sequences +- Evaluate branch promisingness based on simulations + +**Components:** +``` +core/mcts_planner.py +β”œβ”€β”€ MCTSNode +β”‚ β”œβ”€β”€ state: AgentGraphState +β”‚ β”œβ”€β”€ action: AgentCall | None +β”‚ β”œβ”€β”€ visits: int +β”‚ β”œβ”€β”€ value: float +β”‚ └── children: Dict[action, MCTSNode] +β”œβ”€β”€ MCTSPlanner +β”‚ β”œβ”€β”€ select(node) -> node +β”‚ β”œβ”€β”€ expand(node) -> new_nodes +β”‚ β”œβ”€β”€ simulate(state) -> reward +β”‚ β”œβ”€β”€ backpropagate(path, reward) +β”‚ └── plan(initial_state, n_iterations=1000) -> best_action_sequence +└── StateEvaluator + β”œβ”€β”€ evaluate_state(state) -> float + └── heuristic_value(partial_trajectory) +``` + +**Algorithm:** +```python +def mcts_plan(initial_state, n_iterations=1000): + root = MCTSNode(initial_state) + + for _ in range(n_iterations): + node = root + state = initial_state.copy() + + # Selection + while node.is_fully_expanded(): + node = node.select_best_child() + state = state.apply_action(node.action) + + # Expansion + if not state.is_terminal(): + actions = state.get_legal_actions() + for action in actions: + new_state = state.apply_action(action) + node.add_child(action, new_state) + + # Simulation + reward = simulate_random_rollout(state) + + # Backpropagation + node.backpropagate(reward) + + return root.get_best_action() +``` + +**Priority:** 🟑 Medium +**Complexity:** Very High +**Dependencies:** 2.1 +**Status:** ⏳ Planned + +--- + +### 2.3 Training Loop + +#### Task 2.3.1: Self-play Infrastructure +**Files:** new: `core/training/` package, modification: `core/mcts_planner.py` +**Description:** +- Generate training data from MCTS trajectories +- Update router/Meta-Agent based on successful trajectories +- Iterative improvement of agent selection strategy + +**Components:** +``` +core/training/ +β”œβ”€β”€ __init__.py +β”œβ”€β”€ trajectory_collector.py +β”‚ └── TrajectoryCollector +β”‚ β”œβ”€β”€ record_mcts_trajectory(trajectory) +β”‚ β”œβ”€β”€ label_trajectory(success_metric) +β”‚ └── export_training_dataset() +β”œβ”€β”€ model_updater.py +β”‚ └── ModelUpdater +β”‚ β”œβ”€β”€ update_router_policy(trajectories) +β”‚ β”œβ”€β”€ update_agent_embeddings(success_patterns) +β”‚ └── save_checkpoint(model_state) +└── self_play_loop.py + └── SelfPlayLoop + β”œβ”€β”€ run_episode() -> trajectory + β”œβ”€β”€ evaluate_episode(trajectory) -> reward + └── train_on_batch(trajectories) +``` + +**Priority:** 🟒 Low (most complex stage) +**Complexity:** Extreme +**Dependencies:** 2.2, all Phase 1 and 2 +**Status:** ⏳ Planned + +--- + +## πŸ“Š Priority Summary Table + +| # | Task | Priority | Complexity | Time Estimate | +|---|------|----------|------------|---------------| +| 1.1.1 | Tool Reranking | πŸ”΄ High | Medium | 3-5 days | +| 1.1.2 | Reject Option | πŸ”΄ High | Low | 1-2 days | +| 1.2.1 | ToolReflection Cycle | 🟑 Medium | High | 7-10 days | +| 1.3.1 | Context Repository | 🟑 Medium | Medium | 4-6 days | +| 1.4.1 | Extended Metrics | πŸ”΄ High | Low | 2-3 days | +| 1.4.2 | RecVAE Recommender | 🟒 Low | Very High | 14-21 days | +| 2.1.1 | Agent Graph | πŸ”΄ High | Very High | 10-14 days | +| 2.1.2 | Specialized Agents | πŸ”΄ High | High | 7-10 days | +| 2.2.1 | MCTS Planner | 🟑 Medium | Very High | 14-21 days | +| 2.3.1 | Training Loop | 🟒 Low | Extreme | 21-30 days | + +--- + +## πŸ—ΊοΈ Sprint Roadmap + +### Sprint 1 (Weeks 1-2): Phase 1 Foundation +- βœ… 1.1.1 Tool Reranking β€” **Completed** +- βœ… 1.1.2 Reject Option β€” **Completed** +- βœ… 1.4.1 Extended Metrics β€” **Completed** + +**Deliverable:** Basic tool reranking and rejection system, extended metrics with success_score and trajectory_id. Implemented in `core/tool_management.py` and `core/metrics.py`. + +--- + +### Sprint 2 (Weeks 3-4): Phase 1 Intelligence +- ⏳ 1.3.1 Context Repository β€” **Planned** +- ⏳ 1.2.1 ToolReflection Cycle (start) β€” **In Progress** + +**Goal:** Context prompt repository and beginning of self-reflection cycle implementation. + +--- + +### Sprint 3 (Weeks 5-6): Phase 1 Completion +- ⏳ 1.2.1 ToolReflection Cycle (completion) β€” **Planned** +- ⏳ Start 2.1.1 Agent Graph (design) β€” **Planned** + +**Goal:** Full ToolReflection implementation and agent communication graph design. + +--- + +### Sprint 4-5 (Weeks 7-10): Phase 2 Core +- ⏳ 2.1.1 Agent Graph (implementation) β€” **Planned** +- ⏳ 2.1.2 Specialized Agents β€” **Planned** + +**Goal:** Dynamic agent graph and specialized roles (Planner, Coder, Critic, Executor). + +--- + +### Sprint 6-8 (Weeks 11-16): Planning and Learning +- ⏳ 2.2.1 MCTS Planner β€” **Planned** +- ⏳ 2.3.1 Training Loop (prototype) β€” **Planned** + +**Goal:** MCTS for strategic planning and self-play learning prototype. + +--- + +### Sprint 9+ (Weeks 17+): Optimization and Scaling +- ⏳ 1.4.2 RecVAE (if required) β€” **Planned** +- ⏳ Polishing, testing, documentation β€” **Planned** + +--- + +## πŸ”§ Technical Requirements + +### New Dependencies +```txt +# For semantic search in context repository +sentence-transformers>=2.2.0 + +# For VAE in RecVAE +torch>=2.0.0 +scikit-learn>=1.0.0 + +# For graph visualization (optional) +networkx>=2.8.0 +pyvis>=0.3.0 +``` + +### Testing Requirements +- Test coverage β‰₯80% for all new modules +- Integration tests for multi-agent scenarios +- Load tests for MCTS (planning time verification) + +### Monitoring and Observability +- Log all MCTS decisions +- Real-time agent success metrics +- Communication graph visualization + +--- + +## πŸ“ˆ Success Criteria + +### Phase 1 Completion Criteria +- [x] All tools go through reranking +- [x] System can argumentatively reject a tool +- [ ] ToolReflection cycle works automatically on errors +- [ ] Context Repository provides relevant prompts +- [x] Metrics include success_score and trajectory_id + +### Phase 2 Completion Criteria +- [ ] Dynamic graph of 3+ agents assembled and executes task +- [ ] MCTS finds optimal sequence of 5+ steps +- [ ] Self-play improves success rate by 20% after 100 episodes +- [ ] Meta-Agent analyzes errors and suggests corrections + +--- + +## ⚠️ Risks and Mitigation + +| Risk | Probability | Impact | Mitigation | +|------|-------------|---------|------------| +| MCTS complexity exceeds estimate | High | High | Start with simplified version (depth-limited) | +| Performance degradation | Medium | High | Result caching, asynchronicity | +| Overfitting in self-play | Medium | Medium | Regularization, diversity in simulations | +| Graph debugging complexity | High | Medium | Detailed logging, visualization | + +--- + +## πŸ“š Additional Materials + +- [MemPalace Paper](https://arxiv.org/abs/...) β€” long-context memory management +- [Claude Code TAOR](https://claude.ai/code) β€” reference architecture +- [MCTS Survey](https://arxiv.org/abs/...) β€” comprehensive MCTS review +- [Multi-Agent Systems](https://www.mas-book.org/) β€” foundational concepts + +--- + +*Document created: 2025* +*Version: 1.0* +*Status: Planning* diff --git a/README.md b/README.md index 67aad88..700bdcb 100644 --- a/README.md +++ b/README.md @@ -20,18 +20,24 @@ The core (orchestrator) is **immutable** and deliberately "dumb" (~150 lines). A ## Status -βœ… **Implemented** β€” core, HTTP plugin, tests and CI. +βœ… **Core & HTTP Plugin** β€” implemented, tested, CI passing. +βœ… **Sprint 1 (Multi-Agent Foundation)** β€” advanced metrics, tool reranking, and rejection handling completed. +⏳ **Sprint 2 (Context & Reflection)** β€” in progress: context repository and self-correction loop. ## Quick start ```bash -# 1. Install dependencies -pip install -r requirements.txt +# 1. Install dependencies via Poetry or pip +poetry install +# or +pip install -e . # 2. Create .env with your Anthropic API key echo "ANTHROPIC_API_KEY=sk-ant-..." > .env # 3. Start the orchestrator (HTTP server on port 8080) +poetry run python run.py +# or python run.py # 4. Send a request @@ -193,11 +199,14 @@ rawllm deps reject requests # reject a module ``` ### Metrics & analytics + ```bash rawllm metrics show # all plugins, table format rawllm metrics show --plugin my_plugin # one plugin rawllm metrics show --format json # JSON output rawllm metrics evolution my_plugin # chronological timeline +rawllm metrics trajectory # view specific execution trajectory +rawllm metrics success-rate # aggregate success scores ``` ### Configuration @@ -233,8 +242,8 @@ rawllm/ β”‚ β”œβ”€β”€ tool_executor.py # Tool-call routing + dependency gating β”‚ β”œβ”€β”€ taor_loop.py # Think β†’ Act β†’ Observe β†’ Repeat loop β”‚ β”œβ”€β”€ config.py # Settings: trusted_plugins, allowed_requirements -β”‚ β”œβ”€β”€ metrics.py # Event logging to metrics.jsonl -β”‚ β”œβ”€β”€ sandbox_wrapper.py # Isolated subprocess wrapper for untrusted plugins +β”‚ β”œβ”€β”€ tool_management.py # Tool reranking and rejection handling (Sprint 1) +β”‚ β”œβ”€β”€ metrics.py # Event logging with success_score and trajectory tracking β”‚ └── utils.py # Shared utilities + extract_imports β”œβ”€β”€ plugins/ β”‚ └── http.py # HTTP transport plugin (port set via HTTP_PORT) @@ -250,3 +259,4 @@ rawllm/ MIT β€” use the ideas freely, fork, and improve. +└── HOLOBIONT_ROADMAP.md # Development roadmap and future phases diff --git a/core/metrics.py b/core/metrics.py index 3b5b924..dbee6af 100644 --- a/core/metrics.py +++ b/core/metrics.py @@ -35,21 +35,43 @@ def log_execution( traceback_str: str | None = None, import_risk_score: int = 0, metrics_file: Path | None = None, + success_score: float | None = None, + trajectory_id: str | None = None, + step_number: int | None = None, ) -> None: - """Log a single plugin execution.""" - log_event( - "plugin_execution", - { - "plugin_name": plugin_name, - "version": version, - "execution_time_ms": execution_time_ms, - "success": success, - "error_type": error_type, - "traceback": traceback_str, - "import_risk_score": import_risk_score, - }, - metrics_file=metrics_file, - ) + """Log a single plugin execution. + + Args: + plugin_name: Name of the executed plugin. + version: Plugin version. + execution_time_ms: Execution time in milliseconds. + success: Whether execution was successful. + error_type: Type of error if failed. + traceback_str: Full traceback string if failed. + import_risk_score: Risk score from import analysis. + success_score: Float score 0.0-1.0 indicating degree of success. + trajectory_id: Unique ID for multi-step operation sequence. + step_number: Position in the trajectory (1-indexed). + """ + data: dict[str, Any] = { + "plugin_name": plugin_name, + "version": version, + "execution_time_ms": execution_time_ms, + "success": success, + "error_type": error_type, + "traceback": traceback_str, + "import_risk_score": import_risk_score, + } + + # Add extended tracking fields + if success_score is not None: + data["success_score"] = max(0.0, min(1.0, success_score)) + if trajectory_id is not None: + data["trajectory_id"] = trajectory_id + if step_number is not None: + data["step_number"] = step_number + + log_event("plugin_execution", data, metrics_file=metrics_file) def log_version_change( @@ -148,6 +170,8 @@ def aggregate_by_plugin( "rollbacks": int, "dependency_requests": int, "import_risk_score": int, # latest recorded score + "avg_success_score": float, # average success_score (0.0-1.0) + "trajectory_count": int, # number of unique trajectories } """ events = get_events(plugin_name=plugin_name, metrics_file=metrics_file) @@ -165,6 +189,9 @@ def _plugin_stats(name: str) -> dict[str, Any]: "rollbacks": 0, "dependency_requests": 0, "import_risk_score": 0, + "total_success_score": 0.0, + "success_score_count": 0, + "trajectories": set(), } for entry in events: @@ -182,6 +209,16 @@ def _plugin_stats(name: str) -> dict[str, Any]: s["failed"] += 1 s["total_exec_ms"] += entry.get("execution_time_ms", 0.0) s["import_risk_score"] = entry.get("import_risk_score", s["import_risk_score"]) + + # Extended success_score tracking + if "success_score" in entry: + s["total_success_score"] += entry["success_score"] + s["success_score_count"] += 1 + + # Extended trajectory tracking + if "trajectory_id" in entry: + s["trajectories"].add(entry["trajectory_id"]) + elif etype == "version_change": s["version_changes"] += 1 elif etype == "rollback": @@ -189,10 +226,23 @@ def _plugin_stats(name: str) -> dict[str, Any]: elif etype == "dependency_request": s["dependency_requests"] += 1 - # Compute derived fields and remove internal accumulator. + # Compute derived fields and remove internal accumulators. for s in stats.values(): total = s["total_executions"] s["avg_exec_ms"] = s["total_exec_ms"] / total if total > 0 else 0.0 + + # Compute avg_success_score + if s["success_score_count"] > 0: + s["avg_success_score"] = s["total_success_score"] / s["success_score_count"] + else: + s["avg_success_score"] = 0.0 + + # Count unique trajectories + s["trajectory_count"] = len(s["trajectories"]) + del s["total_exec_ms"] + del s["total_success_score"] + del s["success_score_count"] + del s["trajectories"] return stats diff --git a/core/taor_loop.py b/core/taor_loop.py index f284dc7..20b0292 100644 --- a/core/taor_loop.py +++ b/core/taor_loop.py @@ -33,6 +33,7 @@ from core.llm.protocol import LLMClientProtocol from core.tool_executor import ToolExecutor +from core.tool_management import ToolReranker, ToolRejectionHandler logger = logging.getLogger(__name__) @@ -162,6 +163,10 @@ def __init__( self._startup_prompt = startup_prompt self._max_iterations = max_iterations + # Initialize tool management components (Sprint 1) + self._reranker = ToolReranker() + self._rejection_handler = ToolRejectionHandler() + def process_request( self, user_prompt: str | None = None, @@ -209,7 +214,61 @@ async def process_request_async( # tool_calls branch tool_calls = response["tool_calls"] - # Append the assistant's tool-call turn in OpenAI format. + # Sprint 1: Apply reranking and rejection handling before execution + context_for_reranking = {"messages": messages, "iteration": iteration} + reranked_calls, scores = self._reranker.rerank_tools( + tool_calls, context_for_reranking + ) + + # Filter out rejected calls + accepted_calls = [] + for call, score in zip(reranked_calls, scores): + # Convert to dict format expected by rejection handler + call_dict = {"name": call["name"], "input": call["input"]} + rejection_result = self._rejection_handler.reject_tool_call( + call_dict, confidence=score.confidence + ) + + if not rejection_result.rejected: + accepted_calls.append(call) + logger.debug( + "Accepted tool call '%s' with score %.2f", + call["name"], + score.reranked_score, + ) + else: + logger.warning( + "Rejected tool call '%s': %s", + call["name"], + rejection_result.explanation, + ) + # Append rejection as a tool result message for LLM feedback + messages.append( + { + "role": "tool", + "tool_call_id": call["id"], + "content": json.dumps( + {"error": "Rejected", "reason": rejection_result.explanation}, + ensure_ascii=False, + ), + } + ) + + if not accepted_calls: + # All calls rejected - prompt LLM to reconsider + logger.info("All tool calls rejected in iteration %d", iteration + 1) + messages.append( + { + "role": "system", + "content": ( + "All proposed tool calls were rejected. " + "Please reconsider your approach based on the feedback above." + ), + } + ) + continue + + # Append the assistant's tool-call turn in OpenAI format for accepted calls only messages.append( { "role": "assistant", @@ -223,16 +282,21 @@ async def process_request_async( "arguments": json.dumps(tc["input"], ensure_ascii=False), }, } - for tc in tool_calls + for tc in accepted_calls ], } ) - # Execute all tool calls in parallel and append individual tool-result messages. + # Execute all accepted tool calls in parallel results = await asyncio.gather( - *[self._dispatch_async(call["name"], call["input"]) for call in tool_calls] + *[self._dispatch_async(call["name"], call["input"]) for call in accepted_calls] ) - for call, result in zip(tool_calls, results): + + # Update success rates for reranker history + for call, result in zip(accepted_calls, results): + success = "error" not in result + self._reranker.update_success_rate(call["name"], success) + messages.append( { "role": "tool", diff --git a/core/tool_management.py b/core/tool_management.py new file mode 100644 index 0000000..ad36d19 --- /dev/null +++ b/core/tool_management.py @@ -0,0 +1,463 @@ +"""Tool management enhancements for multi-agent systems. + +This module provides: +- Tool reranking based on context and confidence scoring +- Reject option for declining tool execution with explanations +""" + +import logging +from collections import deque +from dataclasses import dataclass, field +from enum import Enum, auto +from typing import Any + +logger = logging.getLogger(__name__) + + +class RejectionReason(Enum): + """Reasons for rejecting a tool call.""" + + LOW_CONFIDENCE = auto() + """Tool selection confidence below threshold.""" + + UNSAFE_PARAMETERS = auto() + """Parameters detected as potentially harmful.""" + + DUPLICATE_CALL = auto() + """Same tool was recently called with same parameters.""" + + RESOURCE_CONSTRAINT = auto() + """System resources insufficient for this operation.""" + + POLICY_VIOLATION = auto() + """Call violates configured policies.""" + + CONTEXT_MISMATCH = auto() + """Tool not appropriate for current context.""" + + MANUAL_REJECT = auto() + """Explicit manual rejection by system/orchestrator.""" + + +@dataclass +class ToolCallScore: + """Score assigned to a tool call during reranking.""" + + tool_name: str + original_rank: int + reranked_score: float # 0.0 - 1.0 + confidence: float # 0.0 - 1.0 + factors: dict[str, float] = field(default_factory=dict) + """Breakdown of scoring factors (e.g., relevance, recency, success_rate).""" + + def to_dict(self) -> dict[str, Any]: + return { + "tool_name": self.tool_name, + "original_rank": self.original_rank, + "reranked_score": self.reranked_score, + "confidence": self.confidence, + "factors": self.factors, + } + + +@dataclass +class RejectionResult: + """Result of rejecting a tool call.""" + + rejected: bool + reason: RejectionReason | None = None + explanation: str = "" + alternative_suggestion: str | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "rejected": self.rejected, + "reason": self.reason.name if self.reason else None, + "explanation": self.explanation, + "alternative_suggestion": self.alternative_suggestion, + } + + +class ToolReranker: + """Reranks tool calls based on context, history, and confidence. + + Provides intelligent reranking of tool candidates using multiple factors + including success history, context relevance, and recency. + """ + + def __init__( + self, + confidence_threshold: float = 0.3, + success_history_weight: float = 0.4, + relevance_weight: float = 0.4, + recency_weight: float = 0.2, + ) -> None: + """Initialize the reranker. + + Args: + confidence_threshold: Minimum confidence to keep a tool call. + success_history_weight: Weight for historical success rate. + relevance_weight: Weight for contextual relevance. + recency_weight: Weight for recent usage patterns. + """ + self._confidence_threshold = confidence_threshold + self._success_history_weight = success_history_weight + self._relevance_weight = relevance_weight + self._recency_weight = recency_weight + + # History tracking using deque for O(1) operations + self._success_rates: dict[str, float] = {} + self._recent_calls: deque[str] = deque(maxlen=100) + self._max_recent = 100 + + def update_success_rate(self, tool_name: str, success: bool) -> None: + """Update the success rate for a tool. + + Args: + tool_name: Name of the tool. + success: Whether the execution was successful. + """ + if tool_name not in self._success_rates: + self._success_rates[tool_name] = 0.5 # Default prior + + # Exponential moving average + alpha = 0.1 + current = self._success_rates[tool_name] + new_value = 1.0 if success else 0.0 + self._success_rates[tool_name] = (1 - alpha) * current + alpha * new_value + + def record_call(self, tool_name: str) -> None: + """Record a tool call for recency tracking. + + Deque with maxlen automatically handles size limits. + """ + self._recent_calls.append(tool_name) + + def compute_recency_score(self, tool_name: str) -> float: + """Compute recency score based on recent call frequency. + + Returns higher scores for tools called recently (assumes useful momentum). + """ + if not self._recent_calls: + return 0.5 + + count = self._recent_calls.count(tool_name) + # Normalize by window size + return min(1.0, count / 10.0) # Cap at 10 calls + + def rerank_tools( + self, + tool_calls: list[dict[str, Any]], + context: dict[str, Any] | None = None, + ) -> tuple[list[dict[str, Any]], list[ToolCallScore]]: + """Rerank tool calls based on context and history. + + Args: + tool_calls: List of tool call dicts from LLM. + context: Optional context for relevance scoring. + + Returns: + Tuple of (reranked_tool_calls, scores). + Tools below confidence threshold are filtered out. + """ + if not tool_calls: + return [], [] + + scores: list[ToolCallScore] = [] + + for idx, call in enumerate(tool_calls): + tool_name = call.get("name", "unknown") + + # Factor 1: Historical success rate + success_rate = self._success_rates.get(tool_name, 0.5) + + # Factor 2: Recency score + recency_score = self.compute_recency_score(tool_name) + + # Factor 3: Contextual relevance (simplified - can be enhanced with embeddings) + relevance_score = self._compute_relevance(tool_name, context or {}) + + # Weighted combination + final_score = ( + self._success_history_weight * success_rate + + self._recency_weight * recency_score + + self._relevance_weight * relevance_score + ) + + # Confidence is based on score magnitude and consistency + confidence = min(1.0, final_score + 0.2) # Small boost + + score = ToolCallScore( + tool_name=tool_name, + original_rank=idx, + reranked_score=final_score, + confidence=confidence, + factors={ + "success_rate": success_rate, + "recency": recency_score, + "relevance": relevance_score, + }, + ) + scores.append(score) + + # Filter by confidence threshold + accepted_calls = [] + accepted_scores = [] + + for call, score in zip(tool_calls, scores): + if score.confidence >= self._confidence_threshold: + accepted_calls.append(call) + accepted_scores.append(score) + else: + logger.info( + "Filtered tool call '%s': confidence %.2f < threshold %.2f", + score.tool_name, + score.confidence, + self._confidence_threshold, + ) + + # Sort by reranked score (descending) + if accepted_calls: + sorted_pairs = sorted( + zip(accepted_calls, accepted_scores), + key=lambda x: x[1].reranked_score, + reverse=True, + ) + accepted_calls = [c for c, _ in sorted_pairs] + accepted_scores = [s for _, s in sorted_pairs] + + return accepted_calls, accepted_scores + + def _compute_relevance(self, tool_name: str, context: dict[str, Any]) -> float: + """Compute contextual relevance score. + + Searches for keywords in both keys and values of the context dictionary. + """ + # Basic heuristics based on tool name patterns + tool_lower = tool_name.lower() + + # Convert context to searchable string (keys + values) + context_text = " ".join( + [str(k).lower() + " " + str(v).lower() for k, v in context.items()] + ) + + # Check if context hints match tool capabilities + if "code" in context_text or "programming" in context_text: + if any(kw in tool_lower for kw in ["code", "exec", "run", "python"]): + return 0.9 + + if "file" in context_text or "read" in context_text: + if any(kw in tool_lower for kw in ["read", "load", "file"]): + return 0.8 + + if "network" in context_text or "http" in context_text: + if any(kw in tool_lower for kw in ["http", "request", "fetch"]): + return 0.9 + + # Default neutral score + return 0.5 + + +class ToolRejectionHandler: + """Handles rejection of tool calls with explanations. + + Provides automatic detection of problematic tool calls and generates + helpful feedback with alternative suggestions. + """ + + def __init__( + self, + auto_reject_low_confidence: bool = True, + confidence_threshold: float = 0.2, + max_duplicate_window: int = 5, + ) -> None: + """Initialize the rejection handler. + + Args: + auto_reject_low_confidence: Automatically reject low-confidence calls. + confidence_threshold: Threshold for auto-rejection. + max_duplicate_window: Number of recent calls to check for duplicates. + """ + self._auto_reject = auto_reject_low_confidence + self._confidence_threshold = confidence_threshold + self._recent_calls: deque[dict[str, Any]] = deque(maxlen=max_duplicate_window) + + # Policy rules (can be extended) + self._blocked_tools: set[str] = set() + self._parameter_constraints: dict[str, list[str]] = {} + + def block_tool(self, tool_name: str) -> None: + """Block a tool from being executed.""" + self._blocked_tools.add(tool_name) + logger.warning("Tool '%s' has been blocked by policy", tool_name) + + def unblock_tool(self, tool_name: str) -> None: + """Unblock a previously blocked tool.""" + self._blocked_tools.discard(tool_name) + + def add_parameter_constraint(self, tool_name: str, forbidden_params: list[str]) -> None: + """Add parameter constraints for a tool.""" + self._parameter_constraints[tool_name] = forbidden_params + + def check_duplicate(self, tool_call: dict[str, Any]) -> bool: + """Check if this is a duplicate of a recent call. + + Deque automatically maintains window size via maxlen. + """ + for recent in self._recent_calls: + if ( + recent.get("name") == tool_call.get("name") + and recent.get("input") == tool_call.get("input") + ): + return True + return False + + def check_parameters(self, tool_call: dict[str, Any]) -> tuple[bool, str | None]: + """Check if parameters violate any constraints. + + Returns: + Tuple of (is_safe, violation_description). + """ + tool_name = tool_call.get("name", "") + params = tool_call.get("input", {}) + + if tool_name in self._parameter_constraints: + forbidden = self._parameter_constraints[tool_name] + for param in params: + if param in forbidden: + return False, f"Parameter '{param}' is forbidden for tool '{tool_name}'" + + return True, None + + def reject_tool_call( + self, + tool_call: dict[str, Any], + reason: RejectionReason | None = None, + custom_explanation: str | None = None, + confidence: float | None = None, + ) -> RejectionResult: + """Reject a tool call with explanation. + + Args: + tool_call: The tool call to potentially reject. + reason: Reason for rejection (auto-detected if None). + custom_explanation: Custom explanation override. + confidence: Confidence score for auto-rejection logic. + + Returns: + RejectionResult indicating whether and why the call was rejected. + """ + tool_name = tool_call.get("name", "unknown") + + # Auto-detect rejection reasons + if reason is None: + # Check if tool is blocked + if tool_name in self._blocked_tools: + reason = RejectionReason.POLICY_VIOLATION + custom_explanation = f"Tool '{tool_name}' is blocked by administrator policy." + + # Check for duplicates + elif self.check_duplicate(tool_call): + reason = RejectionReason.DUPLICATE_CALL + custom_explanation = ( + f"This is a duplicate call to '{tool_name}' with identical parameters. " + "Consider using the previous result or modifying the input." + ) + + # Check parameter constraints + is_safe, violation = self.check_parameters(tool_call) + if not is_safe: + reason = RejectionReason.UNSAFE_PARAMETERS + custom_explanation = violation + + # Check confidence + elif ( + self._auto_reject + and confidence is not None + and confidence < self._confidence_threshold + ): + reason = RejectionReason.LOW_CONFIDENCE + custom_explanation = ( + f"Tool selection confidence ({confidence:.2f}) is below " + f"threshold ({self._confidence_threshold}). " + "Consider reformulating the request or choosing a different approach." + ) + + # If no rejection reason, accept the call + if reason is None: + # Record for duplicate tracking (deque auto-manages size) + self._recent_calls.append(tool_call.copy()) + + return RejectionResult(rejected=False) + + # Build explanation + explanation = custom_explanation or f"Tool call rejected: {reason.name}" + + # Generate alternative suggestion + alternative = self._suggest_alternative(tool_call, reason) + + logger.info( + "Rejected tool call '%s': %s", + tool_name, + explanation, + ) + + return RejectionResult( + rejected=True, + reason=reason, + explanation=explanation, + alternative_suggestion=alternative, + ) + + def _suggest_alternative( + self, + tool_call: dict[str, Any], + reason: RejectionReason, + ) -> str | None: + """Suggest an alternative action when rejecting.""" + tool_name = tool_call.get("name", "") + + if reason == RejectionReason.DUPLICATE_CALL: + return "Use the result from the previous identical call instead." + + elif reason == RejectionReason.LOW_CONFIDENCE: + return ( + "Try breaking down your request into smaller steps, " + "or explicitly specify which tool should be used." + ) + + elif reason == RejectionReason.UNSAFE_PARAMETERS: + return "Review the tool documentation for allowed parameters." + + elif reason == RejectionReason.POLICY_VIOLATION: + return f"Tool '{tool_name}' is not available. Consider alternative approaches." + + return None + + def process_with_rejection( + self, + tool_calls: list[dict[str, Any]], + scores: list[ToolCallScore] | None = None, + ) -> tuple[list[dict[str, Any]], list[RejectionResult]]: + """Process tool calls through rejection logic. + + Args: + tool_calls: List of tool calls to process. + scores: Optional scores from reranker for confidence-based rejection. + + Returns: + Tuple of (accepted_calls, rejection_results). + """ + accepted = [] + results = [] + + for idx, call in enumerate(tool_calls): + confidence = scores[idx].confidence if scores and idx < len(scores) else None + + result = self.reject_tool_call(call, confidence=confidence) + results.append(result) + + if not result.rejected: + accepted.append(call) + + return accepted, results diff --git a/pyproject.toml b/pyproject.toml index 4f81914..6bbeb3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,15 @@ Changelog = "https://github.com/cherninkiy/rawllm/blob/main/CHANGELOG.md" [project.scripts] rawllm = "cli:cli" +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "pytest-cov>=4.0.0", + "pytest-timeout>=2.0.0", + "flake8>=6.0.0", + "mypy>=1.0.0", +] + [tool.setuptools.packages.find] where = ["."] include = ["core*", "plugins*"] diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index cd74693..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,6 +0,0 @@ -pytest>=7.0 -pytest-timeout -pytest-cov -mypy -flake8 -requests diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 5337ab6..0000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -anthropic>=0.30.0 -httpx>=0.27.0 -python-dotenv>=1.0.0 -click>=8.0.0 diff --git a/tests/test_metrics.py b/tests/test_metrics.py index cdd53ae..9944a87 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -181,3 +181,100 @@ def test_aggregate_all_plugins(metrics_file: Path) -> None: assert "plugin_b" in result assert result["plugin_a"]["successful"] == 1 assert result["plugin_b"]["failed"] == 1 + + +# --------------------------------------------------------------------------- +# Extended metrics: success_score and trajectory tracking +# --------------------------------------------------------------------------- + + +def test_log_execution_with_success_score(metrics_file: Path) -> None: + metrics.log_execution( + plugin_name="my_plugin", + version="v1", + execution_time_ms=42.5, + success=True, + success_score=0.85, + metrics_file=metrics_file, + ) + entries = metrics.get_events(metrics_file=metrics_file) + assert len(entries) == 1 + e = entries[0] + assert e["success_score"] == 0.85 + + +def test_log_execution_with_trajectory(metrics_file: Path) -> None: + metrics.log_execution( + plugin_name="my_plugin", + version="v1", + execution_time_ms=42.5, + success=True, + trajectory_id="traj_123", + step_number=2, + metrics_file=metrics_file, + ) + entries = metrics.get_events(metrics_file=metrics_file) + assert len(entries) == 1 + e = entries[0] + assert e["trajectory_id"] == "traj_123" + assert e["step_number"] == 2 + + +def test_log_execution_success_score_clamped(metrics_file: Path) -> None: + """Test that success_score is clamped to [0.0, 1.0].""" + metrics.log_execution( + plugin_name="my_plugin", + version="v1", + execution_time_ms=10.0, + success=True, + success_score=1.5, # Should be clamped to 1.0 + metrics_file=metrics_file, + ) + metrics.log_execution( + plugin_name="my_plugin", + version="v1", + execution_time_ms=10.0, + success=True, + success_score=-0.3, # Should be clamped to 0.0 + metrics_file=metrics_file, + ) + entries = metrics.get_events(metrics_file=metrics_file) + assert entries[0]["success_score"] == 1.0 + assert entries[1]["success_score"] == 0.0 + + +def test_aggregate_with_success_score(metrics_file: Path) -> None: + metrics.log_execution("calc", "v1", 10.0, True, success_score=0.9, metrics_file=metrics_file) + metrics.log_execution("calc", "v1", 20.0, True, success_score=0.7, metrics_file=metrics_file) + metrics.log_execution("calc", "v1", 30.0, False, success_score=0.2, metrics_file=metrics_file) + + result = metrics.aggregate_by_plugin("calc", metrics_file=metrics_file) + stats = result["calc"] + assert "avg_success_score" in stats + # (0.9 + 0.7 + 0.2) / 3 = 0.6 + assert abs(stats["avg_success_score"] - 0.6) < 0.01 + + +def test_aggregate_with_trajectories(metrics_file: Path) -> None: + metrics.log_execution("calc", "v1", 10.0, True, trajectory_id="traj_1", step_number=1, metrics_file=metrics_file) + metrics.log_execution("calc", "v1", 20.0, True, trajectory_id="traj_1", step_number=2, metrics_file=metrics_file) + metrics.log_execution("calc", "v1", 30.0, True, trajectory_id="traj_2", step_number=1, metrics_file=metrics_file) + metrics.log_execution("calc", "v1", 40.0, True, trajectory_id="traj_2", step_number=2, metrics_file=metrics_file) + metrics.log_execution("calc", "v1", 50.0, True, trajectory_id="traj_3", step_number=1, metrics_file=metrics_file) + + result = metrics.aggregate_by_plugin("calc", metrics_file=metrics_file) + stats = result["calc"] + assert "trajectory_count" in stats + assert stats["trajectory_count"] == 3 # traj_1, traj_2, traj_3 + + +def test_aggregate_mixed_with_and_without_scores(metrics_file: Path) -> None: + """Test aggregation when some events have scores and others don't.""" + metrics.log_execution("calc", "v1", 10.0, True, success_score=0.8, metrics_file=metrics_file) + metrics.log_execution("calc", "v1", 20.0, True, metrics_file=metrics_file) # No score + metrics.log_execution("calc", "v1", 30.0, True, success_score=0.6, metrics_file=metrics_file) + + result = metrics.aggregate_by_plugin("calc", metrics_file=metrics_file) + stats = result["calc"] + assert stats["total_executions"] == 3 + assert stats["avg_success_score"] == 0.7 # Only 2 events with scores: (0.8 + 0.6) / 2 diff --git a/tests/test_tool_management.py b/tests/test_tool_management.py new file mode 100644 index 0000000..b5684b8 --- /dev/null +++ b/tests/test_tool_management.py @@ -0,0 +1,304 @@ +"""Tests for core.tool_management - reranking and rejection.""" + +import pytest + +from core.tool_management import ( + RejectionReason, + RejectionResult, + ToolCallScore, + ToolRejectionHandler, + ToolReranker, +) + + +# --------------------------------------------------------------------------- +# ToolCallScore tests +# --------------------------------------------------------------------------- + + +def test_tool_call_score_to_dict() -> None: + score = ToolCallScore( + tool_name="test_tool", + original_rank=0, + reranked_score=0.85, + confidence=0.75, + factors={"success_rate": 0.9, "relevance": 0.8}, + ) + result = score.to_dict() + assert result["tool_name"] == "test_tool" + assert result["reranked_score"] == 0.85 + assert result["confidence"] == 0.75 + assert "success_rate" in result["factors"] + + +# --------------------------------------------------------------------------- +# RejectionResult tests +# --------------------------------------------------------------------------- + + +def test_rejection_result_accepted() -> None: + result = RejectionResult(rejected=False) + d = result.to_dict() + assert d["rejected"] is False + assert d["reason"] is None + + +def test_rejection_result_rejected() -> None: + result = RejectionResult( + rejected=True, + reason=RejectionReason.LOW_CONFIDENCE, + explanation="Confidence too low", + alternative_suggestion="Try again", + ) + d = result.to_dict() + assert d["rejected"] is True + assert d["reason"] == "LOW_CONFIDENCE" + assert d["explanation"] == "Confidence too low" + assert d["alternative_suggestion"] == "Try again" + + +# --------------------------------------------------------------------------- +# ToolReranker tests +# --------------------------------------------------------------------------- + + +def test_reranker_empty_calls() -> None: + reranker = ToolReranker() + calls, scores = reranker.rerank_tools([]) + assert calls == [] + assert scores == [] + + +def test_reranker_single_tool() -> None: + reranker = ToolReranker(confidence_threshold=0.1) + tool_calls = [{"name": "run_plugin", "input": {"name": "test"}}] + + calls, scores = reranker.rerank_tools(tool_calls) + + assert len(calls) == 1 + assert len(scores) == 1 + assert scores[0].tool_name == "run_plugin" + assert 0.0 <= scores[0].reranked_score <= 1.0 + assert 0.0 <= scores[0].confidence <= 1.0 + + +def test_reranker_filters_low_confidence() -> None: + reranker = ToolReranker(confidence_threshold=0.9) # Very high threshold + tool_calls = [{"name": "unknown_tool", "input": {}}] + + calls, scores = reranker.rerank_tools(tool_calls) + + # Should be filtered out due to low confidence + assert len(calls) == 0 + + +def test_reranker_updates_success_rate() -> None: + reranker = ToolReranker(confidence_threshold=0.1) + + # Simulate successful calls + for _ in range(10): + reranker.update_success_rate("good_tool", True) + + # Simulate failed calls + for _ in range(10): + reranker.update_success_rate("bad_tool", False) + + # good_tool should have higher success rate + assert reranker._success_rates["good_tool"] > 0.7 + assert reranker._success_rates["bad_tool"] < 0.3 + + +def test_reranker_sorts_by_score() -> None: + reranker = ToolReranker(confidence_threshold=0.1) + + # Create tools with different names to get different relevance scores + tool_calls = [ + {"name": "http_request", "input": {}}, + {"name": "run_code", "input": {}}, + {"name": "read_file", "input": {}}, + ] + + context = {"code": "programming task"} + calls, scores = reranker.rerank_tools(tool_calls, context=context) + + # run_code should be ranked highest due to code context + assert len(calls) == 3 + assert scores[0].tool_name == "run_code" + + +def test_reranker_records_calls_for_recency() -> None: + reranker = ToolReranker() + + for _ in range(150): # More than max_recent (100) + reranker.record_call("frequent_tool") + + # Should cap at max_recent + assert len(reranker._recent_calls) == reranker._max_recent + + # Recency score should be high + score = reranker.compute_recency_score("frequent_tool") + assert score > 0.5 + + +def test_reranker_context_relevance_code() -> None: + reranker = ToolReranker() + + # Code context should boost code-related tools + score = reranker._compute_relevance("run_python", {"code": "test"}) + assert score > 0.7 + + # Non-code tool should get neutral score + score = reranker._compute_relevance("http_get", {"code": "test"}) + assert abs(score - 0.5) < 0.1 + + +# --------------------------------------------------------------------------- +# ToolRejectionHandler tests +# --------------------------------------------------------------------------- + + +def test_rejection_handler_accepts_valid_call() -> None: + handler = ToolRejectionHandler() + tool_call = {"name": "run_plugin", "input": {"name": "test"}} + + result = handler.reject_tool_call(tool_call) + + assert result.rejected is False + assert result.reason is None + + +def test_rejection_handler_blocks_tool() -> None: + handler = ToolRejectionHandler() + handler.block_tool("dangerous_tool") + + tool_call = {"name": "dangerous_tool", "input": {}} + result = handler.reject_tool_call(tool_call) + + assert result.rejected is True + assert result.reason == RejectionReason.POLICY_VIOLATION + assert "blocked" in result.explanation.lower() + + +def test_rejection_handler_unblocks_tool() -> None: + handler = ToolRejectionHandler() + handler.block_tool("temp_blocked") + handler.unblock_tool("temp_blocked") + + tool_call = {"name": "temp_blocked", "input": {}} + result = handler.reject_tool_call(tool_call) + + assert result.rejected is False + + +def test_rejection_handler_detects_duplicate() -> None: + handler = ToolRejectionHandler(max_duplicate_window=5) + + tool_call = {"name": "run_plugin", "input": {"name": "test"}} + + # First call should be accepted + result1 = handler.reject_tool_call(tool_call) + assert result1.rejected is False + + # Duplicate should be rejected + result2 = handler.reject_tool_call(tool_call) + assert result2.rejected is True + assert result2.reason == RejectionReason.DUPLICATE_CALL + + +def test_rejection_handler_parameter_constraint() -> None: + handler = ToolRejectionHandler() + handler.add_parameter_constraint("run_plugin", ["dangerous_param"]) + + tool_call = {"name": "run_plugin", "input": {"dangerous_param": "value"}} + result = handler.reject_tool_call(tool_call) + + assert result.rejected is True + assert result.reason == RejectionReason.UNSAFE_PARAMETERS + assert "dangerous_param" in result.explanation + + +def test_rejection_handler_low_confidence_auto_reject() -> None: + handler = ToolRejectionHandler( + auto_reject_low_confidence=True, + confidence_threshold=0.5, + ) + + tool_call = {"name": "test_tool", "input": {}} + result = handler.reject_tool_call(tool_call, confidence=0.3) + + assert result.rejected is True + assert result.reason == RejectionReason.LOW_CONFIDENCE + + +def test_rejection_handler_low_confidence_allowed() -> None: + handler = ToolRejectionHandler( + auto_reject_low_confidence=False, # Disabled + confidence_threshold=0.5, + ) + + tool_call = {"name": "test_tool", "input": {}} + result = handler.reject_tool_call(tool_call, confidence=0.3) + + # Should be accepted since auto-reject is disabled + assert result.rejected is False + + +def test_rejection_handler_alternative_suggestions() -> None: + handler = ToolRejectionHandler() + + # Test duplicate suggestion + handler.block_tool("dup_tool") + result = handler.reject_tool_call({"name": "dup_tool", "input": {}}) + assert result.alternative_suggestion is not None + + # Test low confidence suggestion + result = handler.reject_tool_call( + {"name": "test", "input": {}}, + reason=RejectionReason.LOW_CONFIDENCE, + ) + assert "breaking down" in result.alternative_suggestion.lower() + + +def test_rejection_handler_process_with_rejection() -> None: + handler = ToolRejectionHandler(auto_reject_low_confidence=False) + + tool_calls = [ + {"name": "tool_a", "input": {}}, + {"name": "tool_b", "input": {}}, + {"name": "tool_c", "input": {}}, + ] + + scores = [ + ToolCallScore("tool_a", 0, 0.9, 0.8), + ToolCallScore("tool_b", 1, 0.5, 0.4), + ToolCallScore("tool_c", 2, 0.3, 0.2), + ] + + # Block tool_b + handler.block_tool("tool_b") + + accepted, results = handler.process_with_rejection(tool_calls, scores) + + # tool_a should be accepted, tool_b blocked, tool_c accepted (no auto-reject) + assert len(accepted) == 2 + assert accepted[0]["name"] == "tool_a" + assert accepted[1]["name"] == "tool_c" + + assert results[0].rejected is False + assert results[1].rejected is True + assert results[2].rejected is False + + +def test_rejection_handler_custom_reason() -> None: + handler = ToolRejectionHandler() + + tool_call = {"name": "test_tool", "input": {}} + result = handler.reject_tool_call( + tool_call, + reason=RejectionReason.MANUAL_REJECT, + custom_explanation="Custom rejection message", + ) + + assert result.rejected is True + assert result.reason == RejectionReason.MANUAL_REJECT + assert result.explanation == "Custom rejection message"