From a2f2cfc4d3c5cc231628b03f76d1f9735bbcd6a6 Mon Sep 17 00:00:00 2001 From: Coding Agent Date: Sat, 18 Jul 2026 03:50:03 +0800 Subject: [PATCH 1/3] fix: harden agent safety and privacy controls --- README.md | 59 ++++++++++++++++++-------- agent/config.py | 14 ++++++ agent/direct_agent.py | 16 ++++++- agent/history.py | 4 ++ agent/repl.py | 82 ++++++++++++++++++++++++++++++------ agent/tools/execute_shell.py | 8 ---- agent/tools/fetch_url.py | 36 ++++++++++++++++ agent/worker/worker.py | 5 +++ config.toml | 3 +- swe_bench/runner.py | 21 +++------ tests/test_config.py | 14 ++++++ tests/test_history.py | 4 ++ tests/test_repl.py | 82 ++++++++++++++++++++++++++++++++++-- tests/test_tools.py | 31 ++++++++++++++ 14 files changed, 322 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 1333602..e6818b3 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,16 @@ - **16 个内置工具**：读文件、写文件、局部替换、执行 shell、列目录、glob 搜索、代码搜索、网页搜索、抓取网页、询问用户、待办管理、多文件读取、批量补丁、符号搜索、定义跳转、引用查找。 - **安全策略**：写操作、危险 shell 命令需要用户确认；禁止访问工作目录外路径。 - **历史持久化**：会话消息和待办事项自动保存到 SQLite，支持跨会话恢复。 -- **双模型后端**：默认 Kimi，支持 OpenAI 兼容接口切换。 +- **OpenAI 兼容后端**：默认配置为 Kimi，也可切换到其他云端或本地兼容接口。 +- **多语言文件工具，Python 结构化索引**：文件读写和文本搜索适用于多种语言；符号搜索、定义跳转和引用查找目前仅解析 Python。 ## 安装 ```bash +# 运行环境 +pip install -e . + +# 仅开发者需要测试、格式化和类型检查依赖 pip install -e ".[dev]" ``` @@ -63,7 +68,7 @@ coding-agent> 写一个 hello.py，内容是 print("hello")，然后运行它 | `/agent [list\|]` | 列出或切换角色 | | `/mcp` | MCP 服务器状态（实验性） | | `/reload` | 重新加载配置与角色 | -| `/yolo on\|off\|status` | 切换危险操作确认模式 | +| `/yolo on\|off\|status` | 切换危险操作确认模式；开启时需再次输入 `YOLO` | | `exit` / `quit` | 退出 | ## 代码质量 @@ -97,7 +102,8 @@ model = "kimi-for-coding" base_url = "https://api.kimi.com/coding/v1" api_key = "" max_steps_per_turn = 100 -max_retries_per_step = 3 +max_total_tokens_per_turn = 100000 +max_retries_per_step = 5 [security] confirm_dangerous = true @@ -115,9 +121,30 @@ max_messages = 20 - `CODING_AGENT_LLM_MODEL` - `CODING_AGENT_LLM_API_KEY` - `CODING_AGENT_LLM_BASE_URL` +- `CODING_AGENT_LLM_MAX_TOTAL_TOKENS_PER_TURN` - `CODING_AGENT_HISTORY_DB` +- `CODING_AGENT_HISTORY_KEEP`（默认保留最近 200 个会话；设为 `0` 关闭清理） +- `CODING_AGENT_BACKUP_KEEP_DAYS`（默认 30 天；设为 `0` 关闭清理） - `CODING_AGENT_CONFIG` +### 数据与隐私 + +- 默认配置连接 Kimi 云端。用户消息、模型上下文，以及模型请求读取后返回的代码或工具输出，会发送到 `base_url` 指向的服务。处理敏感仓库前，请先确认服务方的数据政策，或改用可信的 OpenAI 兼容本地端点。 +- API key 应通过 `CODING_AGENT_LLM_API_KEY` 或已被 Git 忽略的 `.env` 提供，不要提交到 `config.toml`。项目不会加密配置文件。 +- 历史数据库和撤销备份是本机明文文件，但会以仅当前用户可读的权限创建。可设置 `history.enabled = false` 停止保存消息；旧会话默认只保留最近 200 个，备份默认保留 30 天。 +- `safety.log` 只记录工具名、参数字段名、安全分类和成功状态，不记录参数值、命令、文件内容或工具输出。 +- `fetch_url` 会拒绝 localhost、私网/链路本地 IP、含凭据 URL 和非 HTTP(S) 协议；实际抓取由 Kimi 服务执行，因此 DNS 重绑定和重定向防护仍依赖上游服务。 + +本地兼容端点示例： + +```toml +[llm] +provider = "local" +model = "your-local-model" +base_url = "http://127.0.0.1:8000/v1" +api_key = "" +``` + ### 使用 `.env` 文件（推荐） 在工作目录下创建 `.env` 文件： @@ -179,29 +206,25 @@ python -m twine upload dist/* ## SWE-bench-lite 基准测试 -我们在 [SWE-bench-lite](https://www.swebench.com/) 的 20 个任务上对比了三种执行模式，统一使用 `deepseek-v4-flash` 模型，coding-agent 与 Claude Code 用内置 `SWEBenchEvaluator` 评估，SWE-agent 用 `DockerEvaluator` 评估： +仓库曾在 SWE-bench-lite 的 20 个任务上做过探索性对比。该样本、执行环境和 harness 均不足以支持跨系统排名，因此目前不发布可引用的解决率；下面只保留复现方法和已知限制。 - **direct**：coding-agent 的零 IPC in-process 单 agent 模式 - **Claude Code**：通过 `cc-switch` 代理到本地端点的 Claude Code v2.1.187 - **SWE-agent**：v0.7.0，本地 persistent bash 环境 -### 结果（20 task） +### 历史结果状态 -> ⚠️ **以下数值为历史运行结果，已失效，待合规重跑后更新。** 它们存在两个已知问题，不能作为当前真实水平参考： +> ⚠️ **历史数值已撤下，待使用同一公开 harness、固定环境、完整任务集并多次重复后更新。** 旧实验存在以下问题： > 1. **数据泄露（已修）**：早期 `runner.py::_build_goal_description` 向 agent 泄露了 `FAIL_TO_PASS` 测试名，相当于给出验收标准，违反 SWE-bench 盲改合规。已移除。 -> 2. **SWE-agent 环境已修但未取得可靠分数**：历史运行中 SWE-agent 20 个任务全部 `exit code 1`（启动即崩），原因是其 conda 环境 `swe_agent_py311` 的 numpy(1.24)/pandas(3.0) 版本冲突。numpy 已升级修复，SWE-agent 现可正常启动并能产出正确 patch（单任务冒烟验证）。但 SWE-agent 交互式 bash 模式极慢（单任务 275+ 次 API 调用），默认 1200s 超时内常未跑完验证步骤就被 kill，导致 patch 未被收集、判为未解决。曾报告的 7/20 是更早期旧值，不代表当前配置。完整 20 任务需调大 timeout（预计 6h+）才能取得可靠分数，尚未执行。 - -| 系统 | 历史值 | 状态 | -|---|---|---| -| coding-agent direct | 16/20 | 含 fail_to_pass 泄露，待合规重跑 | -| Claude Code | 14/20 | 待合规重跑 | -| SWE-agent | 7/20（旧值）/ 0/20（超时）| 环境已修，能解题但超时，待调 timeout 重跑 | +> 2. **对比条件不一致**：工具集、执行环境、超时和评估器不同，结果不能作为公平的系统间比较。 +> 3. **样本过小且未重复**：20 个任务的单次结果统计波动很大。 +> 4. **模型标签不规范**：`deepseek-v4-flash` 是当时本地代理使用的自定义别名，不代表 DeepSeek 官方公开型号。复现时必须记录实际 provider、模型版本和端点配置。 ### 关键优化 -direct 模式从 12/20 提升到 16/20，主要得益于： +历史探索中采用过以下实现调整；这里不再把它们与已撤下的分数绑定： -1. **shell 安全策略放宽**：SWE-bench 场景下通过 `CODING_AGENT_SWEBENCH_FORCE=1` 允许 `cd && pytest`、`python -c`、`python -m pytest` 等验证命令执行（safety.py 将 `python -m pytest/py_compile/compileall` 归类为 HARMLESS）。 +1. **评测运行器显式授权**：只有受信任的 SWE-bench runner 实例能调用危险 shell 的授权入口；环境变量不能关闭普通用户的安全检查，forbidden 命令始终拒绝。 2. **Prompt 收紧**：强制最小改动、禁止安装依赖/修改配置、要求验证后再结束。 3. **合规修正**：移除 goal description 中的 `FAIL_TO_PASS` 测试名泄露，agent 只看 issue 描述，验收测试由评估 harness 在不可见情况下运行。 @@ -337,11 +360,11 @@ coding-agent/ - 根据 `config.llm.stream` 选择 `_run_turn_stream()` 或 `_run_turn_non_stream()`。 - 将 LLM 返回的 `AssistantResponse` 保存为 `assistant` 消息。 - 如果存在 `tool_calls`，逐个调用 `_execute_tool_call()`，结果保存为 `tool` 消息并再次请求 LLM。 - - 工具失败（非用户拒绝/禁止）时自动重试 1 次。 - - 达到 `max_steps_per_turn` 上限后停止并提示用户。 + - 只有无副作用的本地读取/搜索工具失败时自动重试 1 次；shell、写操作和网络请求不自动重试。 + - 达到 `max_steps_per_turn` 或 `max_total_tokens_per_turn` 上限后停止并提示用户。 - **历史加载 `_load_history()`**：从 SQLite 恢复最近消息，并清洗不完整的 `assistant(tool_calls)` 以及 `tool_call_id` 为空或不匹配的脏 tool 消息。 - **会话管理**：`/sessions`、`/switch`、`/rename`、`/delete` 基于 `HistoryManager` 实现；新会话自动用第一条用户消息前 30 字生成标题。 -- **撤销 `/undo`**：写操作前备份原文件到 `~/.coding-agent/backups///`，`/undo` 恢复最近一次备份。 +- **撤销 `/undo`**：写操作前备份原文件到 `~/.coding-agent/backups///`，`/undo` 恢复最近一次备份；默认自动清理 30 天前的备份。 - **Git 状态**：启动时与 `/git` 命令通过 `git status --short` 和 `git branch --show-current` 展示当前分支与未提交文件。 ### LLM 调用层（`agent/llm/`） diff --git a/agent/config.py b/agent/config.py index 7de9ed4..81d9677 100644 --- a/agent/config.py +++ b/agent/config.py @@ -21,6 +21,7 @@ class LLMConfig(BaseModel): timeout: float | None = 300.0 stream_read_timeout: float | None = 120.0 max_steps_per_turn: int = 100 + max_total_tokens_per_turn: int = 100_000 max_retries_per_step: int = 5 system_prompt: str | None = None @@ -48,6 +49,13 @@ def _validate_max_retries_per_step(cls, v: int) -> int: raise ValueError("max_retries_per_step must be >= 0") return v + @field_validator("max_total_tokens_per_turn") + @classmethod + def _validate_max_total_tokens_per_turn(cls, v: int) -> int: + if v < 1: + raise ValueError("max_total_tokens_per_turn must be >= 1") + return v + class SecurityConfig(BaseModel): confirm_dangerous: bool = True @@ -155,6 +163,12 @@ def _env_override_data() -> dict[str, Any]: stream = os.getenv("CODING_AGENT_LLM_STREAM") if stream is not None: overrides.setdefault("llm", {})["stream"] = stream.lower() in ("1", "true", "yes") + token_budget = os.getenv("CODING_AGENT_LLM_MAX_TOTAL_TOKENS_PER_TURN") + if token_budget: + try: + overrides.setdefault("llm", {})["max_total_tokens_per_turn"] = int(token_budget) + except ValueError: + pass db_path = os.getenv("CODING_AGENT_HISTORY_DB") if db_path: overrides.setdefault("history", {})["db_path"] = db_path diff --git a/agent/direct_agent.py b/agent/direct_agent.py index c9d1c1f..7691ea2 100644 --- a/agent/direct_agent.py +++ b/agent/direct_agent.py @@ -46,10 +46,12 @@ def __init__( allowed_tools: list[str] | None = None, log_path: str | Path | None = None, conda_env: str | None = None, + allow_dangerous_shell: bool = False, ): self.llm = llm self.workspace = Path(workspace).resolve() self.conda_env = conda_env + self.allow_dangerous_shell = allow_dangerous_shell # Build tool list all_tools = TOOL_REGISTRY if allowed_tools is None: @@ -65,6 +67,8 @@ def __init__( if self.log_path: self.log_path.parent.mkdir(parents=True, exist_ok=True) # Start fresh log file for each run. + self.log_path.touch(mode=0o600, exist_ok=True) + self.log_path.chmod(0o600) self.log_path.write_text("", encoding="utf-8") def _build_system_prompt(self, base: str) -> str: @@ -170,7 +174,13 @@ def run(self, goal_description: str, max_steps: int = 50) -> str: } ) + total_tokens = 0 + max_tokens = self.llm.config.max_total_tokens_per_turn for step in range(1, max_steps + 1): + if step > 1 and total_tokens >= max_tokens: + message = f"Reached token budget ({max_tokens}) without final answer." + self._log_event({"type": "token_budget_reached", "max_tokens": max_tokens}) + return message messages = self._compact_messages(messages, max_turns=20) logger.info("step %d/%d: calling LLM", step, max_steps) try: @@ -185,6 +195,7 @@ def run(self, goal_description: str, max_steps: int = 50) -> str: } ) return f"LLM error at step {step}: {exc}" + total_tokens += response.usage.total_tokens self._log_event( { @@ -244,7 +255,10 @@ def run(self, goal_description: str, max_steps: int = 50) -> str: if call.name == "str_replace_file": call, result = self._ensure_file_read_before_edit(call, ctx, messages) else: - result = tool.execute(call.arguments, ctx) + if call.name == "execute_shell" and self.allow_dangerous_shell: + result = tool.execute_forced(call.arguments, ctx) + else: + result = tool.execute(call.arguments, ctx) if call.name == "read_file" and result.success: path = call.arguments.get("path") if path: diff --git a/agent/history.py b/agent/history.py index 799280d..2accaa4 100644 --- a/agent/history.py +++ b/agent/history.py @@ -17,6 +17,10 @@ class HistoryManager: def __init__(self, db_path: str | None = None): self.db_path = Path(db_path or DEFAULT_DB_PATH).expanduser() self.db_path.parent.mkdir(parents=True, exist_ok=True) + if self.db_path.parent == Path.home() / ".coding-agent": + self.db_path.parent.chmod(0o700) + self.db_path.touch(mode=0o600, exist_ok=True) + self.db_path.chmod(0o600) self._init_db() def _connect(self) -> sqlite3.Connection: diff --git a/agent/repl.py b/agent/repl.py index 102740a..639649f 100644 --- a/agent/repl.py +++ b/agent/repl.py @@ -15,8 +15,10 @@ import json import logging import os +import shutil import subprocess import threading +import uuid from pathlib import Path from typing import Any, Callable @@ -46,6 +48,16 @@ from agent.tools.apply_patch import parse_diff _FILE_WRITE_TOOLS = {"write_file", "str_replace_file", "apply_patch"} +_AUTO_RETRY_TOOLS = { + "read_file", + "read_multiple_files", + "list_directory", + "glob_search", + "code_search", + "symbol_search", + "find_definition", + "find_references", +} logger = logging.getLogger("agent.repl") @@ -101,7 +113,12 @@ def __init__( self.input_func = input_func or self._default_input self.history = history_manager or HistoryManager(self.config.history.db_path) self._maybe_prune_history() - self.session_id = self.history.get_or_create_session(self.workspace) + self._maybe_prune_backups() + self.session_id = ( + self.history.get_or_create_session(self.workspace) + if self.config.history.enabled + else str(uuid.uuid4()) + ) self.llm = llm_client or LLMClient(self.config.llm) self.tools_schema = build_tools_payload(list(TOOL_REGISTRY.values())) self._always_allowed_tools: set[str] = set() @@ -181,6 +198,30 @@ def _maybe_prune_history(self) -> None: except Exception: logger.debug("history pruning skipped", exc_info=True) + def _maybe_prune_backups(self) -> None: + """Best-effort removal of undo snapshots older than 30 days.""" + try: + keep_days = int(os.getenv("CODING_AGENT_BACKUP_KEEP_DAYS", "30")) + except ValueError: + keep_days = 30 + if keep_days <= 0: + return + root = Path.home() / ".coding-agent" / "backups" + if not root.is_dir(): + return + cutoff = datetime.datetime.now().timestamp() - keep_days * 86400 + try: + for session_dir in root.iterdir(): + if not session_dir.is_dir(): + continue + for snapshot in session_dir.iterdir(): + if snapshot.is_dir() and snapshot.stat().st_mtime < cutoff: + shutil.rmtree(snapshot) + if not any(session_dir.iterdir()): + session_dir.rmdir() + except OSError: + logger.debug("backup pruning skipped", exc_info=True) + def _load_history(self) -> None: if not self.config.history.enabled: return @@ -508,6 +549,12 @@ def _handle_yolo_command(self, arg: str) -> None: """切换危险操作确认开关（yolo 模式）。""" arg = arg.strip().lower() if arg == "on": + confirmation = self.input_func( + "YOLO 模式会跳过写文件和危险命令确认。输入 YOLO 继续：" + ).strip() + if confirmation != "YOLO": + self.console.print("[green]已取消，仍处于安全模式[/green]") + return self.config.security.confirm_dangerous = False self.console.print("[yellow]已切换到 YOLO 模式：危险操作不再确认[/yellow]") elif arg == "off": @@ -815,7 +862,17 @@ def _process_user_input(self, text: str) -> bool: def _run_turn(self) -> AssistantResponse: """执行一次完整的 LLM 交互 turn。""" max_steps = self.config.llm.max_steps_per_turn + max_tokens = self.config.llm.max_total_tokens_per_turn + turn_tokens = 0 for step in range(max_steps): + if step > 0 and turn_tokens >= max_tokens: + limit_msg = f"⚠️ 本轮累计 token 已达到上限（{max_tokens}），停止执行。" + self.console.print(limit_msg) + limit_response = AssistantResponse(content=limit_msg) + limit_message = Message(role="assistant", content=limit_msg) + self._save_message(limit_message) + self.messages.append(limit_message) + return limit_response if self.config.llm.stream: response = self._run_turn_stream() else: @@ -835,6 +892,7 @@ def _run_turn(self) -> AssistantResponse: self._total_usage.prompt_tokens += response.usage.prompt_tokens self._total_usage.completion_tokens += response.usage.completion_tokens self._total_usage.total_tokens += response.usage.total_tokens + turn_tokens += response.usage.total_tokens if not response.tool_calls: if self._maybe_auto_compact(): @@ -846,7 +904,7 @@ def _run_turn(self) -> AssistantResponse: result = self._execute_tool_call(call) if ( not result.success - and call.name != "ask_user" + and call.name in _AUTO_RETRY_TOOLS and "User declined" not in (result.error or "") and "forbidden" not in (result.error or "").lower() ): @@ -962,8 +1020,12 @@ def _backup_file(self, relative_path: str) -> Path | None: timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_%f") backup_dir = Path.home() / ".coding-agent" / "backups" / self.session_id / timestamp backup_dir.mkdir(parents=True, exist_ok=True) + (Path.home() / ".coding-agent").chmod(0o700) + backup_dir.chmod(0o700) backup_path = backup_dir / relative_path backup_path.parent.mkdir(parents=True, exist_ok=True) + backup_path.touch(mode=0o600, exist_ok=True) + backup_path.chmod(0o600) backup_path.write_text(target.read_text(encoding="utf-8"), encoding="utf-8") self._write_backups.append({"path": relative_path, "backup_path": str(backup_path)}) return backup_path @@ -1126,23 +1188,19 @@ def _log_safety_event( log_dir = Path.home() / ".coding-agent" log_dir.mkdir(parents=True, exist_ok=True) + log_dir.chmod(0o700) log_path = log_dir / "safety.log" + log_path.touch(mode=0o600, exist_ok=True) + log_path.chmod(0o600) - safe_arguments = { - k: ("***" if k in ("api_key", "token", "password", "secret") else v) - for k, v in call.arguments.items() - } entry = { "timestamp": datetime.datetime.now().isoformat(), "tool": call.name, - "arguments": safe_arguments, + "argument_keys": sorted(call.arguments), "classification": classification.value, "confirmed": confirmed, - "result": { - "success": result.success, - "output": result.output, - "error": result.error, - }, + "success": result.success, + "error_type": "tool_error" if result.error else None, } with open(log_path, "a", encoding="utf-8") as f: f.write(json.dumps(entry, ensure_ascii=False, default=str) + "\n") diff --git a/agent/tools/execute_shell.py b/agent/tools/execute_shell.py index 30851ee..4709158 100644 --- a/agent/tools/execute_shell.py +++ b/agent/tools/execute_shell.py @@ -99,14 +99,6 @@ def _execute(self, input: dict, ctx: ToolContext, *, force: bool) -> ToolResult: success=False, error=f"Command classified as forbidden and will not be executed: '{command}'.", ) - # In SWE-bench mode, skip dangerous-command confirmation so test/repro - # commands (python -c, cd, &&, etc.) can run without interactive consent. - if ( - not force - and classification == CommandClass.DANGEROUS - and os.environ.get("CODING_AGENT_SWEBENCH_FORCE") == "1" - ): - force = True if classification == CommandClass.DANGEROUS and not force: return ToolResult( success=False, diff --git a/agent/tools/fetch_url.py b/agent/tools/fetch_url.py index 1335a91..5d5f926 100644 --- a/agent/tools/fetch_url.py +++ b/agent/tools/fetch_url.py @@ -1,4 +1,6 @@ +import ipaddress import os +from urllib.parse import urlsplit import requests from pydantic import BaseModel, Field @@ -9,6 +11,37 @@ DEFAULT_TIMEOUT = 10 +def _validate_public_url(url: str) -> str | None: + """Reject URLs that obviously target local or private services.""" + try: + parsed = urlsplit(url) + hostname = parsed.hostname + port = parsed.port + except ValueError: + return "URL is malformed" + if parsed.scheme not in {"http", "https"}: + return "Only http and https URLs are allowed" + if not hostname: + return "URL must include a hostname" + if parsed.username is not None or parsed.password is not None: + return "URLs containing credentials are not allowed" + if port is not None and not (1 <= port <= 65535): + return "URL port is invalid" + + normalized = hostname.rstrip(".").lower() + if normalized == "localhost" or normalized.endswith((".localhost", ".local", ".internal")): + return "Local and private network hosts are not allowed" + if normalized.replace(".", "").isdigit() or normalized.startswith("0x"): + return "Ambiguous numeric hostnames are not allowed" + try: + address = ipaddress.ip_address(normalized) + except ValueError: + return None + if not address.is_global: + return "Local and private network addresses are not allowed" + return None + + class FetchUrlInput(BaseModel): url: str = Field(..., description="抓取网页 URL") max_length: int = Field(default=DEFAULT_MAX_LENGTH, description="返回内容的最大长度") @@ -30,6 +63,9 @@ def execute(self, input: dict, ctx: ToolContext) -> ToolResult: success=False, error="URL cannot be empty", ) + validation_error = _validate_public_url(url) + if validation_error: + return ToolResult(success=False, error=validation_error) api_key = os.getenv("CODING_AGENT_LLM_API_KEY", "") if not api_key: diff --git a/agent/worker/worker.py b/agent/worker/worker.py index 5d85784..ccda793 100644 --- a/agent/worker/worker.py +++ b/agent/worker/worker.py @@ -137,11 +137,16 @@ def _execute_goal(self) -> str: tools_schema = self._build_tools_schema() max_steps = self.role.max_steps_per_turn or self.llm.config.max_steps_per_turn + max_tokens = self.llm.config.max_total_tokens_per_turn + total_tokens = 0 goal_id = self.goal.id if self.goal else "unknown" for step in range(max_steps): + if step > 0 and total_tokens >= max_tokens: + return f"Reached token budget ({max_tokens}) without final answer." logger.info("goal %s step %d/%d: calling LLM", goal_id, step + 1, max_steps) response = self.llm.chat(messages, tools=tools_schema) + total_tokens += response.usage.total_tokens messages.append(self._assistant_message(response)) if response.tool_calls: diff --git a/config.toml b/config.toml index 2735885..6f76759 100644 --- a/config.toml +++ b/config.toml @@ -4,10 +4,11 @@ model = "kimi-for-coding" base_url = "https://api.kimi.com/coding/v1" api_key = "" max_steps_per_turn = 50 +max_total_tokens_per_turn = 100000 max_retries_per_step = 5 [security] -confirm_dangerous = false +confirm_dangerous = true log_safety_events = true allow_outside_workspace = false diff --git a/swe_bench/runner.py b/swe_bench/runner.py index 9532739..8ac4162 100644 --- a/swe_bench/runner.py +++ b/swe_bench/runner.py @@ -374,22 +374,15 @@ def _run_task_direct( allowed_tools=coder_role.allowed_tools, log_path=task_output_dir / "agent.log", conda_env=env_name, + allow_dangerous_shell=True, ) - # Run the agent. In SWE-bench mode let the agent execute test/repro - # commands without interactive confirmation. - old_swebench_force = os.environ.get("CODING_AGENT_SWEBENCH_FORCE") - os.environ["CODING_AGENT_SWEBENCH_FORCE"] = "1" - try: - agent.run( - goal_description=description, - max_steps=self.config.llm.max_steps_per_turn, - ) - finally: - if old_swebench_force is None: - os.environ.pop("CODING_AGENT_SWEBENCH_FORCE", None) - else: - os.environ["CODING_AGENT_SWEBENCH_FORCE"] = old_swebench_force + # The trusted benchmark runner grants shell consent explicitly. + # Normal users cannot enable this path with an environment variable. + agent.run( + goal_description=description, + max_steps=self.config.llm.max_steps_per_turn, + ) # Collect patch patch_path = task_output_dir / "agent.patch" diff --git a/tests/test_config.py b/tests/test_config.py index 5394c69..9df3eb7 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -18,6 +18,7 @@ def test_load_default_config(isolated_home): assert config.llm.base_url == "https://api.kimi.com/coding/v1" assert config.llm.api_key == "" assert config.llm.max_steps_per_turn == 100 + assert config.llm.max_total_tokens_per_turn == 100_000 assert config.llm.max_retries_per_step == 5 assert config.history.enabled is True assert config.history.max_messages == 20 @@ -108,6 +109,19 @@ def test_negative_retries(isolated_home): load_config() +def test_invalid_token_budget(isolated_home): + _write_user_config(isolated_home, "[llm]\nmax_total_tokens_per_turn = 0\n") + + with pytest.raises(ValidationError): + load_config() + + +def test_token_budget_env_override(isolated_home, monkeypatch): + monkeypatch.setenv("CODING_AGENT_LLM_MAX_TOTAL_TOKENS_PER_TURN", "1234") + + assert load_config().llm.max_total_tokens_per_turn == 1234 + + def test_negative_max_messages(isolated_home): """history.max_messages 为负数触发校验错误。""" _write_user_config(isolated_home, "[history]\nmax_messages = -5\n") diff --git a/tests/test_history.py b/tests/test_history.py index 7f05b3f..35d7343 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -1,3 +1,4 @@ +import stat import uuid from pathlib import Path @@ -42,6 +43,9 @@ def test_tables_are_created(self, history): } assert {"sessions", "messages", "todos"}.issubset(tables) + def test_database_permissions_are_private(self, history): + assert stat.S_IMODE(history.db_path.stat().st_mode) == 0o600 + class TestSessions: def test_create_session_returns_uuid(self, history, sample_workspace): diff --git a/tests/test_repl.py b/tests/test_repl.py index 2abf9db..a0f4338 100644 --- a/tests/test_repl.py +++ b/tests/test_repl.py @@ -13,7 +13,7 @@ from agent.config import Config, LLMConfig from agent.history import HistoryManager -from agent.llm.schema import AssistantResponse, LLMError, Message, ToolCall +from agent.llm.schema import AssistantResponse, LLMError, Message, ToolCall, Usage from agent.repl import REPL, _format_tool_result, main from agent.tools.base import ToolResult from tests.conftest import MockLLM @@ -86,6 +86,32 @@ def test_repl_exit_by_command(tmp_path): assert "再见" in output.getvalue() +def test_disabled_history_does_not_persist_workspace(tmp_path): + history = HistoryManager(str(tmp_path / "history.db")) + config = _make_config(history={"enabled": False, "db_path": str(tmp_path / "history.db")}) + + _make_repl(tmp_path, inputs=["exit"], history=history, config=config) + + assert history.list_recent_sessions() == [] + + +def test_yolo_mode_requires_explicit_confirmation(tmp_path): + repl, output = _make_repl(tmp_path, inputs=["no"]) + + repl._handle_yolo_command("on") + + assert repl.config.security.confirm_dangerous is True + assert "取消" in output.getvalue() + + +def test_yolo_mode_can_be_explicitly_enabled(tmp_path): + repl, _ = _make_repl(tmp_path, inputs=["YOLO"]) + + repl._handle_yolo_command("on") + + assert repl.config.security.confirm_dangerous is False + + def test_repl_tokens_and_history_commands(tmp_path): """/tokens 和 /history 命令应正常显示。""" llm = MockLLM(responses=[AssistantResponse(content="收到")]) @@ -453,6 +479,53 @@ def test_repl_max_steps_per_turn(tmp_path): assert "最大" in output.getvalue() or "上限" in output.getvalue() +def test_repl_token_budget_stops_before_next_llm_call(tmp_path): + config = _make_config( + llm=LLMConfig(api_key="test-key", max_steps_per_turn=5, max_total_tokens_per_turn=5) + ) + llm = MockLLM( + responses=[ + AssistantResponse( + tool_calls=[ToolCall(id="call-1", name="list_directory", arguments={"path": "."})], + usage=Usage(total_tokens=10), + ) + ] + ) + + repl, output = _make_repl(tmp_path, inputs=["go", "exit"], llm=llm, config=config) + repl.run() + + assert llm.call_count == 1 + assert "token" in output.getvalue() + + +def test_repl_does_not_retry_network_tool_failure(tmp_path): + llm = MockLLM( + responses=[ + AssistantResponse( + tool_calls=[ + ToolCall( + id="call-1", name="fetch_url", arguments={"url": "https://example.com"} + ) + ] + ), + AssistantResponse(content="done"), + ] + ) + repl, _ = _make_repl(tmp_path, inputs=["go", "exit"], llm=llm) + calls = 0 + + def fail_once(call): + nonlocal calls + calls += 1 + return ToolResult(success=False, error="network failed") + + repl._execute_tool_call = fail_once # type: ignore[method-assign] + repl.run() + + assert calls == 1 + + # --------------------------------------------------------------------------- # 安全与确认 # --------------------------------------------------------------------------- @@ -632,7 +705,10 @@ def test_repl_dangerous_shell_logs_safety_event(tmp_path, isolated_home): assert entry["tool"] == "execute_shell" assert entry["classification"] == "dangerous" assert entry["confirmed"] is True - assert entry["result"]["success"] is True + assert entry["success"] is True + assert entry["argument_keys"] == ["command"] + assert "arguments" not in entry + assert "result" not in entry def test_repl_declined_dangerous_shell_logs_safety_event(tmp_path, isolated_home): @@ -707,7 +783,7 @@ def test_repl_forbidden_shell_logs_safety_event(tmp_path, isolated_home): assert entry["tool"] == "execute_shell" assert entry["classification"] == "forbidden" assert entry["confirmed"] is None - assert entry["result"]["success"] is False + assert entry["success"] is False def test_repl_safety_log_disabled(tmp_path, isolated_home): diff --git a/tests/test_tools.py b/tests/test_tools.py index ca6f608..26c071f 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -490,6 +490,15 @@ def test_execute_shell_dangerous_blocked(self, shell_tool, workspace): assert "dangerous" in result.error.lower() assert not target.exists() + def test_execute_shell_env_cannot_bypass_confirmation(self, shell_tool, workspace, monkeypatch): + monkeypatch.setenv("CODING_AGENT_SWEBENCH_FORCE", "1") + ctx = ToolContext(workspace=str(workspace)) + + result = shell_tool.execute({"command": "echo x > bypass.txt"}, ctx) + + assert not result.success + assert not (workspace / "bypass.txt").exists() + def test_execute_shell_forbidden_blocked(self, shell_tool, workspace): ctx = ToolContext(workspace=str(workspace)) @@ -910,6 +919,28 @@ def test_fetch_url_empty_url(self, web_tools, workspace): assert not result.success assert "empty" in result.error.lower() + @pytest.mark.parametrize( + "url", + [ + "http://127.0.0.1/admin", + "http://[::1]/admin", + "http://169.254.169.254/latest/meta-data", + "http://10.0.0.1/", + "http://2130706433/", + "http://0x7f000001/", + "http://localhost/", + "file:///etc/passwd", + "https://user:pass@example.com/", + ], + ) + def test_fetch_url_rejects_private_targets(self, web_tools, workspace, url): + _, fetch_url_tool = web_tools + ctx = ToolContext(workspace=str(workspace)) + + result = fetch_url_tool.execute({"url": url}, ctx) + + assert not result.success + @pytest.fixture def interactive_tools(isolated_registry): From eee645b3386cb73ee61988ea3cbbdec02e718434 Mon Sep 17 00:00:00 2001 From: Coding Agent Date: Sat, 18 Jul 2026 03:55:28 +0800 Subject: [PATCH 2/3] fix: make three-system benchmark comparison fairer --- README.md | 2 +- scripts/compare_three_systems.py | 49 +++++++++++++++++++--------- swe_bench/runner.py | 3 -- tests/test_swe_agent_local_runner.py | 3 +- 4 files changed, 37 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index e6818b3..8342c92 100644 --- a/README.md +++ b/README.md @@ -215,7 +215,7 @@ python -m twine upload dist/* ### 历史结果状态 > ⚠️ **历史数值已撤下，待使用同一公开 harness、固定环境、完整任务集并多次重复后更新。** 旧实验存在以下问题： -> 1. **数据泄露（已修）**：早期 `runner.py::_build_goal_description` 向 agent 泄露了 `FAIL_TO_PASS` 测试名，相当于给出验收标准，违反 SWE-bench 盲改合规。已移除。 +> 1. **数据泄露（已修）**：早期 runner 向 agent 暴露了 `FAIL_TO_PASS` 测试名，并且不同系统对 `hints_text` 的可见性不一致。当前三种模式都只接收 issue 标题和正文，隐藏测试仅供评估器使用。 > 2. **对比条件不一致**：工具集、执行环境、超时和评估器不同，结果不能作为公平的系统间比较。 > 3. **样本过小且未重复**：20 个任务的单次结果统计波动很大。 > 4. **模型标签不规范**：`deepseek-v4-flash` 是当时本地代理使用的自定义别名，不代表 DeepSeek 官方公开型号。复现时必须记录实际 provider、模型版本和端点配置。 diff --git a/scripts/compare_three_systems.py b/scripts/compare_three_systems.py index d8795e6..9b67389 100644 --- a/scripts/compare_three_systems.py +++ b/scripts/compare_three_systems.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 -"""A/B/C comparison: coding-agent direct vs Claude Code vs SWE-agent (semi-official). +"""A/B/C comparison: coding-agent direct vs Claude Code vs SWE-agent. -All three use `deepseek-v4-flash`. Evaluation is done with coding-agent's -DockerEvaluator to keep scoring consistent. +All systems receive only the issue title/body, use the configured model alias, +and are evaluated with the same DockerEvaluator. Tooling and agent prompts +still differ, so this is a practical system comparison rather than a model-only +ablation. """ from __future__ import annotations @@ -89,9 +91,6 @@ def build_goal_description(task: SWEBenchTask) -> str: parts.append(f"Title: {task.issue_title}") if task.issue_body: parts.append(f"Description:\n{task.issue_body}") - if task.hints_text: - parts.append(f"Hints: {task.hints_text}") - instructions = ( "You are fixing a real bug in this repository.\n\n" "Use the problem statement above to understand the required behavior " @@ -171,8 +170,7 @@ def run_claude( ) -> dict[str, Any]: start = time.monotonic() task_output_dir.mkdir(parents=True, exist_ok=True) - prompt_path = task_output_dir / "prompt.txt" - prompt_path.write_text(build_goal_description(task), encoding="utf-8") + prompt = build_goal_description(task) env = dict(os.environ) env["CLAUDE_CODE_DEBUG"] = "1" @@ -191,9 +189,7 @@ def run_claude( "claude", "-p", "--verbose", - "Fix the bug described in ../prompt.txt. Read the file for the prompt. " - "Make minimal changes to src/pytest to make the failing tests pass. " - "Stop after editing.", + prompt, ], cwd=str(workspace), env=env, @@ -233,7 +229,9 @@ def run_claude( return {"resolved": False, "duration": duration, "error": "empty patch", "patch": patch} (task_output_dir / "agent.patch").write_text(patch, encoding="utf-8") - return evaluate_patch(task, workspace, patch, task_output_dir / "docker_eval") + evaluated = evaluate_patch(task, workspace, patch, task_output_dir / "docker_eval") + evaluated["duration"] = duration + return evaluated def run_direct( @@ -353,7 +351,9 @@ def run_swe_agent( return {"resolved": False, "duration": duration, "error": "empty patch", "patch": patch} (task_output_dir / "agent.patch").write_text(patch, encoding="utf-8") - return evaluate_patch(task, workspace, patch, task_output_dir / "docker_eval") + evaluated = evaluate_patch(task, workspace, patch, task_output_dir / "docker_eval") + evaluated["duration"] = duration + return evaluated INFRA_ERROR_PATTERNS = ( @@ -432,6 +432,12 @@ def main() -> int: parser.add_argument( "--swe-agent-timeout", type=int, default=1200, help="Per-task timeout for SWE-agent (s)" ) + parser.add_argument( + "--direct-timeout", type=int, default=1200, help="Per-task timeout for direct mode (s)" + ) + parser.add_argument( + "--claude-timeout", type=int, default=1200, help="Per-task timeout for Claude Code (s)" + ) parser.add_argument( "--swe-agent-max-steps", type=int, default=100, help="Max SWE-agent steps per task" ) @@ -511,7 +517,14 @@ def main() -> int: workspace = task_output_dir / "direct_workspace" prepare_workspace(task, workspace) logger.info("running direct for %s", task.id) - direct = run_direct(task, task_output_dir / "direct", workspace, config, args.model) # type: ignore[arg-type] + direct = run_direct( + task, + task_output_dir / "direct", + workspace, + config, # type: ignore[arg-type] + args.model, + timeout_seconds=args.direct_timeout, + ) r.direct_resolved = direct["resolved"] r.direct_duration = direct["duration"] r.direct_error = direct["error"] @@ -523,7 +536,13 @@ def main() -> int: workspace = task_output_dir / "claude_workspace" prepare_workspace(task, workspace) logger.info("running Claude Code for %s", task.id) - claude = run_claude(task, task_output_dir / "claude", workspace, args.model) + claude = run_claude( + task, + task_output_dir / "claude", + workspace, + args.model, + timeout_seconds=args.claude_timeout, + ) r.claude_resolved = claude["resolved"] r.claude_duration = claude["duration"] r.claude_error = claude["error"] diff --git a/swe_bench/runner.py b/swe_bench/runner.py index 8ac4162..f213c69 100644 --- a/swe_bench/runner.py +++ b/swe_bench/runner.py @@ -771,9 +771,6 @@ def _build_goal_description(self, task: SWEBenchTask) -> str: parts.append(f"Title: {task.issue_title}") if task.issue_body: parts.append(f"Description:\n{task.issue_body}") - if task.hints_text: - parts.append(f"Hints: {task.hints_text}") - # SWE-bench specific workflow: focus the agent on finding and fixing the # bug with minimal steps. The previous "run tests first" strategy wasted # ~30% of turns on environment issues (especially when conda is broken). diff --git a/tests/test_swe_agent_local_runner.py b/tests/test_swe_agent_local_runner.py index b409772..94a7b74 100644 --- a/tests/test_swe_agent_local_runner.py +++ b/tests/test_swe_agent_local_runner.py @@ -115,10 +115,11 @@ def test_runtime_config_resolves_relative_command_files(tmp_path): def test_comparison_prompt_does_not_expose_hidden_tests(): - prompt = build_goal_description(_task()) + prompt = build_goal_description(_task(hints_text="private implementation hint")) assert "hidden/test_secret.py" not in prompt assert "FAIL_TO_PASS" not in prompt + assert "private implementation hint" not in prompt assert "public issue description" in prompt From ea56505619bdb1a1e9866bf0747caeb40def80f5 Mon Sep 17 00:00:00 2001 From: Coding Agent Date: Sat, 18 Jul 2026 04:12:41 +0800 Subject: [PATCH 3/3] fix: abort benchmark runs on endpoint failures --- agent/llm/client.py | 34 +++++++++++++++++++++++++++++--- scripts/compare_three_systems.py | 27 ++++++++++++++++++++++++- swe_bench/runner.py | 8 ++++++-- tests/test_llm.py | 21 ++++++++++++++++++-- 4 files changed, 82 insertions(+), 8 deletions(-) diff --git a/agent/llm/client.py b/agent/llm/client.py index d400dd8..175e5b1 100644 --- a/agent/llm/client.py +++ b/agent/llm/client.py @@ -18,6 +18,23 @@ logger = logging.getLogger("agent.llm.client") +_NON_RETRYABLE_ERROR_MARKERS = ( + "insufficient_quota", + "quota has been exhausted", + "invalid_api_key", + "authentication_error", +) + + +def _is_non_retryable_api_error(exc: Exception) -> bool: + """Return True for failures that waiting and retrying cannot repair.""" + status_code = getattr(exc, "status_code", None) + if status_code in (401, 403): + return True + body = getattr(exc, "body", None) + text = f"{body} {exc}".lower() + return any(marker in text for marker in _NON_RETRYABLE_ERROR_MARKERS) + class LLMClient: """封装 OpenAI 兼容接口的 LLM 客户端，支持重试与流式输出。""" @@ -45,6 +62,9 @@ def _build_client(self) -> OpenAI: base_url=self.config.base_url, default_headers=headers, timeout=timeout, + # Retry policy is implemented below so quota/auth failures can be + # classified correctly and are not retried inside the SDK first. + max_retries=0, ) def _prepare_messages(self, messages: list[Message]) -> list[dict[str, Any]]: @@ -127,7 +147,9 @@ def chat( kwargs = self._build_kwargs(messages, tools, temperature, stream=False) last_error: Exception | None = None max_attempts = self.config.max_retries_per_step + 1 + attempts_made = 0 for attempt in range(max_attempts): + attempts_made = attempt + 1 try: response = self._client.chat.completions.create(**kwargs) return parse_assistant_response(response) @@ -140,14 +162,16 @@ def chat( RemoteProtocolError, ) as exc: last_error = exc + if _is_non_retryable_api_error(exc): + break if attempt < self.config.max_retries_per_step: delay = min(2**attempt + random.random(), 60) time.sleep(delay) continue break - logger.error("LLM request failed after %s attempts: %s", max_attempts, last_error) - raise LLMError(f"LLM request failed after {max_attempts} attempts: {last_error}") + logger.error("LLM request failed after %s attempts: %s", attempts_made, last_error) + raise LLMError(f"LLM request failed after {attempts_made} attempts: {last_error}") def chat_stream( self, @@ -168,8 +192,10 @@ def chat_stream( kwargs = self._build_kwargs(messages, tools, temperature, stream=True) last_error: Exception | None = None max_attempts = self.config.max_retries_per_step + 1 + attempts_made = 0 for attempt in range(max_attempts): + attempts_made = attempt + 1 try: stream = self._client.chat.completions.create(**kwargs) yield from self._parse_stream(stream) @@ -183,13 +209,15 @@ def chat_stream( RemoteProtocolError, ) as exc: last_error = exc + if _is_non_retryable_api_error(exc): + break if attempt < self.config.max_retries_per_step: delay = min(2**attempt + random.random(), 60) time.sleep(delay) continue break - raise LLMError(f"LLM request failed after {max_attempts} attempts: {last_error}") + raise LLMError(f"LLM request failed after {attempts_made} attempts: {last_error}") def _parse_stream(self, stream: Any) -> Generator[str | AssistantResponse, None, None]: """解析 OpenAI 流式响应。""" diff --git a/scripts/compare_three_systems.py b/scripts/compare_three_systems.py index 9b67389..7fef0cc 100644 --- a/scripts/compare_three_systems.py +++ b/scripts/compare_three_systems.py @@ -27,6 +27,7 @@ sys.path.insert(0, str(REPO_ROOT)) from agent.config import Config, load_config # noqa: E402 +from agent.llm import LLMClient, Message # noqa: E402 from swe_bench.dataset import SWEBenchDataset, SWEBenchTask # noqa: E402 logger = logging.getLogger("compare_three_systems") @@ -361,6 +362,11 @@ def run_swe_agent( "ImportError", "No module named", "can't open file", + "insufficient_quota", + "quota has been exhausted", + "authentication", + "invalid_api_key", + "llm error", ) @@ -370,6 +376,19 @@ def is_infra_error(error: str | None) -> bool: return any(p.lower() in error.lower() for p in INFRA_ERROR_PATTERNS) +def preflight_openai_compatible_endpoint(config: Config, model: str) -> None: + """Fail before workspace setup when the shared direct/SWE endpoint is unusable.""" + original_model = config.llm.model + config.llm.model = model + try: + LLMClient(config.llm).chat( + [Message(role="user", content="Reply with exactly OK.")], + temperature=0.0, + ) + finally: + config.llm.model = original_model + + def evaluate_patch( task: SWEBenchTask, workspace: Path, patch: str, eval_output_dir: Path ) -> dict[str, Any]: @@ -491,7 +510,13 @@ def main() -> int: load_dotenv(REPO_ROOT / ".env") except ImportError: pass - config = load_config(args.config) if args.mode in ("direct", "all") else None + config = load_config(args.config) if args.mode in ("direct", "swe-agent", "all") else None + if config is not None: + try: + preflight_openai_compatible_endpoint(config, args.model) + except Exception as exc: + logger.error("shared direct/SWE-agent endpoint preflight failed: %s", exc) + return 2 for task in tasks: r = by_id[task.id] diff --git a/swe_bench/runner.py b/swe_bench/runner.py index f213c69..d78d1c8 100644 --- a/swe_bench/runner.py +++ b/swe_bench/runner.py @@ -379,7 +379,7 @@ def _run_task_direct( # The trusted benchmark runner grants shell consent explicitly. # Normal users cannot enable this path with an environment variable. - agent.run( + agent_answer = agent.run( goal_description=description, max_steps=self.config.llm.max_steps_per_turn, ) @@ -394,7 +394,11 @@ def _run_task_direct( success=False, resolved=False, duration_seconds=time.monotonic() - start, - error="agent produced an empty patch", + error=( + agent_answer + if agent_answer.startswith(("LLM error", "Reached token budget")) + else "agent produced an empty patch" + ), ) # Evaluate diff --git a/tests/test_llm.py b/tests/test_llm.py index 6fc4bb4..cbaa356 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -186,6 +186,23 @@ def test_client_retry_exhausted(): assert fake.call_count == 3 +def test_client_does_not_retry_exhausted_quota(): + fake = _FakeOpenAIClient( + responses=[ + _APIError( + "rate limited", + body={"error": {"code": "insufficient_quota"}}, + ) + ] + ) + config = LLMConfig(api_key="test-key", max_retries_per_step=5) + client = LLMClient(config=config, client=fake) + + with pytest.raises(LLMError, match="failed after 1 attempts"): + client.chat([Message(role="user", content="hi")]) + assert fake.call_count == 1 + + def test_client_prepares_tool_messages(): fake = _FakeOpenAIClient(responses=[_make_response(content="ok")]) config = LLMConfig(api_key="test-key") @@ -394,8 +411,8 @@ def _make_response( class _APIError(APIError): - def __init__(self, message: str): - super().__init__(message, request=None, body=None) # type: ignore[arg-type] + def __init__(self, message: str, body: Any = None): + super().__init__(message, request=None, body=body) # type: ignore[arg-type] class _FakeOpenAIClient: