diff --git a/CHANGELOG.md b/CHANGELOG.md index 57deee3a..2ef30e83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ All notable changes to SkillEvaluator are documented in this file. ### Added +- Tier 3 log converters now rebuild ATIF trajectories from OpenCode JSON streams + (`opencode.txt`) and structured Codex tee logs (`codex.txt`) when + `trajectory.json` is missing or empty. - SARIF 2.1.0 reporter (`-r sarif`) for GitHub Code Scanning and other SARIF consumers. Findings map to rule IDs, severity levels, and file locations from Tier 1 validation results. diff --git a/src/skillevaluator/tier3/eval_core/log_converters.py b/src/skillevaluator/tier3/eval_core/log_converters.py index 5e29f2e7..88d782ce 100644 --- a/src/skillevaluator/tier3/eval_core/log_converters.py +++ b/src/skillevaluator/tier3/eval_core/log_converters.py @@ -385,6 +385,300 @@ def synthetic_trajectory_from_cline_cli(text: str) -> dict[str, Any] | None: } +def _normalize_opencode_tool(tool_name: str, raw_input: Any) -> tuple[str, dict[str, Any]]: + """Map OpenCode tool names and inputs to ATIF tool-call fields.""" + if not isinstance(raw_input, dict): + return tool_name, {} + name = str(tool_name or "").strip() + lowered = name.lower() + arguments = dict(raw_input) + if lowered in {"read", "read_file"}: + function_name = "read" + if "filePath" in arguments and "path" not in arguments: + arguments["path"] = arguments["filePath"] + if "file_path" in arguments and "path" not in arguments: + arguments["path"] = arguments["file_path"] + elif lowered in {"bash", "shell"}: + function_name = "bash" + elif lowered in {"write", "edit"}: + function_name = lowered + if "filePath" in arguments and "path" not in arguments: + arguments["path"] = arguments["filePath"] + else: + function_name = name or lowered or "tool" + return function_name, arguments + + +def _opencode_output_text(state: dict[str, Any]) -> str: + output = state.get("output") + if output is None: + return "" + if isinstance(output, str): + return output + if isinstance(output, dict): + message = output.get("message") or output.get("text") + if message is not None: + return str(message) + return json.dumps(output, ensure_ascii=False) + return str(output) + + +def synthetic_trajectory_from_opencode_json(text: str) -> dict[str, Any] | None: + """Parse OpenCode ``run --format=json`` JSONL (tee'd to ``opencode.txt``).""" + if not text or not text.strip(): + return None + + steps: list[dict[str, Any]] = [] + saw_content = False + + for raw_line in text.splitlines(): + line = raw_line.strip() + if not line: + continue + try: + evt = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(evt, dict): + continue + + et = str(evt.get("type") or "") + if et == "text": + part = evt.get("part") + if not isinstance(part, dict): + continue + msg = str(part.get("text") or "").strip() + if not msg: + continue + saw_content = True + if steps and not steps[-1].get("tool_calls"): + prev = (steps[-1].get("message") or "").strip() + steps[-1]["message"] = (prev + "\n" + msg).strip() if prev else msg + else: + steps.append( + { + "source": "agent", + "message": msg, + "tool_calls": [], + "observation": {"results": []}, + } + ) + continue + + if et != "tool_use": + continue + + part = evt.get("part") + if not isinstance(part, dict): + continue + state = part.get("state") + if not isinstance(state, dict): + continue + status = str(state.get("status") or "").lower() + if status in {"pending", "running"}: + continue + + tool_name = str(part.get("tool") or part.get("name") or "") + call_id = str(part.get("callID") or part.get("call_id") or part.get("id") or "") + function_name, arguments = _normalize_opencode_tool(tool_name, state.get("input")) + if not call_id: + call_id = f"opencode-{len(steps) + 1}" + + saw_content = True + step = { + "source": "agent", + "message": "", + "tool_calls": [ + { + "tool_call_id": call_id, + "function_name": function_name, + "arguments": arguments, + } + ], + "observation": {"results": []}, + } + output_text = _opencode_output_text(state) + if output_text: + step["observation"]["results"].append( + { + "source_call_id": call_id, + "content": output_text[:8000], + } + ) + steps.append(step) + + if not saw_content or not steps: + return None + return { + "steps": steps, + "schema_version": "ATIF-v1.2-synthetic-opencode-log", + "final_metrics": {}, + } + + +def _iter_jsonl_dicts(text: str) -> list[dict[str, Any]]: + """Parse JSONL lines, skipping non-JSON noise (e.g. stderr mixed into tee logs).""" + events: list[dict[str, Any]] = [] + for raw_line in text.splitlines(): + line = raw_line.strip() + if not line.startswith("{") or not line.endswith("}"): + continue + try: + evt = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(evt, dict): + events.append(evt) + return events + + +def _codex_item_text(item: dict[str, Any]) -> str | None: + text = item.get("text") + if isinstance(text, str) and text.strip(): + return text.strip() + item_type = str(item.get("type") or "") + if item_type not in {"message", "agent_message"}: + return None + content = item.get("content") + if not isinstance(content, list): + return None + parts: list[str] = [] + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + parts.append(str(block.get("text") or "")) + joined = "\n".join(parts).strip() + return joined or None + + +def _codex_function_arguments(function_call: dict[str, Any]) -> dict[str, Any]: + args = function_call.get("arguments") + if isinstance(args, dict): + return dict(args) + return {} + + +def _codex_thread_item(evt: dict[str, Any]) -> dict[str, Any] | None: + """Return the Codex thread item payload from a completed event.""" + if str(evt.get("type") or "") == "item.completed": + item = evt.get("item") + return item if isinstance(item, dict) else None + if str(evt.get("type") or "") == "item" and evt.get("item.completed", True): + item = evt.get("item") + return item if isinstance(item, dict) else None + return None + + +def synthetic_trajectory_from_codex_json(text: str) -> dict[str, Any] | None: + """Parse Codex ``exec --json`` ThreadEvent JSONL.""" + if not text or not text.strip(): + return None + + steps: list[dict[str, Any]] = [] + saw_content = False + tool_index = 0 + + for evt in _iter_jsonl_dicts(text): + item = _codex_thread_item(evt) + if item is None: + continue + + item_type = str(item.get("type") or "") + + if item_type in {"agent_message", "message"}: + message = _codex_item_text(item) + function_call = item.get("function_call") + if not message and not isinstance(function_call, dict): + continue + saw_content = True + step: dict[str, Any] = { + "source": "agent", + "message": message or "", + "tool_calls": [], + "observation": {"results": []}, + } + if isinstance(function_call, dict): + call_id = str(item.get("id") or evt.get("item_id") or f"codex-{tool_index + 1}") + function_name = str(function_call.get("name") or "tool") + if function_name.lower() in {"bash", "shell"}: + function_name = "bash" + arguments = _codex_function_arguments(function_call) + step["tool_calls"] = [ + { + "tool_call_id": call_id, + "function_name": function_name, + "arguments": arguments, + } + ] + output = item.get("output") + if output is not None: + step["observation"]["results"].append( + { + "source_call_id": call_id, + "content": str(output)[:8000], + } + ) + tool_index += 1 + steps.append(step) + continue + + if item_type == "command_execution": + command = str(item.get("command") or "").strip() + if not command: + continue + saw_content = True + call_id = str(item.get("id") or evt.get("item_id") or f"codex-{tool_index + 1}") + tool_index += 1 + step = { + "source": "agent", + "message": "", + "tool_calls": [ + { + "tool_call_id": call_id, + "function_name": "bash", + "arguments": {"command": command}, + } + ], + "observation": {"results": []}, + } + output = item.get("aggregated_output") + if output is not None: + step["observation"]["results"].append( + { + "source_call_id": call_id, + "content": str(output)[:8000], + } + ) + steps.append(step) + + if not saw_content or not steps: + return None + return { + "steps": steps, + "schema_version": "ATIF-v1.2-synthetic-codex-log", + "final_metrics": {}, + } + + +def synthetic_trajectory_from_codex_txt(text: str) -> dict[str, Any] | None: + """Reconstruct ATIF from Codex tee logs (``exec --json`` JSONL or OpenCode-shaped JSONL).""" + if not text or not text.strip(): + return None + + synth = synthetic_trajectory_from_codex_json(text) + if synth and synth.get("steps"): + return synth + + lines = [line.strip() for line in text.splitlines() if line.strip()] + if not lines: + return None + if all(line.startswith("{") and line.endswith("}") for line in lines): + synth = synthetic_trajectory_from_opencode_json(text) + if synth and synth.get("steps"): + synth["schema_version"] = "ATIF-v1.2-synthetic-codex-log" + return synth + return None + + def load_trajectory_with_fallback( trajectory_path: Path, logs_dir: Path | None = None, @@ -436,4 +730,22 @@ def load_trajectory_with_fallback( meta["note"] = "Synthetic ATIF from Cline CLI JSONL log" return synth, meta + opencode_path = logs / "opencode.txt" + if opencode_path.exists(): + raw = opencode_path.read_text(encoding="utf-8", errors="replace") + synth = synthetic_trajectory_from_opencode_json(raw) + if synth and synth.get("steps"): + meta["source"] = "opencode.txt" + meta["note"] = "Synthetic ATIF from OpenCode JSON stream" + return synth, meta + + codex_path = logs / "codex.txt" + if codex_path.exists(): + raw = codex_path.read_text(encoding="utf-8", errors="replace") + synth = synthetic_trajectory_from_codex_txt(raw) + if synth and synth.get("steps"): + meta["source"] = "codex.txt" + meta["note"] = "Synthetic ATIF from Codex structured log" + return synth, meta + return None, meta diff --git a/src/skillevaluator/tier3/generate_dataset.py b/src/skillevaluator/tier3/generate_dataset.py index ca21d2dd..c29a5d9d 100644 --- a/src/skillevaluator/tier3/generate_dataset.py +++ b/src/skillevaluator/tier3/generate_dataset.py @@ -523,7 +523,16 @@ def _discover_trajectories( if not results_dir.exists(): return {} - agent_priority = ["claude-code", "cursor-cli", "codex", "openhands", "mini-swe-agent", "aider", "gemini-cli"] + agent_priority = [ + "claude-code", + "cursor-cli", + "opencode", + "codex", + "openhands", + "mini-swe-agent", + "aider", + "gemini-cli", + ] for agent_name in agent_priority: trials_dir = results_dir / agent_name / "with-skill" / "trials" if not trials_dir.exists(): diff --git a/tests/tier3/test_generate_dataset_results.py b/tests/tier3/test_generate_dataset_results.py index 45d1175d..24b93425 100644 --- a/tests/tier3/test_generate_dataset_results.py +++ b/tests/tier3/test_generate_dataset_results.py @@ -37,6 +37,29 @@ def test_discover_trajectories_uses_env_results_root(tmp_path, monkeypatch): assert _discover_trajectories(skill) == {"case-001": trajectory} +def test_discover_trajectories_opencode_txt_fallback(tmp_path): + skill = tmp_path / "my-skill" + skill.mkdir() + results_root = tmp_path / "results" + skill_results = results_root / "my-skill" + run_id = "20260709_120000" + run_dir = skill_results / run_id + trial = run_dir / "opencode" / "with-skill" / "trials" / "case-001" + trial.mkdir(parents=True) + (trial / "opencode.txt").write_text( + '{"type":"tool_use","part":{"type":"tool","tool":"bash","callID":"c1",' + '"state":{"status":"completed","input":{"command":"echo hi"},"output":"hi"}}}\n', + encoding="utf-8", + ) + (run_dir / "run_config.json").write_text("{}", encoding="utf-8") + (run_dir / "result.json").write_text(json.dumps({"run_id": run_id}), encoding="utf-8") + (skill_results / "latest").symlink_to(run_id) + + trajectories = _discover_trajectories(skill, results_dir=results_root) + assert "case-001" in trajectories + assert trajectories["case-001"].get("steps") + + def test_discover_trajectories_results_dir_overrides_env(tmp_path, monkeypatch): skill = tmp_path / "my-skill" skill.mkdir() diff --git a/tests/tier3/test_log_converters.py b/tests/tier3/test_log_converters.py index b4d0c943..a47ce3cb 100644 --- a/tests/tier3/test_log_converters.py +++ b/tests/tier3/test_log_converters.py @@ -12,7 +12,9 @@ from skillevaluator.tier3.eval_core.log_converters import ( load_trajectory_with_fallback, synthetic_trajectory_from_claude_stream_jsonl, + synthetic_trajectory_from_codex_txt, synthetic_trajectory_from_cursor_cli, + synthetic_trajectory_from_opencode_json, ) @@ -89,3 +91,129 @@ def test_load_prefers_claude_log_over_cursor_when_both(tmp_path: Path): assert meta["source"] == "claude-code.txt" tcs = extract_tool_calls_as_dicts(data) assert any(tc["action"] == "Bash" for tc in tcs) + + +def test_opencode_json_tool_use_and_read(): + log = ( + '{"type":"text","part":{"type":"text","text":"Reading skill"}}\n' + '{"type":"tool_use","part":{"type":"tool","tool":"read","callID":"call-1",' + '"state":{"status":"completed","input":{"filePath":"/workspace/skills/calculator/SKILL.md"},' + '"output":"# Calculator skill"}}}\n' + '{"type":"tool_use","part":{"type":"tool","tool":"bash","callID":"call-2",' + '"state":{"status":"completed","input":{"command":"cat skills/foo/SKILL.md"},' + '"output":"ok","metadata":{"exit":0}}}}\n' + ) + traj = synthetic_trajectory_from_opencode_json(log) + assert traj is not None + tcs = extract_tool_calls_as_dicts(traj) + assert len(tcs) == 2 + read_call = next(tc for tc in tcs if tc["action"] == "read") + assert "/calculator/" in json.dumps(read_call["action_input"]) + assert "Calculator" in read_call["observation"] + bash_call = next(tc for tc in tcs if tc["action"] == "bash") + assert "cat" in json.dumps(bash_call["action_input"]).lower() + + +def test_opencode_error_only_log_returns_none(): + log = json.dumps( + { + "type": "error", + "error": { + "name": "UnknownError", + "data": {"message": "ResourceExhausted: Worker local total request limit reached"}, + }, + } + ) + assert synthetic_trajectory_from_opencode_json(log + "\n") is None + + +def test_codex_thread_event_agent_message_and_command_execution(): + log = ( + '{"type":"item.completed","item":{"type":"agent_message","id":"msg-1",' + '"text":"Reading skill file"}}\n' + '{"type":"item.completed","item":{"type":"command_execution","id":"cmd-1",' + '"command":"ls","aggregated_output":"demo"}}\n' + '{"type":"item.completed","item":{"type":"agent_message","id":"msg-2",' + '"text":"Done"}}\n' + ) + traj = synthetic_trajectory_from_codex_txt(log) + assert traj is not None + assert traj["schema_version"] == "ATIF-v1.2-synthetic-codex-log" + tcs = extract_tool_calls_as_dicts(traj) + assert len(tcs) == 1 + assert tcs[0]["action"] == "bash" + assert tcs[0]["action_input"]["command"] == "ls" + + +def test_codex_exec_json_agent_message_and_shell_call(): + log = ( + '{"type":"item","item":{"type":"agent_message","id":"msg-1",' + '"content":[{"type":"text","text":"Reading skill file"}],' + '"function_call":{"name":"shell","arguments":{"command":"ls"}},' + '"output":"demo"},"item.completed":true}\n' + '{"type":"item","item":{"type":"agent_message","id":"msg-2",' + '"content":[{"type":"text","text":"Done"}]},' + '"item.completed":true}\n' + ) + traj = synthetic_trajectory_from_codex_txt(log) + assert traj is not None + assert traj["schema_version"] == "ATIF-v1.2-synthetic-codex-log" + tcs = extract_tool_calls_as_dicts(traj) + assert len(tcs) == 1 + assert tcs[0]["action"] == "bash" + + +def test_codex_exec_json_tolerates_stderr_noise(): + log = ( + "ERROR responses_websocket: HTTP error: 405 Method Not Allowed\n" + '{"type":"item","item":{"type":"agent_message","id":"msg-1",' + '"content":[{"type":"text","text":"hello"}]},' + '"item.completed":true}\n' + ) + traj = synthetic_trajectory_from_codex_txt(log) + assert traj is not None + assert len(traj["steps"]) == 1 + + +def test_codex_opencode_shaped_jsonl_fallback(): + log = ( + '{"type":"tool_use","part":{"type":"tool","tool":"read","callID":"c1",' + '"state":{"status":"completed","input":{"path":"/workspace/skills/demo/SKILL.md"},' + '"output":"# Demo"}}}\n' + ) + traj = synthetic_trajectory_from_codex_txt(log) + assert traj is not None + assert traj["schema_version"] == "ATIF-v1.2-synthetic-codex-log" + tcs = extract_tool_calls_as_dicts(traj) + assert len(tcs) == 1 + assert tcs[0]["action"] == "read" + + +def test_codex_plain_text_errors_return_none(): + text = "ERROR responses_websocket: HTTP error: 405 Method Not Allowed\n" + assert synthetic_trajectory_from_codex_txt(text) is None + + +def test_load_falls_back_to_opencode_txt(tmp_path: Path): + logs = tmp_path / "agent" + logs.mkdir() + (logs / "opencode.txt").write_text( + '{"type":"tool_use","part":{"type":"tool","tool":"bash","callID":"c1",' + '"state":{"status":"completed","input":{"command":"echo hi"},"output":"hi"}}}\n', + encoding="utf-8", + ) + traj_path = logs / "trajectory.json" + data, meta = load_trajectory_with_fallback(traj_path, logs) + assert data is not None + assert meta["source"] == "opencode.txt" + + +def test_opencode_non_dict_tool_input_is_not_treated_as_shell_command(): + log = ( + '{"type":"tool_use","part":{"type":"tool","tool":"bash","callID":"c1",' + '"state":{"status":"completed","input":"cat /etc/passwd","output":"blocked"}}}\n' + ) + traj = synthetic_trajectory_from_opencode_json(log) + assert traj is not None + tcs = extract_tool_calls_as_dicts(traj) + assert tcs[0]["action_input"] == {}