From 2d0c00974933ae1ea13ef11b4dc6f6f7e003782f Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Sat, 29 Aug 2026 03:29:40 +0530 Subject: [PATCH 1/2] feat(tier3): rebuild ATIF from OpenCode and Codex agent logs Parse OpenCode JSON stream events into synthetic trajectories when trajectory.json is empty, and reuse the parser for structured Codex tee logs. Fail closed on non-dict tool inputs. Fixes #118 Signed-off-by: mimran-khan --- CHANGELOG.md | 6 + .../tier3/eval_core/log_converters.py | 165 ++++++++++++++++++ tests/tier3/test_log_converters.py | 80 +++++++++ 3 files changed, 251 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f54c4e84..d8f1ef1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to SkillEvaluator are documented in this file. ## Unreleased +### Added + +- Tier 3 log converters now rebuild ATIF trajectories from OpenCode JSON streams + (`opencode.txt`) and structured Codex tee logs (`codex.txt`) when + `trajectory.json` is missing or empty. + ### Fixed - Tier 3 accuracy and custom goal judges now retry one malformed (including diff --git a/src/skillevaluator/tier3/eval_core/log_converters.py b/src/skillevaluator/tier3/eval_core/log_converters.py index 5e29f2e7..2726ad10 100644 --- a/src/skillevaluator/tier3/eval_core/log_converters.py +++ b/src/skillevaluator/tier3/eval_core/log_converters.py @@ -385,6 +385,153 @@ def synthetic_trajectory_from_cline_cli(text: str) -> dict[str, Any] | None: } +def _normalize_opencode_tool(tool_name: str, raw_input: Any) -> tuple[str, dict[str, Any]]: + """Map OpenCode tool names and inputs to ATIF tool-call fields.""" + if not isinstance(raw_input, dict): + return tool_name, {} + name = str(tool_name or "").strip() + lowered = name.lower() + arguments = dict(raw_input) + if lowered in {"read", "read_file"}: + function_name = "read" + if "filePath" in arguments and "path" not in arguments: + arguments["path"] = arguments["filePath"] + if "file_path" in arguments and "path" not in arguments: + arguments["path"] = arguments["file_path"] + elif lowered in {"bash", "shell"}: + function_name = "bash" + elif lowered in {"write", "edit"}: + function_name = lowered + if "filePath" in arguments and "path" not in arguments: + arguments["path"] = arguments["filePath"] + else: + function_name = name or lowered or "tool" + return function_name, arguments + + +def _opencode_output_text(state: dict[str, Any]) -> str: + output = state.get("output") + if output is None: + return "" + if isinstance(output, str): + return output + if isinstance(output, dict): + message = output.get("message") or output.get("text") + if message is not None: + return str(message) + return json.dumps(output, ensure_ascii=False) + return str(output) + + +def synthetic_trajectory_from_opencode_json(text: str) -> dict[str, Any] | None: + """Parse OpenCode ``run --format=json`` JSONL (tee'd to ``opencode.txt``).""" + if not text or not text.strip(): + return None + + steps: list[dict[str, Any]] = [] + saw_content = False + + for raw_line in text.splitlines(): + line = raw_line.strip() + if not line: + continue + try: + evt = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(evt, dict): + continue + + et = str(evt.get("type") or "") + if et == "text": + part = evt.get("part") + if not isinstance(part, dict): + continue + msg = str(part.get("text") or "").strip() + if not msg: + continue + saw_content = True + if steps and not steps[-1].get("tool_calls"): + prev = (steps[-1].get("message") or "").strip() + steps[-1]["message"] = (prev + "\n" + msg).strip() if prev else msg + else: + steps.append( + { + "source": "agent", + "message": msg, + "tool_calls": [], + "observation": {"results": []}, + } + ) + continue + + if et != "tool_use": + continue + + part = evt.get("part") + if not isinstance(part, dict): + continue + state = part.get("state") + if not isinstance(state, dict): + continue + status = str(state.get("status") or "").lower() + if status in {"pending", "running"}: + continue + + tool_name = str(part.get("tool") or part.get("name") or "") + call_id = str(part.get("callID") or part.get("call_id") or part.get("id") or "") + function_name, arguments = _normalize_opencode_tool(tool_name, state.get("input")) + if not call_id: + call_id = f"opencode-{len(steps) + 1}" + + saw_content = True + step = { + "source": "agent", + "message": "", + "tool_calls": [ + { + "tool_call_id": call_id, + "function_name": function_name, + "arguments": arguments, + } + ], + "observation": {"results": []}, + } + output_text = _opencode_output_text(state) + if output_text: + step["observation"]["results"].append( + { + "source_call_id": call_id, + "content": output_text[:8000], + } + ) + steps.append(step) + + if not saw_content or not steps: + return None + return { + "steps": steps, + "schema_version": "ATIF-v1.2-synthetic-opencode-log", + "final_metrics": {}, + } + + +def synthetic_trajectory_from_codex_txt(text: str) -> dict[str, Any] | None: + """Reconstruct ATIF from Codex tee logs when they contain structured JSONL.""" + if not text or not text.strip(): + return None + + lines = [line.strip() for line in text.splitlines() if line.strip()] + if not lines: + return None + if all(line.startswith("{") and line.endswith("}") for line in lines): + synth = synthetic_trajectory_from_opencode_json(text) + if synth and synth.get("steps"): + synth["schema_version"] = "ATIF-v1.2-synthetic-codex-log" + return synth + return None + + def load_trajectory_with_fallback( trajectory_path: Path, logs_dir: Path | None = None, @@ -436,4 +583,22 @@ def load_trajectory_with_fallback( meta["note"] = "Synthetic ATIF from Cline CLI JSONL log" return synth, meta + opencode_path = logs / "opencode.txt" + if opencode_path.exists(): + raw = opencode_path.read_text(encoding="utf-8", errors="replace") + synth = synthetic_trajectory_from_opencode_json(raw) + if synth and synth.get("steps"): + meta["source"] = "opencode.txt" + meta["note"] = "Synthetic ATIF from OpenCode JSON stream" + return synth, meta + + codex_path = logs / "codex.txt" + if codex_path.exists(): + raw = codex_path.read_text(encoding="utf-8", errors="replace") + synth = synthetic_trajectory_from_codex_txt(raw) + if synth and synth.get("steps"): + meta["source"] = "codex.txt" + meta["note"] = "Synthetic ATIF from Codex structured log" + return synth, meta + return None, meta diff --git a/tests/tier3/test_log_converters.py b/tests/tier3/test_log_converters.py index b4d0c943..24a8a22c 100644 --- a/tests/tier3/test_log_converters.py +++ b/tests/tier3/test_log_converters.py @@ -12,7 +12,9 @@ from skillevaluator.tier3.eval_core.log_converters import ( load_trajectory_with_fallback, synthetic_trajectory_from_claude_stream_jsonl, + synthetic_trajectory_from_codex_txt, synthetic_trajectory_from_cursor_cli, + synthetic_trajectory_from_opencode_json, ) @@ -89,3 +91,81 @@ def test_load_prefers_claude_log_over_cursor_when_both(tmp_path: Path): assert meta["source"] == "claude-code.txt" tcs = extract_tool_calls_as_dicts(data) assert any(tc["action"] == "Bash" for tc in tcs) + + +def test_opencode_json_tool_use_and_read(): + log = ( + '{"type":"text","part":{"type":"text","text":"Reading skill"}}\n' + '{"type":"tool_use","part":{"type":"tool","tool":"read","callID":"call-1",' + '"state":{"status":"completed","input":{"filePath":"/workspace/skills/calculator/SKILL.md"},' + '"output":"# Calculator skill"}}}\n' + '{"type":"tool_use","part":{"type":"tool","tool":"bash","callID":"call-2",' + '"state":{"status":"completed","input":{"command":"cat skills/foo/SKILL.md"},' + '"output":"ok","metadata":{"exit":0}}}}\n' + ) + traj = synthetic_trajectory_from_opencode_json(log) + assert traj is not None + tcs = extract_tool_calls_as_dicts(traj) + assert len(tcs) == 2 + read_call = next(tc for tc in tcs if tc["action"] == "read") + assert "/calculator/" in json.dumps(read_call["action_input"]) + assert "Calculator" in read_call["observation"] + bash_call = next(tc for tc in tcs if tc["action"] == "bash") + assert "cat" in json.dumps(bash_call["action_input"]).lower() + + +def test_opencode_error_only_log_returns_none(): + log = json.dumps( + { + "type": "error", + "error": { + "name": "UnknownError", + "data": {"message": "ResourceExhausted: Worker local total request limit reached"}, + }, + } + ) + assert synthetic_trajectory_from_opencode_json(log + "\n") is None + + +def test_codex_structured_jsonl_reuses_opencode_parser(): + log = ( + '{"type":"tool_use","part":{"type":"tool","tool":"read","callID":"c1",' + '"state":{"status":"completed","input":{"path":"/workspace/skills/demo/SKILL.md"},' + '"output":"# Demo"}}}\n' + ) + traj = synthetic_trajectory_from_codex_txt(log) + assert traj is not None + assert traj["schema_version"] == "ATIF-v1.2-synthetic-codex-log" + tcs = extract_tool_calls_as_dicts(traj) + assert len(tcs) == 1 + assert tcs[0]["action"] == "read" + + +def test_codex_plain_text_errors_return_none(): + text = "ERROR responses_websocket: HTTP error: 405 Method Not Allowed\n" + assert synthetic_trajectory_from_codex_txt(text) is None + + +def test_load_falls_back_to_opencode_txt(tmp_path: Path): + logs = tmp_path / "agent" + logs.mkdir() + (logs / "opencode.txt").write_text( + '{"type":"tool_use","part":{"type":"tool","tool":"bash","callID":"c1",' + '"state":{"status":"completed","input":{"command":"echo hi"},"output":"hi"}}}\n', + encoding="utf-8", + ) + traj_path = logs / "trajectory.json" + data, meta = load_trajectory_with_fallback(traj_path, logs) + assert data is not None + assert meta["source"] == "opencode.txt" + + +def test_opencode_non_dict_tool_input_is_not_treated_as_shell_command(): + log = ( + '{"type":"tool_use","part":{"type":"tool","tool":"bash","callID":"c1",' + '"state":{"status":"completed","input":"cat /etc/passwd","output":"blocked"}}}\n' + ) + traj = synthetic_trajectory_from_opencode_json(log) + assert traj is not None + tcs = extract_tool_calls_as_dicts(traj) + assert tcs[0]["action_input"] == {} From 2c96e370ca8c4d2957c7200f7887f22a8ce645ef Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Sat, 29 Aug 2026 17:19:39 +0530 Subject: [PATCH 2/2] fix(tier3): parse Codex exec --json JSONL for ATIF fallback Codex tee logs use type=item / agent_message events, not OpenCode tool_use. Add a dedicated parser with stderr-tolerant JSONL scanning, keep OpenCode-shaped JSONL as a fallback, and normalize shell calls to bash for skill checks. Signed-off-by: mimran-khan --- .../tier3/eval_core/log_converters.py | 112 +++++++++++++++++- tests/tier3/test_log_converters.py | 32 ++++- 2 files changed, 142 insertions(+), 2 deletions(-) diff --git a/src/skillevaluator/tier3/eval_core/log_converters.py b/src/skillevaluator/tier3/eval_core/log_converters.py index 2726ad10..d0a65714 100644 --- a/src/skillevaluator/tier3/eval_core/log_converters.py +++ b/src/skillevaluator/tier3/eval_core/log_converters.py @@ -516,11 +516,121 @@ def synthetic_trajectory_from_opencode_json(text: str) -> dict[str, Any] | None: } +def _iter_jsonl_dicts(text: str) -> list[dict[str, Any]]: + """Parse JSONL lines, skipping non-JSON noise (e.g. stderr mixed into tee logs).""" + events: list[dict[str, Any]] = [] + for raw_line in text.splitlines(): + line = raw_line.strip() + if not line.startswith("{") or not line.endswith("}"): + continue + try: + evt = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(evt, dict): + events.append(evt) + return events + + +def _codex_item_text(item: dict[str, Any]) -> str | None: + item_type = str(item.get("type") or "") + if item_type not in {"message", "agent_message"}: + return None + content = item.get("content") + if not isinstance(content, list): + return None + parts: list[str] = [] + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + parts.append(str(block.get("text") or "")) + joined = "\n".join(parts).strip() + return joined or None + + +def _codex_function_arguments(function_call: dict[str, Any]) -> dict[str, Any]: + args = function_call.get("arguments") + if isinstance(args, dict): + return dict(args) + return {} + + +def synthetic_trajectory_from_codex_json(text: str) -> dict[str, Any] | None: + """Parse Codex ``exec --json`` JSONL (``type: item`` / ``agent_message``).""" + if not text or not text.strip(): + return None + + steps: list[dict[str, Any]] = [] + saw_content = False + message_index = 0 + + for evt in _iter_jsonl_dicts(text): + if str(evt.get("type") or "") != "item": + continue + item = evt.get("item") + if not isinstance(item, dict): + continue + if str(item.get("type") or "") != "agent_message": + continue + if not evt.get("item.completed", True): + continue + + message = _codex_item_text(item) + function_call = item.get("function_call") + if not message and not function_call: + continue + + saw_content = True + step: dict[str, Any] = { + "source": "agent", + "message": message or "", + "tool_calls": [], + "observation": {"results": []}, + } + + if isinstance(function_call, dict): + call_id = str( + item.get("id") or evt.get("item_id") or f"codex-{message_index + 1}" + ) + function_name = str(function_call.get("name") or "tool") + if function_name.lower() in {"bash", "shell"}: + function_name = "bash" + arguments = _codex_function_arguments(function_call) + step["tool_calls"] = [ + { + "tool_call_id": call_id, + "function_name": function_name, + "arguments": arguments, + } + ] + output = item.get("output") + if output is not None: + step["observation"]["results"].append( + { + "source_call_id": call_id, + "content": str(output)[:8000], + } + ) + message_index += 1 + steps.append(step) + + if not saw_content or not steps: + return None + return { + "steps": steps, + "schema_version": "ATIF-v1.2-synthetic-codex-log", + "final_metrics": {}, + } + + def synthetic_trajectory_from_codex_txt(text: str) -> dict[str, Any] | None: - """Reconstruct ATIF from Codex tee logs when they contain structured JSONL.""" + """Reconstruct ATIF from Codex tee logs (``exec --json`` JSONL or OpenCode-shaped JSONL).""" if not text or not text.strip(): return None + synth = synthetic_trajectory_from_codex_json(text) + if synth and synth.get("steps"): + return synth + lines = [line.strip() for line in text.splitlines() if line.strip()] if not lines: return None diff --git a/tests/tier3/test_log_converters.py b/tests/tier3/test_log_converters.py index 24a8a22c..9672161a 100644 --- a/tests/tier3/test_log_converters.py +++ b/tests/tier3/test_log_converters.py @@ -127,7 +127,37 @@ def test_opencode_error_only_log_returns_none(): assert synthetic_trajectory_from_opencode_json(log + "\n") is None -def test_codex_structured_jsonl_reuses_opencode_parser(): +def test_codex_exec_json_agent_message_and_shell_call(): + log = ( + '{"type":"item","item":{"type":"agent_message","id":"msg-1",' + '"content":[{"type":"text","text":"Reading skill file"}],' + '"function_call":{"name":"shell","arguments":{"command":"ls"}},' + '"output":"demo"},"item.completed":true}\n' + '{"type":"item","item":{"type":"agent_message","id":"msg-2",' + '"content":[{"type":"text","text":"Done"}]},' + '"item.completed":true}\n' + ) + traj = synthetic_trajectory_from_codex_txt(log) + assert traj is not None + assert traj["schema_version"] == "ATIF-v1.2-synthetic-codex-log" + tcs = extract_tool_calls_as_dicts(traj) + assert len(tcs) == 1 + assert tcs[0]["action"] == "bash" + + +def test_codex_exec_json_tolerates_stderr_noise(): + log = ( + "ERROR responses_websocket: HTTP error: 405 Method Not Allowed\n" + '{"type":"item","item":{"type":"agent_message","id":"msg-1",' + '"content":[{"type":"text","text":"hello"}]},' + '"item.completed":true}\n' + ) + traj = synthetic_trajectory_from_codex_txt(log) + assert traj is not None + assert len(traj["steps"]) == 1 + + +def test_codex_opencode_shaped_jsonl_fallback(): log = ( '{"type":"tool_use","part":{"type":"tool","tool":"read","callID":"c1",' '"state":{"status":"completed","input":{"path":"/workspace/skills/demo/SKILL.md"},'