feat(tier3): rebuild ATIF from OpenCode and Codex agent logs

CHANGELOG.md

@@ -6,6 +6,9 @@ All notable changes to SkillEvaluator are documented in this file.
 
 ### Added
 
+- Tier 3 log converters now rebuild ATIF trajectories from OpenCode JSON streams
+  (`opencode.txt`) and structured Codex tee logs (`codex.txt`) when
+  `trajectory.json` is missing or empty.
 - SARIF 2.1.0 reporter (`-r sarif`) for GitHub Code Scanning and other SARIF
   consumers. Findings map to rule IDs, severity levels, and file locations from
   Tier 1 validation results.

src/skillevaluator/tier3/eval_core/log_converters.py

@@ -385,6 +385,300 @@ def synthetic_trajectory_from_cline_cli(text: str) -> dict[str, Any] | None:
     }
 
 
+def _normalize_opencode_tool(tool_name: str, raw_input: Any) -> tuple[str, dict[str, Any]]:
+    """Map OpenCode tool names and inputs to ATIF tool-call fields."""
+    if not isinstance(raw_input, dict):
+        return tool_name, {}
+    name = str(tool_name or "").strip()
+    lowered = name.lower()
+    arguments = dict(raw_input)
+    if lowered in {"read", "read_file"}:
+        function_name = "read"
+        if "filePath" in arguments and "path" not in arguments:
+            arguments["path"] = arguments["filePath"]
+        if "file_path" in arguments and "path" not in arguments:
+            arguments["path"] = arguments["file_path"]
+    elif lowered in {"bash", "shell"}:
+        function_name = "bash"
+    elif lowered in {"write", "edit"}:
+        function_name = lowered
+        if "filePath" in arguments and "path" not in arguments:
+            arguments["path"] = arguments["filePath"]
+    else:
+        function_name = name or lowered or "tool"
+    return function_name, arguments
+
+
+def _opencode_output_text(state: dict[str, Any]) -> str:
+    output = state.get("output")
+    if output is None:
+        return ""
+    if isinstance(output, str):
+        return output
+    if isinstance(output, dict):
+        message = output.get("message") or output.get("text")
+        if message is not None:
+            return str(message)
+        return json.dumps(output, ensure_ascii=False)
+    return str(output)
+
+
+def synthetic_trajectory_from_opencode_json(text: str) -> dict[str, Any] | None:
+    """Parse OpenCode ``run --format=json`` JSONL (tee'd to ``opencode.txt``)."""
+    if not text or not text.strip():
+        return None
+
+    steps: list[dict[str, Any]] = []
+    saw_content = False
+
+    for raw_line in text.splitlines():
+        line = raw_line.strip()
+        if not line:
+            continue
+        try:
+            evt = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if not isinstance(evt, dict):
+            continue
+
+        et = str(evt.get("type") or "")
+        if et == "text":
+            part = evt.get("part")
+            if not isinstance(part, dict):
+                continue
+            msg = str(part.get("text") or "").strip()
+            if not msg:
+                continue
+            saw_content = True
+            if steps and not steps[-1].get("tool_calls"):
+                prev = (steps[-1].get("message") or "").strip()
+                steps[-1]["message"] = (prev + "\n" + msg).strip() if prev else msg
+            else:
+                steps.append(
+                    {
+                        "source": "agent",
+                        "message": msg,
+                        "tool_calls": [],
+                        "observation": {"results": []},
+                    }
+                )
+            continue
+
+        if et != "tool_use":
+            continue
+
+        part = evt.get("part")
+        if not isinstance(part, dict):
+            continue
+        state = part.get("state")
+        if not isinstance(state, dict):
+            continue
+        status = str(state.get("status") or "").lower()
+        if status in {"pending", "running"}:
+            continue
+
+        tool_name = str(part.get("tool") or part.get("name") or "")
+        call_id = str(part.get("callID") or part.get("call_id") or part.get("id") or "")
+        function_name, arguments = _normalize_opencode_tool(tool_name, state.get("input"))
+        if not call_id:
+            call_id = f"opencode-{len(steps) + 1}"
+
+        saw_content = True
+        step = {
+            "source": "agent",
+            "message": "",
+            "tool_calls": [
+                {
+                    "tool_call_id": call_id,
+                    "function_name": function_name,
+                    "arguments": arguments,
+                }
+            ],
+            "observation": {"results": []},
+        }
+        output_text = _opencode_output_text(state)
+        if output_text:
+            step["observation"]["results"].append(
+                {
+                    "source_call_id": call_id,
+                    "content": output_text[:8000],
+                }
+            )
+        steps.append(step)
+
+    if not saw_content or not steps:
+        return None
+    return {
+        "steps": steps,
+        "schema_version": "ATIF-v1.2-synthetic-opencode-log",
+        "final_metrics": {},
+    }
+
+
+def _iter_jsonl_dicts(text: str) -> list[dict[str, Any]]:
+    """Parse JSONL lines, skipping non-JSON noise (e.g. stderr mixed into tee logs)."""
+    events: list[dict[str, Any]] = []
+    for raw_line in text.splitlines():
+        line = raw_line.strip()
+        if not line.startswith("{") or not line.endswith("}"):
+            continue
+        try:
+            evt = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(evt, dict):
+            events.append(evt)
+    return events
+
+
+def _codex_item_text(item: dict[str, Any]) -> str | None:
+    text = item.get("text")
+    if isinstance(text, str) and text.strip():
+        return text.strip()
+    item_type = str(item.get("type") or "")
+    if item_type not in {"message", "agent_message"}:
+        return None
+    content = item.get("content")
+    if not isinstance(content, list):
+        return None
+    parts: list[str] = []
+    for block in content:
+        if isinstance(block, dict) and block.get("type") == "text":
+            parts.append(str(block.get("text") or ""))
+    joined = "\n".join(parts).strip()
+    return joined or None
+
+
+def _codex_function_arguments(function_call: dict[str, Any]) -> dict[str, Any]:
+    args = function_call.get("arguments")
+    if isinstance(args, dict):
+        return dict(args)
+    return {}
+
+
+def _codex_thread_item(evt: dict[str, Any]) -> dict[str, Any] | None:
+    """Return the Codex thread item payload from a completed event."""
+    if str(evt.get("type") or "") == "item.completed":
+        item = evt.get("item")
+        return item if isinstance(item, dict) else None
+    if str(evt.get("type") or "") == "item" and evt.get("item.completed", True):
+        item = evt.get("item")
+        return item if isinstance(item, dict) else None
+    return None
+
+
+def synthetic_trajectory_from_codex_json(text: str) -> dict[str, Any] | None:
+    """Parse Codex ``exec --json`` ThreadEvent JSONL."""
+    if not text or not text.strip():
+        return None
+
+    steps: list[dict[str, Any]] = []
+    saw_content = False
+    tool_index = 0
+
+    for evt in _iter_jsonl_dicts(text):
+        item = _codex_thread_item(evt)
+        if item is None:
+            continue
+
+        item_type = str(item.get("type") or "")
+
+        if item_type in {"agent_message", "message"}:
+            message = _codex_item_text(item)
+            function_call = item.get("function_call")
+            if not message and not isinstance(function_call, dict):
+                continue
+            saw_content = True
+            step: dict[str, Any] = {
+                "source": "agent",
+                "message": message or "",
+                "tool_calls": [],
+                "observation": {"results": []},
+            }
+            if isinstance(function_call, dict):
+                call_id = str(item.get("id") or evt.get("item_id") or f"codex-{tool_index + 1}")
+                function_name = str(function_call.get("name") or "tool")
+                if function_name.lower() in {"bash", "shell"}:
+                    function_name = "bash"
+                arguments = _codex_function_arguments(function_call)
+                step["tool_calls"] = [
+                    {
+                        "tool_call_id": call_id,
+                        "function_name": function_name,
+                        "arguments": arguments,
+                    }
+                ]
+                output = item.get("output")
+                if output is not None:
+                    step["observation"]["results"].append(
+                        {
+                            "source_call_id": call_id,
+                            "content": str(output)[:8000],
+                        }
+                    )
+                tool_index += 1
+            steps.append(step)
+            continue
+
+        if item_type == "command_execution":
+            command = str(item.get("command") or "").strip()
+            if not command:
+                continue
+            saw_content = True
+            call_id = str(item.get("id") or evt.get("item_id") or f"codex-{tool_index + 1}")
+            tool_index += 1
+            step = {
+                "source": "agent",
+                "message": "",
+                "tool_calls": [
+                    {
+                        "tool_call_id": call_id,
+                        "function_name": "bash",
+                        "arguments": {"command": command},
+                    }
+                ],
+                "observation": {"results": []},
+            }
+            output = item.get("aggregated_output")
+            if output is not None:
+                step["observation"]["results"].append(
+                    {
+                        "source_call_id": call_id,
+                        "content": str(output)[:8000],
+                    }
+                )
+            steps.append(step)
+
+    if not saw_content or not steps:
+        return None
+    return {
+        "steps": steps,
+        "schema_version": "ATIF-v1.2-synthetic-codex-log",
+        "final_metrics": {},
+    }
+
+
+def synthetic_trajectory_from_codex_txt(text: str) -> dict[str, Any] | None:
+    """Reconstruct ATIF from Codex tee logs (``exec --json`` JSONL or OpenCode-shaped JSONL)."""
+    if not text or not text.strip():
+        return None
+
+    synth = synthetic_trajectory_from_codex_json(text)
+    if synth and synth.get("steps"):
+        return synth
+
+    lines = [line.strip() for line in text.splitlines() if line.strip()]
+    if not lines:
+        return None
+    if all(line.startswith("{") and line.endswith("}") for line in lines):
+        synth = synthetic_trajectory_from_opencode_json(text)
+        if synth and synth.get("steps"):
+            synth["schema_version"] = "ATIF-v1.2-synthetic-codex-log"
+            return synth
+    return None
+
+
 def load_trajectory_with_fallback(
     trajectory_path: Path,
     logs_dir: Path | None = None,
@@ -436,4 +730,22 @@ def load_trajectory_with_fallback(
             meta["note"] = "Synthetic ATIF from Cline CLI JSONL log"
             return synth, meta
 
+    opencode_path = logs / "opencode.txt"
+    if opencode_path.exists():
+        raw = opencode_path.read_text(encoding="utf-8", errors="replace")
+        synth = synthetic_trajectory_from_opencode_json(raw)
+        if synth and synth.get("steps"):
+            meta["source"] = "opencode.txt"
+            meta["note"] = "Synthetic ATIF from OpenCode JSON stream"
+            return synth, meta
+
+    codex_path = logs / "codex.txt"
+    if codex_path.exists():
+        raw = codex_path.read_text(encoding="utf-8", errors="replace")
+        synth = synthetic_trajectory_from_codex_txt(raw)
+        if synth and synth.get("steps"):
+            meta["source"] = "codex.txt"
+            meta["note"] = "Synthetic ATIF from Codex structured log"
+            return synth, meta
+
     return None, meta

src/skillevaluator/tier3/generate_dataset.py

@@ -523,7 +523,16 @@ def _discover_trajectories(
     if not results_dir.exists():
         return {}
 
-    agent_priority = ["claude-code", "cursor-cli", "codex", "openhands", "mini-swe-agent", "aider", "gemini-cli"]
+    agent_priority = [
+        "claude-code",
+        "cursor-cli",
+        "opencode",
+        "codex",
+        "openhands",
+        "mini-swe-agent",
+        "aider",
+        "gemini-cli",
+    ]
     for agent_name in agent_priority:
         trials_dir = results_dir / agent_name / "with-skill" / "trials"
         if not trials_dir.exists():

tests/tier3/test_generate_dataset_results.py

@@ -37,6 +37,29 @@ def test_discover_trajectories_uses_env_results_root(tmp_path, monkeypatch):
     assert _discover_trajectories(skill) == {"case-001": trajectory}
 
 
+def test_discover_trajectories_opencode_txt_fallback(tmp_path):
+    skill = tmp_path / "my-skill"
+    skill.mkdir()
+    results_root = tmp_path / "results"
+    skill_results = results_root / "my-skill"
+    run_id = "20260709_120000"
+    run_dir = skill_results / run_id
+    trial = run_dir / "opencode" / "with-skill" / "trials" / "case-001"
+    trial.mkdir(parents=True)
+    (trial / "opencode.txt").write_text(
+        '{"type":"tool_use","part":{"type":"tool","tool":"bash","callID":"c1",'
+        '"state":{"status":"completed","input":{"command":"echo hi"},"output":"hi"}}}\n',
+        encoding="utf-8",
+    )
+    (run_dir / "run_config.json").write_text("{}", encoding="utf-8")
+    (run_dir / "result.json").write_text(json.dumps({"run_id": run_id}), encoding="utf-8")
+    (skill_results / "latest").symlink_to(run_id)
+
+    trajectories = _discover_trajectories(skill, results_dir=results_root)
+    assert "case-001" in trajectories
+    assert trajectories["case-001"].get("steps")
+
+
 def test_discover_trajectories_results_dir_overrides_env(tmp_path, monkeypatch):
     skill = tmp_path / "my-skill"
     skill.mkdir()