From 1240fbe4792423c98eba0512323ea68f947ed174 Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Thu, 27 Aug 2026 01:45:33 +0530 Subject: [PATCH] fix(tier3): map Harbor trial folders to eval case ids --refine keyed trajectories by the raw trial directory name, but Harbor writes folders like case-001__Lmi47iy. Split off the suffix so trajectories.get(case id) actually finds them. Fixes #93 Signed-off-by: mimran-khan --- CHANGELOG.md | 2 ++ src/skillevaluator/tier3/generate_dataset.py | 9 +++++++- tests/tier3/test_generate_dataset_results.py | 23 ++++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2ad6515..77bd5899 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ All notable changes to SkillEvaluator are documented in this file. ### Fixed +- `create-eval-dataset --refine` now maps Harbor trial folders + (`case-id__suffix`) back to eval case ids so trajectories attach. - Tier 3 paired pass@k evidence now respects Python's active integer-string conversion limit, preserves nonzero Wilson interval widths and paired-effect directions at large case counts, and documents exact-rational omission diff --git a/src/skillevaluator/tier3/generate_dataset.py b/src/skillevaluator/tier3/generate_dataset.py index ca21d2dd..36cb215e 100644 --- a/src/skillevaluator/tier3/generate_dataset.py +++ b/src/skillevaluator/tier3/generate_dataset.py @@ -496,6 +496,11 @@ def _ensure_project_imports(): sys.path.insert(0, src_dir) +def _case_id_from_trial_dir(trial_dir_name: str) -> str: + """Map Harbor trial folders like ``case-001__Lmi47iy`` back to the eval case id.""" + return trial_dir_name.split("__", 1)[0] + + def _discover_trajectories( skill_path: Path, from_results: str | None = None, @@ -534,7 +539,9 @@ def _discover_trajectories( for trial_dir in sorted(trials_dir.iterdir()): if not trial_dir.is_dir(): continue - case_id = trial_dir.name + case_id = _case_id_from_trial_dir(trial_dir.name) + if not case_id: + continue traj_path = trial_dir / "trajectory.json" traj, meta = load_trajectory_with_fallback(traj_path, logs_dir=trial_dir) if traj and traj.get("steps"): diff --git a/tests/tier3/test_generate_dataset_results.py b/tests/tier3/test_generate_dataset_results.py index 45d1175d..52f6c97f 100644 --- a/tests/tier3/test_generate_dataset_results.py +++ b/tests/tier3/test_generate_dataset_results.py @@ -37,6 +37,29 @@ def test_discover_trajectories_uses_env_results_root(tmp_path, monkeypatch): assert _discover_trajectories(skill) == {"case-001": trajectory} +def test_discover_trajectories_maps_harbor_trial_folder_to_case_id(tmp_path, monkeypatch): + """Harbor persists trials as ``{case_id}__{suffix}``; refine looks up by case id.""" + skill = tmp_path / "demo" + skill.mkdir() + results_root = tmp_path / "results" + run_id = "20260709_120000" + run_dir = results_root / "demo" / run_id + trial = run_dir / "claude-code" / "with-skill" / "trials" / "demo-001__Lmi47iy" + trial.mkdir(parents=True) + trajectory = {"steps": [{"tool_calls": [{"tool": "Read"}]}]} + trial.joinpath("trajectory.json").write_text(json.dumps(trajectory), encoding="utf-8") + (run_dir / "run_config.json").write_text("{}", encoding="utf-8") + (run_dir / "result.json").write_text(json.dumps({"run_id": run_id}), encoding="utf-8") + (results_root / "demo" / "latest").symlink_to(run_id) + + monkeypatch.setenv("SKILLEVALUATOR_RESULTS_DIR", str(results_root)) + + found = _discover_trajectories(skill) + assert "demo-001" in found + assert "demo-001__Lmi47iy" not in found + assert found["demo-001"] == trajectory + + def test_discover_trajectories_results_dir_overrides_env(tmp_path, monkeypatch): skill = tmp_path / "my-skill" skill.mkdir()