fix(data): 생성·평가에서 실제 라벨 검증

README.md

@@ -26,7 +26,7 @@ No credential is embedded in the repository. Generation and OpenAI-compatible ev
 
 ## Generate semantic data
 
-The generator accepts one or more OpenAI-compatible endpoints. `--strict-target-tier` rejects missing or mismatched teacher labels instead of overwriting them.
+The generator accepts one or more OpenAI-compatible endpoints. Missing, invalid, or mismatched teacher labels trigger a new example generation; labels are never replaced with the target tier. With the default `--case-retries 2`, a case gets at most three attempts before it is excluded and recorded in `failures.jsonl`. The old `--strict-target-tier` flag remains accepted for compatibility.
 
 ```bash
 export VLLM_API_KEY=replace-with-local-serving-key
@@ -39,8 +39,7 @@ uv run python scripts/generate_semantic_synthetic_6tier.py \
   --model teacher-model \
   --concurrency 32 \
   --max-tokens 1536 \
-  --case-retries 2 \
-  --strict-target-tier
+  --case-retries 2
 ```
 
 ## Train a LoRA judge

scripts/evaluate_drift_sentry_model.py

@@ -11,16 +11,6 @@
 from pathlib import Path
 from typing import Any
 
-import torch
-from sklearn.metrics import classification_report, f1_score, precision_recall_fscore_support
-from transformers import AutoModelForCausalLM, AutoTokenizer
-
-try:
-    from peft import PeftModel
-except ModuleNotFoundError:  # pragma: no cover
-    PeftModel = None  # type: ignore[assignment]
-
-
 TIERS = ["goal", "reasoning", "environment", "integration", "memory", "reward", "normal"]
 RISK_TIERS = [tier for tier in TIERS if tier != "normal"]
 HIGH_RISK_TIERS = {"memory", "reward"}
@@ -32,6 +22,9 @@ def main() -> int:
     output_dir.mkdir(parents=True, exist_ok=True)
     rows = load_rows(Path(args.data_file), args.max_samples)
 
+    import torch
+    from transformers import AutoModelForCausalLM, AutoTokenizer
+
     tokenizer = AutoTokenizer.from_pretrained(
         args.model,
         revision=args.model_revision or None,
@@ -49,8 +42,10 @@ def main() -> int:
         trust_remote_code=args.trust_remote_code,
     )
     if args.adapter:
-        if PeftModel is None:
-            raise SystemExit("peft is required when --adapter is set")
+        try:
+            from peft import PeftModel
+        except ModuleNotFoundError as exc:
+            raise SystemExit("peft is required when --adapter is set") from exc
         model = PeftModel.from_pretrained(model, args.adapter)
     model.eval()
 
@@ -105,12 +100,17 @@ def load_rows(path: Path, max_samples: int) -> list[dict[str, Any]]:
             if not line.strip():
                 continue
             row = json.loads(line)
-            assistant = row["messages"][-1]
+            messages = row.get("messages")
+            if not isinstance(messages, list) or not messages:
+                raise ValueError(f"missing messages for {row.get('id')}")
+            assistant = messages[-1]
+            if not isinstance(assistant, dict) or assistant.get("role") != "assistant":
+                raise ValueError(f"missing assistant gold for {row.get('id')}")
             gold_payload = parse_jsonish(assistant["content"])
-            gold = normalize_tier(gold_payload.get("tier") or row.get("metadata", {}).get("target_tier"))
+            gold = normalize_tier(gold_payload.get("tier"))
             if gold not in TIERS:
                 raise ValueError(f"invalid gold tier for {row.get('id')}: {gold!r}")
-            rows.append({"id": row.get("id", ""), "messages": row["messages"][:-1], "gold_tier": gold})
+            rows.append({"id": row.get("id", ""), "messages": messages[:-1], "gold_tier": gold})
             if max_samples and len(rows) >= max_samples:
                 break
     return rows
@@ -126,6 +126,8 @@ def evaluate(
     max_input_tokens: int,
     max_new_tokens: int,
 ) -> dict[str, Any]:
+    import torch
+
     golds: list[str] = []
     preds: list[str] = []
     batch_elapsed: list[float] = []
@@ -178,6 +180,8 @@ def evaluate(
 
 
 def compute_metrics(golds: list[str], preds: list[str], batch_elapsed: list[float], batch_size: int) -> dict[str, Any]:
+    from sklearn.metrics import classification_report, f1_score, precision_recall_fscore_support
+
     labels_with_unknown = TIERS + ["unknown"]
     confusion = {gold: {pred: 0 for pred in labels_with_unknown} for gold in TIERS}
     for gold, pred in zip(golds, preds, strict=True):

scripts/finalize_drift_sentry_benchmark.py

@@ -235,7 +235,7 @@ def strict_tier(row: dict[str, Any]) -> str:
     tier = str(payload.get("tier") or "").strip().lower()
     metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
     target = str(metadata.get("target_tier") or "").strip().lower()
-    teacher = str(metadata.get("teacher_tier") or tier).strip().lower()
+    teacher = str(metadata.get("teacher_tier") or "").strip().lower()
     if tier not in TIERS or target != tier or teacher != tier:
         raise ValueError("tier mismatch")
     return tier

scripts/generate_semantic_synthetic_6tier.py

@@ -388,7 +388,6 @@ def main() -> int:
                     response_format_json=not args.no_response_format,
                     case_retries=args.case_retries,
                     labels=labels,
-                    strict_target_tier=args.strict_target_tier,
                     include_private_metadata=not args.public_records,
                 )
             )
@@ -457,7 +456,7 @@ def main() -> int:
         "taxonomy_path": str(args.taxonomy),
         "catalog_path": str(catalog_path),
         "selected_tiers": selected_tiers,
-        "strict_target_tier": args.strict_target_tier,
+        "strict_target_tier": True,
         "public_records": args.public_records,
         "max_failure_rate": args.max_failure_rate,
     }
@@ -479,7 +478,7 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument("--concurrency", type=int, default=32)
     parser.add_argument("--timeout", type=float, default=240.0)
     parser.add_argument("--max-retries", type=int, default=3)
-    parser.add_argument("--case-retries", type=int, default=2, help="Retry malformed teacher JSON/case outputs.")
+    parser.add_argument("--case-retries", type=int, default=2, help="Regenerate invalid cases or mismatched teacher labels.")
     parser.add_argument("--max-tokens", type=int, default=1536)
     parser.add_argument("--temperature", type=float, default=0.75)
     parser.add_argument("--enable-thinking", action="store_true", help="Allow Qwen thinking mode. Default disables it.")
@@ -505,7 +504,7 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument(
         "--strict-target-tier",
         action="store_true",
-        help="Reject and retry responses whose explicit teacher tier is missing, invalid, or differs from the target.",
+        help="Accepted for compatibility; target-tier validation is always enabled.",
     )
     parser.add_argument(
         "--public-records",
@@ -676,11 +675,9 @@ def generate_one(
     response_format_json: bool,
     case_retries: int,
     labels: list[str],
-    strict_target_tier: bool,
     include_private_metadata: bool,
 ) -> dict[str, Any]:
     archetype = target["archetype"]
-    rng = random.Random(target["seed"])
     domain = DOMAINS[archetype["domain"]]
     generation_messages = [
         {"role": "system", "content": GEN_SYSTEM_PROMPT},
@@ -711,8 +708,9 @@ def generate_one(
             retry_instruction = {
                 "role": "user",
                 "content": (
-                    "The previous response was malformed or incomplete. "
-                    "Return one complete JSON object only, with all required string fields and label."
+                    "The previous response was invalid or did not match the target tier. "
+                    "Generate a new trace that genuinely matches the requested tier, then label it. "
+                    "Do not change only the label. Return one complete JSON object with all required fields."
                 ),
             }
             attempt_messages = generation_messages + [retry_instruction]
@@ -733,8 +731,6 @@ def generate_one(
                 parse_json_object(raw_text),
                 target_tier=target["target_tier"],
                 labels=labels,
-                rng=rng,
-                strict_target_tier=strict_target_tier,
             )
             break
         except Exception as exc:  # noqa: BLE001
@@ -801,8 +797,6 @@ def normalize_case(
     *,
     target_tier: str,
     labels: list[str],
-    rng: random.Random,
-    strict_target_tier: bool = False,
 ) -> dict[str, Any]:
     case: dict[str, Any] = {}
     for field in CASE_FIELDS:
@@ -817,8 +811,6 @@ def normalize_case(
         label_raw,
         target_tier=target_tier,
         labels=labels,
-        rng=rng,
-        strict_target_tier=strict_target_tier,
     )
     case["label"] = label
     return case
@@ -829,18 +821,12 @@ def normalize_label(
     *,
     target_tier: str,
     labels: list[str],
-    rng: random.Random,
-    strict_target_tier: bool = False,
 ) -> dict[str, Any]:
-    raw_tier = str(raw.get("tier") or "").strip().lower()
-    if strict_target_tier:
-        if raw_tier not in labels:
-            raise ValueError(f"teacher tier missing or invalid: {raw_tier!r}")
-        if raw_tier != target_tier:
-            raise ValueError(f"teacher tier mismatch: target={target_tier} teacher={raw_tier}")
-    tier = raw_tier or target_tier
+    tier = str(raw.get("tier") or "").strip().lower()
     if tier not in labels:
-        tier = target_tier
+        raise ValueError(f"teacher tier missing or invalid: {tier!r}")
+    if tier != target_tier:
+        raise ValueError(f"teacher tier mismatch: target={target_tier} teacher={tier}")
     default = deterministic_label(tier)
     verdict = str(raw.get("verdict") or default["verdict"]).strip().upper()
     if verdict not in {"PASS", "BLOCK", "ESCALATE"}:

tests/test_label_integrity.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+from scripts import evaluate_drift_sentry_model as evaluator
+from scripts import generate_semantic_synthetic_6tier as generator
+
+
+class LabelIntegrityTest(unittest.TestCase):
+    def test_generation_retries_then_accepts_or_excludes(self) -> None:
+        target = {
+            "sample_index": 0,
+            "split": "train",
+            "target_tier": "goal",
+            "seed": 42,
+            "archetype": generator.build_archetype_catalog(["goal"])[0],
+        }
+        for tiers in (("goal",), ("reward", "goal"), ("reward",) * 3, (None,) * 3, ("invalid",) * 3):
+            replies = [
+                json.dumps(
+                    {
+                        **{field: f"example {index}: {field}" for field in generator.CASE_FIELDS},
+                        "label": {} if tier is None else {"tier": tier},
+                    }
+                )
+                for index, tier in enumerate(tiers)
+            ]
+            with self.subTest(tiers=tiers), patch.object(generator.time, "sleep"):
+                with patch.object(generator, "call_teacher", side_effect=replies) as teacher:
+                    result = generator.generate_one(
+                        target=target,
+                        taxonomy_brief="test taxonomy",
+                        base_url="http://localhost/v1",
+                        model="test",
+                        api_key="test",
+                        timeout=1,
+                        max_retries=0,
+                        max_tokens=512,
+                        temperature=0,
+                        enable_thinking=False,
+                        response_format_json=True,
+                        case_retries=2,
+                        labels=evaluator.TIERS,
+                        include_private_metadata=True,
+                    )
+                self.assertEqual(teacher.call_count, len(tiers))
+                self.assertEqual(result["ok"], tiers[-1] == "goal")
+                if result["ok"]:
+                    record = result["record"]
+                    self.assertEqual(record["metadata"]["raw_teacher_text"], replies[-1])
+                    self.assertEqual(json.loads(record["messages"][-1]["content"])["tier"], "goal")
+                else:
+                    self.assertNotIn("record", result)
+                    expected = "teacher_tier_mismatch" if tiers[-1] == "reward" else "teacher_tier_missing_or_invalid"
+                    self.assertEqual(result["failure_reason"], expected)
+
+    def test_evaluation_requires_assistant_gold(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "test.jsonl"
+            for role, label in (
+                ("assistant", {"tier": "normal"}),
+                ("assistant", {}),
+                ("assistant", {"tier": "unknown"}),
+                ("user", {"tier": "normal"}),
+            ):
+                with self.subTest(role=role, label=label):
+                    path.write_text(
+                        json.dumps(
+                            {
+                                "id": "case",
+                                "metadata": {"target_tier": "reward"},
+                                "messages": [{"role": role, "content": json.dumps(label)}],
+                            }
+                        )
+                        + "\n",
+                        encoding="utf-8",
+                    )
+                    if role == "assistant" and label.get("tier") == "normal":
+                        self.assertEqual(evaluator.load_rows(path, 0)[0]["gold_tier"], "normal")
+                    else:
+                        with self.assertRaises(ValueError):
+                            evaluator.load_rows(path, 0)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_public_benchmark_sanitizer.py

@@ -43,6 +43,19 @@ def test_finalizer_rejects_hyphen_secret_token(self) -> None:
         self.assertEqual(FINALIZER.find_pii("sk-" + "abcdefghijklmnop"), "secret_token")
         self.assertEqual(FINALIZER.find_pii("sk-" + "abcdefghijklmno-"), "secret_token")
 
+    def test_finalizer_requires_matching_teacher_tier(self) -> None:
+        for teacher_fields in ({"teacher_tier": "reward"}, {}, {"teacher_tier": ""}, {"teacher_tier": "goal"}):
+            with self.subTest(teacher_fields=teacher_fields):
+                row = {
+                    "messages": [{"role": "assistant", "content": '{"tier": "reward"}'}],
+                    "metadata": {"target_tier": "reward", **teacher_fields},
+                }
+                if teacher_fields.get("teacher_tier") == "reward":
+                    self.assertEqual(FINALIZER.strict_tier(row), "reward")
+                else:
+                    with self.assertRaisesRegex(ValueError, "tier mismatch"):
+                        FINALIZER.strict_tier(row)
+
     def test_transform_normalizes_public_provenance(self) -> None:
         row = {
             "messages": [{"role": "user", "content": "safe text"}],