From c3e50d6da1645c474ba1ff881da5c9b34d9fada5 Mon Sep 17 00:00:00 2001 From: Myungsub Choi Date: Wed, 7 Oct 2026 11:23:03 +0900 Subject: [PATCH] fix(data): require observed labels in generation and evaluation --- README.md | 5 +- scripts/evaluate_drift_sentry_model.py | 34 ++++---- scripts/finalize_drift_sentry_benchmark.py | 2 +- scripts/generate_semantic_synthetic_6tier.py | 34 +++----- tests/test_label_integrity.py | 90 ++++++++++++++++++++ tests/test_public_benchmark_sanitizer.py | 13 +++ 6 files changed, 135 insertions(+), 43 deletions(-) create mode 100644 tests/test_label_integrity.py diff --git a/README.md b/README.md index bc64dd1..eb6215b 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ No credential is embedded in the repository. Generation and OpenAI-compatible ev ## Generate semantic data -The generator accepts one or more OpenAI-compatible endpoints. `--strict-target-tier` rejects missing or mismatched teacher labels instead of overwriting them. +The generator accepts one or more OpenAI-compatible endpoints. Missing, invalid, or mismatched teacher labels trigger a new example generation; labels are never replaced with the target tier. With the default `--case-retries 2`, a case gets at most three attempts before it is excluded and recorded in `failures.jsonl`. The old `--strict-target-tier` flag remains accepted for compatibility. ```bash export VLLM_API_KEY=replace-with-local-serving-key @@ -39,8 +39,7 @@ uv run python scripts/generate_semantic_synthetic_6tier.py \ --model teacher-model \ --concurrency 32 \ --max-tokens 1536 \ - --case-retries 2 \ - --strict-target-tier + --case-retries 2 ``` ## Train a LoRA judge diff --git a/scripts/evaluate_drift_sentry_model.py b/scripts/evaluate_drift_sentry_model.py index 9c6e5b6..0758255 100644 --- a/scripts/evaluate_drift_sentry_model.py +++ b/scripts/evaluate_drift_sentry_model.py @@ -11,16 +11,6 @@ from pathlib import Path from typing import Any -import torch -from sklearn.metrics import classification_report, f1_score, precision_recall_fscore_support -from transformers import AutoModelForCausalLM, AutoTokenizer - -try: - from peft import PeftModel -except ModuleNotFoundError: # pragma: no cover - PeftModel = None # type: ignore[assignment] - - TIERS = ["goal", "reasoning", "environment", "integration", "memory", "reward", "normal"] RISK_TIERS = [tier for tier in TIERS if tier != "normal"] HIGH_RISK_TIERS = {"memory", "reward"} @@ -32,6 +22,9 @@ def main() -> int: output_dir.mkdir(parents=True, exist_ok=True) rows = load_rows(Path(args.data_file), args.max_samples) + import torch + from transformers import AutoModelForCausalLM, AutoTokenizer + tokenizer = AutoTokenizer.from_pretrained( args.model, revision=args.model_revision or None, @@ -49,8 +42,10 @@ def main() -> int: trust_remote_code=args.trust_remote_code, ) if args.adapter: - if PeftModel is None: - raise SystemExit("peft is required when --adapter is set") + try: + from peft import PeftModel + except ModuleNotFoundError as exc: + raise SystemExit("peft is required when --adapter is set") from exc model = PeftModel.from_pretrained(model, args.adapter) model.eval() @@ -105,12 +100,17 @@ def load_rows(path: Path, max_samples: int) -> list[dict[str, Any]]: if not line.strip(): continue row = json.loads(line) - assistant = row["messages"][-1] + messages = row.get("messages") + if not isinstance(messages, list) or not messages: + raise ValueError(f"missing messages for {row.get('id')}") + assistant = messages[-1] + if not isinstance(assistant, dict) or assistant.get("role") != "assistant": + raise ValueError(f"missing assistant gold for {row.get('id')}") gold_payload = parse_jsonish(assistant["content"]) - gold = normalize_tier(gold_payload.get("tier") or row.get("metadata", {}).get("target_tier")) + gold = normalize_tier(gold_payload.get("tier")) if gold not in TIERS: raise ValueError(f"invalid gold tier for {row.get('id')}: {gold!r}") - rows.append({"id": row.get("id", ""), "messages": row["messages"][:-1], "gold_tier": gold}) + rows.append({"id": row.get("id", ""), "messages": messages[:-1], "gold_tier": gold}) if max_samples and len(rows) >= max_samples: break return rows @@ -126,6 +126,8 @@ def evaluate( max_input_tokens: int, max_new_tokens: int, ) -> dict[str, Any]: + import torch + golds: list[str] = [] preds: list[str] = [] batch_elapsed: list[float] = [] @@ -178,6 +180,8 @@ def evaluate( def compute_metrics(golds: list[str], preds: list[str], batch_elapsed: list[float], batch_size: int) -> dict[str, Any]: + from sklearn.metrics import classification_report, f1_score, precision_recall_fscore_support + labels_with_unknown = TIERS + ["unknown"] confusion = {gold: {pred: 0 for pred in labels_with_unknown} for gold in TIERS} for gold, pred in zip(golds, preds, strict=True): diff --git a/scripts/finalize_drift_sentry_benchmark.py b/scripts/finalize_drift_sentry_benchmark.py index b2c943d..797582c 100644 --- a/scripts/finalize_drift_sentry_benchmark.py +++ b/scripts/finalize_drift_sentry_benchmark.py @@ -235,7 +235,7 @@ def strict_tier(row: dict[str, Any]) -> str: tier = str(payload.get("tier") or "").strip().lower() metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {} target = str(metadata.get("target_tier") or "").strip().lower() - teacher = str(metadata.get("teacher_tier") or tier).strip().lower() + teacher = str(metadata.get("teacher_tier") or "").strip().lower() if tier not in TIERS or target != tier or teacher != tier: raise ValueError("tier mismatch") return tier diff --git a/scripts/generate_semantic_synthetic_6tier.py b/scripts/generate_semantic_synthetic_6tier.py index f7788aa..c60abdd 100644 --- a/scripts/generate_semantic_synthetic_6tier.py +++ b/scripts/generate_semantic_synthetic_6tier.py @@ -388,7 +388,6 @@ def main() -> int: response_format_json=not args.no_response_format, case_retries=args.case_retries, labels=labels, - strict_target_tier=args.strict_target_tier, include_private_metadata=not args.public_records, ) ) @@ -457,7 +456,7 @@ def main() -> int: "taxonomy_path": str(args.taxonomy), "catalog_path": str(catalog_path), "selected_tiers": selected_tiers, - "strict_target_tier": args.strict_target_tier, + "strict_target_tier": True, "public_records": args.public_records, "max_failure_rate": args.max_failure_rate, } @@ -479,7 +478,7 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--concurrency", type=int, default=32) parser.add_argument("--timeout", type=float, default=240.0) parser.add_argument("--max-retries", type=int, default=3) - parser.add_argument("--case-retries", type=int, default=2, help="Retry malformed teacher JSON/case outputs.") + parser.add_argument("--case-retries", type=int, default=2, help="Regenerate invalid cases or mismatched teacher labels.") parser.add_argument("--max-tokens", type=int, default=1536) parser.add_argument("--temperature", type=float, default=0.75) parser.add_argument("--enable-thinking", action="store_true", help="Allow Qwen thinking mode. Default disables it.") @@ -505,7 +504,7 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "--strict-target-tier", action="store_true", - help="Reject and retry responses whose explicit teacher tier is missing, invalid, or differs from the target.", + help="Accepted for compatibility; target-tier validation is always enabled.", ) parser.add_argument( "--public-records", @@ -676,11 +675,9 @@ def generate_one( response_format_json: bool, case_retries: int, labels: list[str], - strict_target_tier: bool, include_private_metadata: bool, ) -> dict[str, Any]: archetype = target["archetype"] - rng = random.Random(target["seed"]) domain = DOMAINS[archetype["domain"]] generation_messages = [ {"role": "system", "content": GEN_SYSTEM_PROMPT}, @@ -711,8 +708,9 @@ def generate_one( retry_instruction = { "role": "user", "content": ( - "The previous response was malformed or incomplete. " - "Return one complete JSON object only, with all required string fields and label." + "The previous response was invalid or did not match the target tier. " + "Generate a new trace that genuinely matches the requested tier, then label it. " + "Do not change only the label. Return one complete JSON object with all required fields." ), } attempt_messages = generation_messages + [retry_instruction] @@ -733,8 +731,6 @@ def generate_one( parse_json_object(raw_text), target_tier=target["target_tier"], labels=labels, - rng=rng, - strict_target_tier=strict_target_tier, ) break except Exception as exc: # noqa: BLE001 @@ -801,8 +797,6 @@ def normalize_case( *, target_tier: str, labels: list[str], - rng: random.Random, - strict_target_tier: bool = False, ) -> dict[str, Any]: case: dict[str, Any] = {} for field in CASE_FIELDS: @@ -817,8 +811,6 @@ def normalize_case( label_raw, target_tier=target_tier, labels=labels, - rng=rng, - strict_target_tier=strict_target_tier, ) case["label"] = label return case @@ -829,18 +821,12 @@ def normalize_label( *, target_tier: str, labels: list[str], - rng: random.Random, - strict_target_tier: bool = False, ) -> dict[str, Any]: - raw_tier = str(raw.get("tier") or "").strip().lower() - if strict_target_tier: - if raw_tier not in labels: - raise ValueError(f"teacher tier missing or invalid: {raw_tier!r}") - if raw_tier != target_tier: - raise ValueError(f"teacher tier mismatch: target={target_tier} teacher={raw_tier}") - tier = raw_tier or target_tier + tier = str(raw.get("tier") or "").strip().lower() if tier not in labels: - tier = target_tier + raise ValueError(f"teacher tier missing or invalid: {tier!r}") + if tier != target_tier: + raise ValueError(f"teacher tier mismatch: target={target_tier} teacher={tier}") default = deterministic_label(tier) verdict = str(raw.get("verdict") or default["verdict"]).strip().upper() if verdict not in {"PASS", "BLOCK", "ESCALATE"}: diff --git a/tests/test_label_integrity.py b/tests/test_label_integrity.py new file mode 100644 index 0000000..389e35d --- /dev/null +++ b/tests/test_label_integrity.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from scripts import evaluate_drift_sentry_model as evaluator +from scripts import generate_semantic_synthetic_6tier as generator + + +class LabelIntegrityTest(unittest.TestCase): + def test_generation_retries_then_accepts_or_excludes(self) -> None: + target = { + "sample_index": 0, + "split": "train", + "target_tier": "goal", + "seed": 42, + "archetype": generator.build_archetype_catalog(["goal"])[0], + } + for tiers in (("goal",), ("reward", "goal"), ("reward",) * 3, (None,) * 3, ("invalid",) * 3): + replies = [ + json.dumps( + { + **{field: f"example {index}: {field}" for field in generator.CASE_FIELDS}, + "label": {} if tier is None else {"tier": tier}, + } + ) + for index, tier in enumerate(tiers) + ] + with self.subTest(tiers=tiers), patch.object(generator.time, "sleep"): + with patch.object(generator, "call_teacher", side_effect=replies) as teacher: + result = generator.generate_one( + target=target, + taxonomy_brief="test taxonomy", + base_url="http://localhost/v1", + model="test", + api_key="test", + timeout=1, + max_retries=0, + max_tokens=512, + temperature=0, + enable_thinking=False, + response_format_json=True, + case_retries=2, + labels=evaluator.TIERS, + include_private_metadata=True, + ) + self.assertEqual(teacher.call_count, len(tiers)) + self.assertEqual(result["ok"], tiers[-1] == "goal") + if result["ok"]: + record = result["record"] + self.assertEqual(record["metadata"]["raw_teacher_text"], replies[-1]) + self.assertEqual(json.loads(record["messages"][-1]["content"])["tier"], "goal") + else: + self.assertNotIn("record", result) + expected = "teacher_tier_mismatch" if tiers[-1] == "reward" else "teacher_tier_missing_or_invalid" + self.assertEqual(result["failure_reason"], expected) + + def test_evaluation_requires_assistant_gold(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "test.jsonl" + for role, label in ( + ("assistant", {"tier": "normal"}), + ("assistant", {}), + ("assistant", {"tier": "unknown"}), + ("user", {"tier": "normal"}), + ): + with self.subTest(role=role, label=label): + path.write_text( + json.dumps( + { + "id": "case", + "metadata": {"target_tier": "reward"}, + "messages": [{"role": role, "content": json.dumps(label)}], + } + ) + + "\n", + encoding="utf-8", + ) + if role == "assistant" and label.get("tier") == "normal": + self.assertEqual(evaluator.load_rows(path, 0)[0]["gold_tier"], "normal") + else: + with self.assertRaises(ValueError): + evaluator.load_rows(path, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_public_benchmark_sanitizer.py b/tests/test_public_benchmark_sanitizer.py index ec7b2e0..913f154 100644 --- a/tests/test_public_benchmark_sanitizer.py +++ b/tests/test_public_benchmark_sanitizer.py @@ -43,6 +43,19 @@ def test_finalizer_rejects_hyphen_secret_token(self) -> None: self.assertEqual(FINALIZER.find_pii("sk-" + "abcdefghijklmnop"), "secret_token") self.assertEqual(FINALIZER.find_pii("sk-" + "abcdefghijklmno-"), "secret_token") + def test_finalizer_requires_matching_teacher_tier(self) -> None: + for teacher_fields in ({"teacher_tier": "reward"}, {}, {"teacher_tier": ""}, {"teacher_tier": "goal"}): + with self.subTest(teacher_fields=teacher_fields): + row = { + "messages": [{"role": "assistant", "content": '{"tier": "reward"}'}], + "metadata": {"target_tier": "reward", **teacher_fields}, + } + if teacher_fields.get("teacher_tier") == "reward": + self.assertEqual(FINALIZER.strict_tier(row), "reward") + else: + with self.assertRaisesRegex(ValueError, "tier mismatch"): + FINALIZER.strict_tier(row) + def test_transform_normalizes_public_provenance(self) -> None: row = { "messages": [{"role": "user", "content": "safe text"}],