From 7fdf316ccca87263f2a19db1b270665a48665193 Mon Sep 17 00:00:00 2001 From: boulea7 Date: Thu, 13 Aug 2026 12:34:05 +0800 Subject: [PATCH] fix(sleep): add per-task regression gate --- docs/sleep/README.md | 9 + plugins/README.md | 9 +- .../skills/skillopt-sleep/SKILL.md | 1 + plugins/codex/README.md | 2 +- plugins/codex/skills/skillopt-sleep/SKILL.md | 1 + .../copilot/copilot-instructions.snippet.md | 5 +- plugins/cursor/skills/skillopt-sleep/SKILL.md | 4 + skillopt_sleep/__main__.py | 8 +- skillopt_sleep/config.py | 1 + skillopt_sleep/consolidate.py | 148 ++++- skillopt_sleep/cycle.py | 112 +++- skillopt_sleep/dream.py | 4 +- skillopt_sleep/evidence.py | 11 +- skillopt_sleep/staging.py | 22 +- skillopt_sleep/types.py | 5 + tests/test_gate_no_regression.py | 589 ++++++++++++++++++ 16 files changed, 880 insertions(+), 51 deletions(-) create mode 100644 tests/test_gate_no_regression.py diff --git a/docs/sleep/README.md b/docs/sleep/README.md index 5b29ebcf..81352c18 100644 --- a/docs/sleep/README.md +++ b/docs/sleep/README.md @@ -26,6 +26,15 @@ It synthesizes **SkillOpt** (validation-gated bounded text edits), **Claude Drea (offline consolidation; review-then-adopt), and the **agent-sleep** idea (short-term experience → long-term competence). +The optional `gate_no_regression` config key strengthens the aggregate gate. +It defaults to `false` for compatibility; when set to `true`, every validation +task must preserve or improve its score under the configured `gate_metric`. +The check applies to intermediate skill and memory candidates and to the fresh +final replay. A missing task result or non-finite task score also blocks the +candidate; an absent numeric score aborts evaluation. Task-level changes are +included in `report.md`, `report.json`, `diagnostics.json`, the CLI's `--json` +output, and the evidence log. + > **Data boundary.** Harvesting is local and read-only. The `mock` backend makes no > provider calls. A real backend, however, sends truncated excerpts from harvested > sessions and derived tasks to the provider you select for mining, replay, judging, diff --git a/plugins/README.md b/plugins/README.md index 7391075d..99380723 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -231,10 +231,11 @@ python -m skillopt_sleep run --project "$(pwd)" \ ### Advanced config The JSON/YAML config under `~/.skillopt-sleep/` supports additional engine keys, -including `gate_mode`, `gate_metric`, `dream_rollouts`, `dream_factor`, `recall_k`, -`evolve_memory`, and `evolve_skill`. These are config keys, not aliases for the -unsupported CLI flags listed above. Shipping defaults are conservative: -`gate_mode="on"`, `dream_rollouts=1`, `dream_factor=0`, and `recall_k=0`. +including `gate_mode`, `gate_metric`, `gate_no_regression`, `dream_rollouts`, +`dream_factor`, `recall_k`, `evolve_memory`, and `evolve_skill`. These are config +keys, not aliases for the unsupported CLI flags listed above. Shipping defaults +are conservative: `gate_mode="on"`, `gate_no_regression=false`, +`dream_rollouts=1`, `dream_factor=0`, and `recall_k=0`. The managed `schedule` command stores only the project, backend, time, and optional auto-adopt setting. It does not copy `--source`, `--cursor-home`, diff --git a/plugins/claude-code/skills/skillopt-sleep/SKILL.md b/plugins/claude-code/skills/skillopt-sleep/SKILL.md index 22337baf..0c45aa8f 100644 --- a/plugins/claude-code/skills/skillopt-sleep/SKILL.md +++ b/plugins/claude-code/skills/skillopt-sleep/SKILL.md @@ -109,6 +109,7 @@ Beyond the CLI flags, advanced behavior is controlled via config: - **`preferences`** — free-text house rules injected into the optimizer's reflect step (e.g. "Always use async/await", "Answers in `\boxed{}`"). - **`gate_mode`** — `on` (default, validation-gated) or `off` (greedy, accept all edits). - **`gate_metric`** — `hard`, `soft`, or `mixed` (default). Controls how the held-out gate scores. +- **`gate_no_regression`** — `false` by default. Set to `true` to reject a candidate when any validation task's configured gate score decreases. - **`dream_rollouts`** — >1 enables multi-rollout contrastive reflection per task. - **`recall_k`** — >0 recalls K similar past tasks into the dream (long-term memory). - **`evolve_memory`** / **`evolve_skill`** — independently toggle CLAUDE.md vs SKILL.md consolidation. diff --git a/plugins/codex/README.md b/plugins/codex/README.md index 97807104..517bcf49 100644 --- a/plugins/codex/README.md +++ b/plugins/codex/README.md @@ -79,7 +79,7 @@ mining, replay, and reflection can be slow and otherwise quiet. Use prefer tasks that match the target skill's path, headings, and content. The implemented main-CLI flags work the same across the shared integrations, and `--preferences "..."` is available for house rules. Advanced keys such as -`gate_mode`, `dream_rollouts`, and `recall_k` belong in the Sleep config; the +`gate_mode`, `gate_no_regression`, `dream_rollouts`, and `recall_k` belong in the Sleep config; the nightly CLI does not expose `--gate`, `--rollouts-k`, token/time-budget, or optimizer/target-split flags. See the [shared CLI reference](../README.md#supported-cli-surface). diff --git a/plugins/codex/skills/skillopt-sleep/SKILL.md b/plugins/codex/skills/skillopt-sleep/SKILL.md index 6d6fd166..688ae377 100644 --- a/plugins/codex/skills/skillopt-sleep/SKILL.md +++ b/plugins/codex/skills/skillopt-sleep/SKILL.md @@ -129,6 +129,7 @@ without `crontab`, `schedule` prints a line for manual installation. - **`preferences`** — free-text house rules for the optimizer - **`gate_mode`** — `on` (validation-gated, default) or `off` (greedy) - **`gate_metric`** — `hard` | `soft` | `mixed` (default) +- **`gate_no_regression`** — `false` by default; set it to `true` to reject a candidate when any validation task's gate score decreases - **`dream_rollouts`** — >1 for multi-rollout contrastive reflection - **`recall_k`** — >0 recalls similar past tasks from the archive diff --git a/plugins/copilot/copilot-instructions.snippet.md b/plugins/copilot/copilot-instructions.snippet.md index 78c97e36..e22a20bf 100644 --- a/plugins/copilot/copilot-instructions.snippet.md +++ b/plugins/copilot/copilot-instructions.snippet.md @@ -38,7 +38,10 @@ my preferences", or "make the agent improve from past usage", use the MCP tools: ### Advanced config (`~/.skillopt-sleep/config.json`) - `preferences` — free-text house rules for the optimizer -- `gate_mode` — `on` (default) or `off`; `dream_rollouts` — >1 for more signal +- `gate_mode` — `on` (default) or `off` +- `gate_no_regression` — `false` by default; set it to `true` to reject a + candidate when any validation task's configured gate score decreases +- `dream_rollouts` — >1 for more signal - `evolve_memory` / `evolve_skill` — toggle which docs consolidate Always show the user the held-out baseline → candidate score and the proposed diff --git a/plugins/cursor/skills/skillopt-sleep/SKILL.md b/plugins/cursor/skills/skillopt-sleep/SKILL.md index 7caa8fc3..fc66d4db 100644 --- a/plugins/cursor/skills/skillopt-sleep/SKILL.md +++ b/plugins/cursor/skills/skillopt-sleep/SKILL.md @@ -166,6 +166,10 @@ Real backends reject task files that remain unreviewed. Never include raw transcripts, credentials, secrets, or sensitive task content in messages, commits, or generated summaries. +`gate_no_regression` is a config-only safeguard in +`~/.skillopt-sleep/config.json`. It defaults to `false`; set it to `true` to +reject a candidate when any validation task's configured gate score decreases. + ## Scheduling Scheduling is opt-in. The scheduler persists project, backend, time, and the diff --git a/skillopt_sleep/__main__.py b/skillopt_sleep/__main__.py index 416922c3..f0934953 100644 --- a/skillopt_sleep/__main__.py +++ b/skillopt_sleep/__main__.py @@ -36,7 +36,7 @@ from skillopt_sleep.harvest_sources import harvest_for_config from skillopt_sleep.mine import mine from skillopt_sleep.staging import adopt as adopt_staging -from skillopt_sleep.staging import latest_staging +from skillopt_sleep.staging import json_safe, latest_staging from skillopt_sleep.state import SleepState from skillopt_sleep.tasks_file import load_tasks_file, make_tasks_payload, write_tasks_file @@ -50,7 +50,7 @@ def _read_text(path: str) -> str: def _report_payload(rep, outcome) -> Dict[str, Any]: - return { + return json_safe({ "night": rep.night, "accepted": rep.accepted, "gate_action": rep.gate_action, @@ -63,10 +63,12 @@ def _report_payload(rep, outcome) -> Dict[str, Any]: "n_rejected_edits": len(rep.rejected_edits), "edits": [e.__dict__ for e in rep.edits], "rejected_edits": [e.__dict__ for e in rep.rejected_edits], + "gate_no_regression": bool(getattr(rep, "gate_no_regression", False)), + "gate_trials": _redact_deep(getattr(rep, "gate_trials", [])), "notes": rep.notes, "staging_dir": outcome.staging_dir, "adopted": outcome.adopted, - } + }) def _add_common(p: argparse.ArgumentParser) -> None: diff --git a/skillopt_sleep/config.py b/skillopt_sleep/config.py index 8d5919f8..a9bd6ea3 100644 --- a/skillopt_sleep/config.py +++ b/skillopt_sleep/config.py @@ -63,6 +63,7 @@ "preferences": "", # free-text house rules injected into reflect as a prior "gate_metric": "mixed", # hard | soft | mixed (mixed best for tiny holdouts) "gate_mixed_weight": 0.5, + "gate_no_regression": False, # reject any candidate that lowers a val-task score "replay_mode": "mock", # report label; fresh-worktree replay is not implemented # ── dream + recall (opt-in; defaults reproduce the prior single-shot loop) ─ "dream_rollouts": 1, # >1 => multi-rollout contrastive reflection per task diff --git a/skillopt_sleep/consolidate.py b/skillopt_sleep/consolidate.py index 754c492c..5b80b4a9 100644 --- a/skillopt_sleep/consolidate.py +++ b/skillopt_sleep/consolidate.py @@ -8,22 +8,27 @@ """ from __future__ import annotations -import os +import math from dataclasses import dataclass, field -from typing import List, Optional, Tuple +from typing import List, Tuple from skillopt_sleep.backend import Backend -from skillopt_sleep.memory import apply_edits_detailed -from skillopt_sleep.replay import aggregate_scores, replay_batch -from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord - # Self-contained validation gate (vendored from SkillOpt; zero dependency on the # research package, so this open-source tool stays decoupled from the paper code). from skillopt_sleep.gate import evaluate_gate, select_gate_score +from skillopt_sleep.memory import apply_edits_detailed +from skillopt_sleep.replay import aggregate_scores, replay_batch +from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord + _HAVE_REPO_GATE = True +def _finite_score(value: float) -> float | None: + """Return a JSON-safe score, preserving finite values only.""" + return value if math.isfinite(value) else None + + @dataclass class ConsolidationResult: accepted: bool @@ -49,6 +54,11 @@ class ConsolidationResult: # saw, so its comparison cannot detect overfitting. A night in this state # stages edits but never certifies them. holdout_leaked: bool = False + # Each gated candidate carries the task-level comparison that produced its + # decision. This includes rejected intermediate skill/memory trials and the + # fresh final replay, so aggregate changes never hide an individual task. + # Keep new optional fields last to preserve positional construction. + gate_trials: List[dict] = field(default_factory=list) def _split(tasks: List[TaskRecord]) -> Tuple[List[TaskRecord], List[TaskRecord], bool]: @@ -108,6 +118,65 @@ def _holdout_detail(pairs: List[Tuple[TaskRecord, ReplayResult]]) -> List[dict]: return out +def _task_deltas( + tasks: List[TaskRecord], + baseline_pairs: List[Tuple[TaskRecord, ReplayResult]], + candidate_pairs: List[Tuple[TaskRecord, ReplayResult]], + metric: str, + mixed_weight: float, +) -> List[dict]: + """Compare candidate scores with the matching baseline validation tasks.""" + baseline_by_id = {task.id: result for task, result in baseline_pairs} + candidate_by_id = {task.id: result for task, result in candidate_pairs} + out: List[dict] = [] + for task in tasks: + baseline = baseline_by_id.get(task.id) + candidate = candidate_by_id.get(task.id) + baseline_score = ( + select_gate_score(baseline.hard, baseline.soft, metric, mixed_weight) + if baseline is not None + else None + ) + candidate_score = ( + select_gate_score(candidate.hard, candidate.soft, metric, mixed_weight) + if candidate is not None + else None + ) + if baseline_score is None or candidate_score is None: + out.append({ + "task_id": task.id, + "tags": list(task.tags or []), + "baseline_score": baseline_score, + "candidate_score": candidate_score, + "status": "regressed", + "scores_are_finite": False, + }) + continue + scores_are_finite = math.isfinite(baseline_score) and math.isfinite( + candidate_score + ) + if not scores_are_finite: + # The aggregate gate already rejects NaN, but +inf could otherwise + # look like an improvement. Strict mode treats any malformed task + # comparison as a regression and therefore fails closed. + status = "regressed" + elif candidate_score > baseline_score: + status = "improved" + elif candidate_score < baseline_score: + status = "regressed" + else: + status = "unchanged" + out.append({ + "task_id": task.id, + "tags": list(task.tags or []), + "baseline_score": _finite_score(baseline_score), + "candidate_score": _finite_score(candidate_score), + "status": status, + "scores_are_finite": scores_are_finite, + }) + return out + + def consolidate( backend: Backend, tasks: List[TaskRecord], @@ -117,6 +186,7 @@ def consolidate( edit_budget: int = 4, gate_metric: str = "mixed", gate_mixed_weight: float = 0.5, + gate_no_regression: bool = False, gate_mode: str = "on", # "on" (hard/soft per gate_metric) | "off" (greedy) rollouts_k: int = 1, # >1 => multi-rollout contrastive reflection evolve_skill: bool = True, @@ -145,6 +215,7 @@ def consolidate( # both wasted cost and contrary to the "no val set required" design. if gate_off: base_hard, base_soft = 0.0, 0.0 + base_pairs: List[Tuple[TaskRecord, ReplayResult]] = [] else: evlog.set_phase(backend, "baseline_val") base_pairs = replay_batch(backend, val_tasks, skill, memory) @@ -167,13 +238,16 @@ def consolidate( all_applied: List[EditRecord] = [] all_rejected: List[EditRecord] = [] all_unmatched: List[EditRecord] = [] + gate_trials: List[dict] = [] + current_pairs = base_pairs def _edits_payload(edits: List[EditRecord]) -> List[dict]: return [{"op": e.op, "content": e.content, "anchor": e.anchor, "rationale": e.rationale} for e in edits] def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str: - nonlocal cand_skill, cand_memory, base_score, all_applied, all_rejected + nonlocal cand_skill, cand_memory, base_score, current_pairs + nonlocal all_applied, all_rejected if ev is not None: ev.log("reflect", "edits_returned", target=which, n_edits=len(edits), edits=_edits_payload(edits)) @@ -201,14 +275,33 @@ def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str: pairs = replay_batch(backend, val_tasks, trial_skill, trial_memory) h, s = aggregate_scores(pairs) cand_score = select_gate_score(h, s, gate_metric, gate_mixed_weight) - improved = cand_score > base_score + task_deltas = _task_deltas( + val_tasks, current_pairs, pairs, gate_metric, gate_mixed_weight + ) + blocked_by_regression = bool( + gate_no_regression + and any(row["status"] == "regressed" for row in task_deltas) + ) + trial_base_score = base_score + improved = cand_score > base_score and not blocked_by_regression + gate_trials.append({ + "target": which, + "baseline_score": _finite_score(trial_base_score), + "candidate_score": _finite_score(cand_score), + "accepted": improved, + "blocked_by_regression": blocked_by_regression, + "task_deltas": task_deltas, + }) if ev is not None: ev.log("gate", "trial", target=which, mode="gated", - baseline_score=base_score, cand_hard=h, cand_soft=s, + baseline_score=trial_base_score, cand_hard=h, cand_soft=s, cand_score=cand_score, accepted=improved, + blocked_by_regression=blocked_by_regression, + task_deltas=task_deltas, n_edits=len(applied)) if improved: - base_score = max(base_score, cand_score) + base_score = cand_score + current_pairs = pairs all_applied.extend(applied) return new_doc all_rejected.extend(applied) @@ -218,12 +311,13 @@ def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str: if rollouts_k > 1: # multi-rollout contrastive reflection: run each train task K times # and distill a rule from the good-vs-bad contrast (the imagination signal). - from skillopt_sleep.rollout import multi_rollout, contrastive_reflect # Parallelize across tasks (each multi_rollout also parallelizes its K # attempts). This dream phase is the dominant cost; serial execution # times out on real backends. Cap total in-flight at the worker env. import os from concurrent.futures import ThreadPoolExecutor + + from skillopt_sleep.rollout import contrastive_reflect, multi_rollout try: _w = int(os.environ.get("SKILLOPT_SLEEP_WORKERS", "1")) except ValueError: @@ -289,6 +383,13 @@ def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str: final_hard, final_soft = aggregate_scores(final_pairs) final_score = select_gate_score(final_hard, final_soft, gate_metric, gate_mixed_weight) base_gate_score = select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight) + final_task_deltas = _task_deltas( + val_tasks, base_pairs, final_pairs, gate_metric, gate_mixed_weight + ) + final_blocked_by_regression = bool( + gate_no_regression + and any(row["status"] == "regressed" for row in final_task_deltas) + ) if _HAVE_REPO_GATE: gate = evaluate_gate( candidate_skill=cand_skill, @@ -304,10 +405,18 @@ def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str: mixed_weight=gate_mixed_weight, ) action = gate.action - accepted = bool(all_applied) and final_score > base_gate_score + accepted = ( + bool(all_applied) + and final_score > base_gate_score + and not final_blocked_by_regression + ) else: action = "accept" if final_score > base_gate_score else "reject" - accepted = bool(all_applied) and final_score > base_gate_score + accepted = ( + bool(all_applied) + and final_score > base_gate_score + and not final_blocked_by_regression + ) # The gate scores documents, not edit bookkeeping: when every proposed # edit was dropped during the per-target trials, `all_applied` is empty # and nothing changed, yet the score comparison can still yield an @@ -332,6 +441,14 @@ def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str: if edit not in all_rejected: all_rejected.append(edit) all_applied = [] + gate_trials.append({ + "target": "final", + "baseline_score": _finite_score(base_gate_score), + "candidate_score": _finite_score(final_score), + "accepted": accepted, + "blocked_by_regression": final_blocked_by_regression, + "task_deltas": final_task_deltas, + }) if ev is not None: w = max(0.0, min(1.0, float(gate_mixed_weight))) @@ -351,6 +468,10 @@ def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str: baseline_hard=base_hard, baseline_soft=base_soft, candidate_hard=final_hard, candidate_soft=final_soft, metric=gate_metric, mixed_weight=gate_mixed_weight, + blocked_by_regression=( + final_blocked_by_regression if not gate_off else False + ), + task_deltas=(final_task_deltas if not gate_off else []), formula=formula, n_applied=len(all_applied), n_rejected=len(all_rejected), n_unmatched=len(all_unmatched), night=night) @@ -368,6 +489,7 @@ def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str: holdout_baseline=base_hard, holdout_candidate=final_hard, holdout_detail=holdout_detail, + gate_trials=gate_trials, reflect_raw=getattr(backend, "last_reflect_raw", "") or "", call_error=getattr(backend, "last_call_error", "") or "", holdout_leaked=holdout_leaked, diff --git a/skillopt_sleep/cycle.py b/skillopt_sleep/cycle.py index afa9c101..5d5654a5 100644 --- a/skillopt_sleep/cycle.py +++ b/skillopt_sleep/cycle.py @@ -9,6 +9,7 @@ """ from __future__ import annotations +import math import os import shutil import sys @@ -17,9 +18,9 @@ from skillopt_sleep import evidence from skillopt_sleep.backend import Backend, CursorBackendError, build_backend -from skillopt_sleep.evidence import EvidenceLog from skillopt_sleep.config import SleepConfig, load_config from skillopt_sleep.dream import dream_consolidate +from skillopt_sleep.evidence import EvidenceLog from skillopt_sleep.harvest_sources import harvest_for_config from skillopt_sleep.memory import ensure_skill_scaffold from skillopt_sleep.mine import group_tasks_by_skill_hint, mine @@ -29,8 +30,7 @@ skill_group_reports, ) from skillopt_sleep.staging import adopt as adopt_staging -from skillopt_sleep.staging import redact_secrets -from skillopt_sleep.staging import write_staging +from skillopt_sleep.staging import json_safe, redact_secrets, write_staging from skillopt_sleep.state import SleepState, _now_iso from skillopt_sleep.types import SessionDigest, SleepReport, TaskRecord @@ -177,6 +177,17 @@ def _markdown_table_text(value: object) -> str: ) +def _report_score(value: object) -> str: + """Render an optional numeric score without breaking the report.""" + if value is None: + return "—" + try: + score = float(value) + except (TypeError, ValueError): + return "—" + return f"{score:.3f}" if math.isfinite(score) else "—" + + def _render_report_md(report: SleepReport, cfg: SleepConfig) -> str: lines = [ f"# SkillOpt-Sleep — night {report.night} report", @@ -185,8 +196,11 @@ def _render_report_md(report: SleepReport, cfg: SleepConfig) -> str: f"- backend: `{cfg.get('backend')}` replay: `{cfg.get('replay_mode')}`", f"- sessions harvested: {report.n_sessions}", f"- tasks mined: {report.n_tasks} (replayed: {report.n_replayed})", - f"- held-out score: {report.baseline_score:.3f} -> {report.candidate_score:.3f}", + f"- held-out score: {_report_score(report.baseline_score)} " + f"-> {_report_score(report.candidate_score)}", f"- gate: **{report.gate_action}** (accepted={report.accepted})", + f"- no-regression gate: " + f"{'enabled' if cfg.get('gate_no_regression', False) else 'disabled'}", f"- tokens used: {report.tokens_used}", "", ] @@ -203,6 +217,42 @@ def _render_report_md(report: SleepReport, cfg: SleepConfig) -> str: "suggestions, not rejections.", "", ] + if report.gate_trials: + lines.append("## Held-out task changes") + lines.append( + "_These are observed score changes under the configured gate metric; " + "they do not establish that a particular edit caused the change._" + ) + lines.append("") + for trial in report.gate_trials: + target = _markdown_table_text(trial.get("target", "candidate")) + accepted = bool(trial.get("accepted", False)) + blocked = bool(trial.get("blocked_by_regression", False)) + decision = "accepted" if accepted else "rejected" + if blocked: + decision += " (task regression)" + baseline = _report_score(trial.get("baseline_score")) + candidate = _report_score(trial.get("candidate_score")) + lines.append( + f"### `{target}` candidate — {decision} " + f"({baseline} → {candidate})" + ) + lines.append("") + lines.append("| Task | Tags | Baseline | Candidate | Change |") + lines.append("|---|---|---:|---:|---|") + for delta in trial.get("task_deltas", []): + task_id = _markdown_table_text(delta.get("task_id", "")) + tags = _markdown_table_text( + ", ".join(str(tag) for tag in delta.get("tags", [])) or "—" + ) + baseline_score = _report_score(delta.get("baseline_score")) + candidate_score = _report_score(delta.get("candidate_score")) + status = _markdown_table_text(delta.get("status", "unchanged")) + lines.append( + f"| `{task_id}` | {tags} | {baseline_score} " + f"| {candidate_score} | {status} |" + ) + lines.append("") if report.edits: lines.append("## Accepted edits") for e in report.edits: @@ -345,7 +395,8 @@ def run_sleep_cycle( config={k: cfg.get(k) for k in ( "backend", "model", "optimizer_backend", "optimizer_model", "target_backend", "target_model", "gate_mode", "gate_metric", - "gate_mixed_weight", "edit_budget", "holdout_fraction", + "gate_mixed_weight", "gate_no_regression", "edit_budget", + "holdout_fraction", "dream_rollouts", "dream_factor", "recall_k", "max_tasks_per_night", "lookback_hours", "llm_mine", "evolve_skill", "evolve_memory")}) @@ -461,6 +512,7 @@ def run_sleep_cycle( report = SleepReport( night=night, project=project, started_at=started, n_sessions=n_sessions, n_tasks=len(tasks), + gate_no_regression=bool(cfg.get("gate_no_regression", False)), ) if not tasks: @@ -496,6 +548,7 @@ def run_sleep_cycle( edit_budget=cfg.get("edit_budget", 4), gate_metric=cfg.get("gate_metric", "mixed"), gate_mixed_weight=cfg.get("gate_mixed_weight", 0.5), + gate_no_regression=cfg.get("gate_no_regression", False), gate_mode=cfg.get("gate_mode", "on"), evolve_skill=cfg.get("evolve_skill", True), evolve_memory=cfg.get("evolve_memory", True), @@ -523,6 +576,7 @@ def run_sleep_cycle( report.edits = result.applied_edits report.rejected_edits = result.rejected_edits report.unmatched_edits = result.unmatched_edits + report.gate_trials = redact_secrets(getattr(result, "gate_trials", [])) # ── 4b. optional per-skill group reporting ─────────────────────────── # Off by default. When enabled, tonight's tasks are grouped by their skill @@ -549,6 +603,7 @@ def run_sleep_cycle( edit_budget=cfg.get("edit_budget", 4), gate_metric=cfg.get("gate_metric", "mixed"), gate_mixed_weight=cfg.get("gate_mixed_weight", 0.5), + gate_no_regression=cfg.get("gate_no_regression", False), gate_mode=cfg.get("gate_mode", "on"), night=night, ) @@ -598,25 +653,34 @@ def run_sleep_cycle( # credentials (e.g. a codex 401 stderr dump), so scrub secret-looking # substrings before persisting them to the on-disk diagnostics. with open(os.path.join(staging_dir, "diagnostics.json"), "w", encoding="utf-8") as _fh: - _json.dump({ - "night": night, - "backend": cfg.get("backend"), - "gate_mode": cfg.get("gate_mode"), - "n_tasks": len(tasks), - "baseline_score": result.baseline_score, - "candidate_score": result.candidate_score, - "accepted": result.accepted, - "gate_action": result.gate_action, - "holdout_leaked": getattr(result, "holdout_leaked", False), - "n_applied_edits": len(result.applied_edits), - "n_rejected_edits": len(result.rejected_edits), - "n_unmatched_edits": len(result.unmatched_edits), - "call_error": redact_secrets(getattr(result, "call_error", "")), - "reflect_raw_head": redact_secrets( - (getattr(result, "reflect_raw", "") or "")[:1200] - ), - "holdout_detail": redact_secrets(getattr(result, "holdout_detail", [])), - }, _fh, indent=2) + _json.dump( + json_safe({ + "night": night, + "backend": cfg.get("backend"), + "gate_mode": cfg.get("gate_mode"), + "gate_no_regression": cfg.get("gate_no_regression", False), + "n_tasks": len(tasks), + "baseline_score": result.baseline_score, + "candidate_score": result.candidate_score, + "accepted": result.accepted, + "gate_action": result.gate_action, + "holdout_leaked": getattr(result, "holdout_leaked", False), + "n_applied_edits": len(result.applied_edits), + "n_rejected_edits": len(result.rejected_edits), + "n_unmatched_edits": len(result.unmatched_edits), + "call_error": redact_secrets(getattr(result, "call_error", "")), + "reflect_raw_head": redact_secrets( + (getattr(result, "reflect_raw", "") or "")[:1200] + ), + "holdout_detail": redact_secrets( + getattr(result, "holdout_detail", []) + ), + "gate_trials": report.gate_trials, + }), + _fh, + indent=2, + allow_nan=False, + ) except Exception: pass state.set_last_harvest(project, started) diff --git a/skillopt_sleep/dream.py b/skillopt_sleep/dream.py index 7223874c..245f7e89 100644 --- a/skillopt_sleep/dream.py +++ b/skillopt_sleep/dream.py @@ -110,6 +110,7 @@ def dream_consolidate( edit_budget: int = 4, gate_metric: str = "mixed", gate_mixed_weight: float = 0.5, + gate_no_regression: bool = False, gate_mode: str = "on", evolve_skill: bool = True, evolve_memory: bool = True, @@ -133,7 +134,8 @@ def dream_consolidate( return consolidate( backend, enlarged, skill, memory, edit_budget=edit_budget, gate_metric=gate_metric, - gate_mixed_weight=gate_mixed_weight, gate_mode=gate_mode, + gate_mixed_weight=gate_mixed_weight, + gate_no_regression=gate_no_regression, gate_mode=gate_mode, rollouts_k=dream_rollouts, evolve_skill=evolve_skill, evolve_memory=evolve_memory, night=night, ) diff --git a/skillopt_sleep/evidence.py b/skillopt_sleep/evidence.py index fa53891d..14dd367e 100644 --- a/skillopt_sleep/evidence.py +++ b/skillopt_sleep/evidence.py @@ -33,7 +33,7 @@ import time from typing import Any, Optional -from skillopt_sleep.staging import redact_secrets +from skillopt_sleep.staging import json_safe, redact_secrets def _now_iso() -> str: @@ -72,12 +72,17 @@ def _clean(self, value: Any) -> Any: # ── the one write path ──────────────────────────────────────────────── def log(self, stage: str, event: str, **data: Any) -> None: record = {"ts": _now_iso(), "stage": stage, "event": event} - record.update(self._clean(data)) + record.update(self._clean(json_safe(data))) with self._lock: self._seq += 1 record["seq"] = self._seq try: - line = json.dumps(record, ensure_ascii=False, default=str) + line = json.dumps( + record, + ensure_ascii=False, + default=str, + allow_nan=False, + ) with open(self.path, "a", encoding="utf-8") as f: f.write(line + "\n") except Exception: diff --git a/skillopt_sleep/staging.py b/skillopt_sleep/staging.py index fb6f8863..2bd757b2 100644 --- a/skillopt_sleep/staging.py +++ b/skillopt_sleep/staging.py @@ -8,6 +8,7 @@ from __future__ import annotations import json +import math import os import re import shutil @@ -236,6 +237,19 @@ def redact_secrets(value: Any) -> Any: return value +def json_safe(value: Any) -> Any: + """Replace non-finite floats recursively so persisted JSON stays standard.""" + if isinstance(value, float) and not math.isfinite(value): + return None + if isinstance(value, list): + return [json_safe(item) for item in value] + if isinstance(value, tuple): + return [json_safe(item) for item in value] + if isinstance(value, dict): + return {key: json_safe(item) for key, item in value.items()} + return value + + class StagingError(ValueError): """A proposal could not be staged safely (bad name, bad target, collision).""" @@ -487,7 +501,13 @@ def write_staging( with open(os.path.join(out, "proposed_CLAUDE.md"), "w", encoding="utf-8") as f: f.write(proposed_memory) with open(os.path.join(out, "report.json"), "w", encoding="utf-8") as f: - json.dump(report.to_dict(), f, ensure_ascii=False, indent=2) + json.dump( + json_safe(report.to_dict()), + f, + ensure_ascii=False, + indent=2, + allow_nan=False, + ) with open(os.path.join(out, "report.md"), "w", encoding="utf-8") as f: f.write(report_md) with open(os.path.join(out, "manifest.json"), "w", encoding="utf-8") as f: diff --git a/skillopt_sleep/types.py b/skillopt_sleep/types.py index c7ee7175..b40578a6 100644 --- a/skillopt_sleep/types.py +++ b/skillopt_sleep/types.py @@ -184,6 +184,11 @@ class SleepReport: skill_groups: List[SkillGroupReport] = field(default_factory=list) tokens_used: int = 0 notes: List[str] = field(default_factory=list) + # Keep new optional fields last to preserve positional construction. + gate_no_regression: bool = False + # Per-candidate held-out comparisons, including rejected intermediate + # skill/memory trials and the final replay. + gate_trials: List[Dict[str, Any]] = field(default_factory=list) def to_dict(self) -> Dict[str, Any]: d = asdict(self) diff --git a/tests/test_gate_no_regression.py b/tests/test_gate_no_regression.py new file mode 100644 index 00000000..9648a6f4 --- /dev/null +++ b/tests/test_gate_no_regression.py @@ -0,0 +1,589 @@ +"""Regression-sensitive validation gate coverage for issue #174.""" + +from __future__ import annotations + +import importlib +import json +import os + +from skillopt_sleep.__main__ import _report_payload +from skillopt_sleep.backend import Backend, MockBackend +from skillopt_sleep.config import DEFAULTS, load_config +from skillopt_sleep.cycle import _render_report_md, run_sleep_cycle +from skillopt_sleep.evidence import EvidenceLog +from skillopt_sleep.experiments.personas import researcher_persona +from skillopt_sleep.memory import set_learned +from skillopt_sleep.mine import assign_splits +from skillopt_sleep.staging import write_staging +from skillopt_sleep.types import EditRecord, ReplayResult, SleepReport, TaskRecord + +EDIT = EditRecord( + target="skill", + op="add", + content="A candidate rule.", + rationale="improve the validation slice", +) + + +class EditingBackend(Backend): + name = "editing-stub" + + def reflect(self, *args, **kwargs): + return [EDIT] + + +def _tasks() -> list[TaskRecord]: + return [ + TaskRecord(id="train", project="test", intent="train", split="train"), + TaskRecord( + id="improved", + project="test", + intent="improve", + split="val", + tags=["essay"], + ), + TaskRecord( + id="regressed", + project="test", + intent="do not regress", + split="val", + tags=["router", "held-out"], + ), + ] + + +def _patch_scores(monkeypatch, batches: list[list[float | None]]) -> None: + consolidate_module = importlib.import_module("skillopt_sleep.consolidate") + scores = iter(batches) + + def fake_replay_batch(backend, tasks, skill, memory): + batch = next(scores) + assert len(batch) == len(tasks) + return [ + ( + task, + ReplayResult( + id=task.id, + hard=score, + soft=score, + response="candidate", + ), + ) + for task, score in zip(tasks, batch) + ] + + monkeypatch.setattr(consolidate_module, "replay_batch", fake_replay_batch) + + +def _patch_score_maps(monkeypatch, batches: list[dict[str, float]]) -> None: + consolidate_module = importlib.import_module("skillopt_sleep.consolidate") + scores = iter(batches) + + def fake_replay_batch(backend, tasks, skill, memory): + batch = next(scores) + return [ + ( + task, + ReplayResult( + id=task.id, + hard=batch[task.id], + soft=batch[task.id], + response="candidate", + ), + ) + for task in tasks + if task.id in batch + ] + + monkeypatch.setattr(consolidate_module, "replay_batch", fake_replay_batch) + + +def _consolidate(**kwargs): + consolidate_module = importlib.import_module("skillopt_sleep.consolidate") + return consolidate_module.consolidate( + EditingBackend(), + _tasks(), + set_learned("# Skill\n", []), + "", + gate_metric="hard", + evolve_memory=False, + **kwargs, + ) + + +def test_default_keeps_aggregate_only_gate_behavior(monkeypatch) -> None: + # Baseline mean 0.4 -> candidate mean 0.85, while one task falls 0.8 -> 0.7. + batches = [[0.0, 0.8], [0.0], [1.0, 0.7], [1.0, 0.7]] + _patch_scores(monkeypatch, batches) + + default_result = _consolidate() + + _patch_scores(monkeypatch, batches) + explicit_result = _consolidate(gate_no_regression=False) + + assert default_result.accepted is True + assert explicit_result.accepted is True + for field in ( + "gate_action", + "baseline_score", + "candidate_score", + "new_skill", + "new_memory", + "applied_edits", + "rejected_edits", + "holdout_baseline", + "holdout_candidate", + ): + assert getattr(default_result, field) == getattr(explicit_result, field) + + +def test_opt_in_gate_accepts_when_no_task_regresses(monkeypatch) -> None: + _patch_scores( + monkeypatch, + [[0.0, 0.8], [0.0], [1.0, 0.8], [1.0, 0.8]], + ) + + result = _consolidate(gate_no_regression=True) + + assert result.accepted is True + assert result.gate_action == "accept_new_best" + assert result.gate_trials[0]["accepted"] is True + assert result.gate_trials[0]["blocked_by_regression"] is False + assert [row["status"] for row in result.gate_trials[0]["task_deltas"]] == [ + "improved", + "unchanged", + ] + + +def test_opt_in_gate_rejects_regression_during_skill_trial(monkeypatch) -> None: + # The rejected trial is followed by a final replay of the unchanged skill. + _patch_scores( + monkeypatch, + [[0.0, 0.8], [0.0], [1.0, 0.7], [0.0, 0.8]], + ) + + result = _consolidate(gate_no_regression=True) + + assert result.accepted is False + assert result.applied_edits == [] + assert result.rejected_edits == [EDIT] + skill_trial = result.gate_trials[0] + assert skill_trial["target"] == "skill" + assert skill_trial["blocked_by_regression"] is True + assert skill_trial["accepted"] is False + assert skill_trial["task_deltas"] == [ + { + "task_id": "improved", + "tags": ["essay"], + "baseline_score": 0.0, + "candidate_score": 1.0, + "status": "improved", + "scores_are_finite": True, + }, + { + "task_id": "regressed", + "tags": ["router", "held-out"], + "baseline_score": 0.8, + "candidate_score": 0.7, + "status": "regressed", + "scores_are_finite": True, + }, + ] + + +def test_opt_in_gate_rechecks_regressions_on_final_replay(monkeypatch) -> None: + # The skill trial has no regression, but the fresh final replay does. + _patch_scores( + monkeypatch, + [[0.0, 0.8], [0.0], [1.0, 0.8], [1.0, 0.7]], + ) + + result = _consolidate(gate_no_regression=True) + + assert result.accepted is False + assert result.applied_edits == [] + assert result.rejected_edits == [EDIT] + final_trial = result.gate_trials[-1] + assert final_trial["target"] == "final" + assert final_trial["blocked_by_regression"] is True + assert final_trial["accepted"] is False + assert [row["status"] for row in final_trial["task_deltas"]] == [ + "improved", + "regressed", + ] + + +def test_opt_in_gate_rejects_missing_task_during_trial(monkeypatch) -> None: + _patch_score_maps( + monkeypatch, + [ + {"improved": 0.0, "regressed": 0.8}, + {"train": 0.0}, + {"improved": 1.0}, + {"improved": 0.0, "regressed": 0.8}, + ], + ) + + result = _consolidate(gate_no_regression=True) + + assert result.accepted is False + missing = result.gate_trials[0]["task_deltas"][1] + assert missing["task_id"] == "regressed" + assert missing["candidate_score"] is None + assert missing["scores_are_finite"] is False + assert missing["status"] == "regressed" + + +def test_opt_in_gate_rejects_missing_baseline_task(monkeypatch) -> None: + _patch_score_maps( + monkeypatch, + [ + {"improved": 0.0}, + {"train": 0.0}, + {"improved": 1.0, "regressed": 0.8}, + {"improved": 0.0, "regressed": 0.8}, + ], + ) + + result = _consolidate(gate_no_regression=True) + + assert result.accepted is False + missing = result.gate_trials[0]["task_deltas"][1] + assert missing["task_id"] == "regressed" + assert missing["baseline_score"] is None + assert missing["candidate_score"] == 0.8 + assert missing["status"] == "regressed" + + +def test_opt_in_gate_rejects_missing_task_during_final_replay(monkeypatch) -> None: + _patch_score_maps( + monkeypatch, + [ + {"improved": 0.0, "regressed": 0.8}, + {"train": 0.0}, + {"improved": 1.0, "regressed": 0.8}, + {"improved": 1.0}, + ], + ) + + result = _consolidate(gate_no_regression=True) + + assert result.accepted is False + final_trial = result.gate_trials[-1] + assert final_trial["blocked_by_regression"] is True + assert final_trial["task_deltas"][1]["candidate_score"] is None + + +def test_opt_in_gate_rejects_regression_during_memory_trial(monkeypatch) -> None: + _patch_scores( + monkeypatch, + [[0.0, 0.8], [0.0], [0.0], [1.0, 0.7], [0.0, 0.8]], + ) + consolidate_module = importlib.import_module("skillopt_sleep.consolidate") + + result = consolidate_module.consolidate( + EditingBackend(), + _tasks(), + "", + set_learned("# Memory\n", []), + gate_metric="hard", + gate_no_regression=True, + evolve_skill=False, + evolve_memory=True, + ) + + assert result.accepted is False + assert result.gate_trials[0]["target"] == "memory" + assert result.gate_trials[0]["blocked_by_regression"] is True + + +def test_opt_in_gate_rejects_non_finite_task_score(monkeypatch) -> None: + _patch_scores( + monkeypatch, + [[0.0, 0.8], [0.0], [float("inf"), 0.8], [0.0, 0.8]], + ) + + result = _consolidate(gate_no_regression=True) + + assert result.accepted is False + trial = result.gate_trials[0] + assert trial["blocked_by_regression"] is True + invalid = trial["task_deltas"][0] + assert invalid["candidate_score"] is None + assert invalid["scores_are_finite"] is False + assert invalid["status"] == "regressed" + json.dumps(result.gate_trials, allow_nan=False) + + +def test_opt_in_gate_rejects_nan_task_score(monkeypatch) -> None: + _patch_scores( + monkeypatch, + [[0.0, 0.8], [0.0], [float("nan"), 0.8], [0.0, 0.8]], + ) + + result = _consolidate(gate_no_regression=True) + + assert result.accepted is False + trial = result.gate_trials[0] + assert trial["blocked_by_regression"] is True + invalid = trial["task_deltas"][0] + assert invalid["candidate_score"] is None + assert invalid["scores_are_finite"] is False + assert invalid["status"] == "regressed" + json.dumps(result.gate_trials, allow_nan=False) + + +def test_opt_in_gate_rejects_nan_on_final_replay(monkeypatch) -> None: + _patch_scores( + monkeypatch, + [[0.0, 0.8], [0.0], [1.0, 0.8], [float("nan"), 0.8]], + ) + + result = _consolidate(gate_no_regression=True) + + assert result.accepted is False + final_trial = result.gate_trials[-1] + assert final_trial["target"] == "final" + assert final_trial["blocked_by_regression"] is True + assert final_trial["candidate_score"] is None + assert final_trial["task_deltas"][0]["candidate_score"] is None + json.dumps(result.gate_trials, allow_nan=False) + + +def test_missing_numeric_score_still_aborts_candidate_evaluation(monkeypatch) -> None: + batches_by_phase = ( + [[0.0, 0.8], [0.0], [None, 0.8]], + [[0.0, 0.8], [0.0], [1.0, 0.8], [None, 0.8]], + ) + + for batches in batches_by_phase: + _patch_scores(monkeypatch, batches) + try: + _consolidate(gate_no_regression=True) + except (TypeError, ValueError): + continue + raise AssertionError("a validation task without a numeric score was accepted") + + +def test_replay_exception_still_propagates(monkeypatch) -> None: + consolidate_module = importlib.import_module("skillopt_sleep.consolidate") + + def failed_replay(*args, **kwargs): + raise RuntimeError("validation replay failed") + + monkeypatch.setattr(consolidate_module, "replay_batch", failed_replay) + + try: + _consolidate(gate_no_regression=True) + except RuntimeError as exc: + assert str(exc) == "validation replay failed" + else: + raise AssertionError("validation replay failure was swallowed") + + +def test_no_regression_setting_defaults_off() -> None: + assert DEFAULTS["gate_no_regression"] is False + + +def test_report_surfaces_task_level_gate_changes() -> None: + report = SleepReport( + night=1, + project="/tmp/project", + gate_action="reject", + gate_no_regression=True, + gate_trials=[ + { + "target": "skill", + "baseline_score": 0.4, + "candidate_score": 0.85, + "accepted": False, + "blocked_by_regression": True, + "task_deltas": [ + { + "task_id": "essay|one", + "tags": ["writing"], + "baseline_score": 0.0, + "candidate_score": 1.0, + "status": "improved", + }, + { + "task_id": "router-two", + "tags": ["routing", "held-out"], + "baseline_score": 0.8, + "candidate_score": 0.7, + "status": "regressed", + }, + { + "task_id": "missing", + "tags": [], + "baseline_score": 0.6, + "candidate_score": None, + "status": "regressed", + }, + ], + } + ], + ) + + markdown = _render_report_md( + report, + load_config(gate_no_regression=True), + ) + + assert "no-regression gate: enabled" in markdown + assert "Held-out task changes" in markdown + assert "essay|one" in markdown + assert "routing, held-out" in markdown + assert "0.800" in markdown + assert "0.700" in markdown + assert "regressed" in markdown + assert "| `missing` | — | 0.600 | — | regressed |" in markdown + + +def test_cli_json_payload_surfaces_redacted_task_deltas() -> None: + synthetic_secret = "api_key=synthetic-example-123456" + report = SleepReport( + night=1, + project="/tmp/project", + gate_action="reject", + gate_no_regression=True, + gate_trials=[ + { + "target": "skill", + "baseline_score": 0.4, + "candidate_score": 0.85, + "accepted": False, + "blocked_by_regression": True, + "task_deltas": [ + { + "task_id": synthetic_secret, + "tags": [synthetic_secret], + "baseline_score": 0.8, + "candidate_score": 0.7, + "status": "regressed", + } + ], + } + ], + ) + outcome = type("Outcome", (), {"staging_dir": "", "adopted": False})() + + payload = _report_payload(report, outcome) + + serialized = json.dumps(payload) + assert payload["gate_no_regression"] is True + assert payload["gate_trials"][0]["task_deltas"][0]["status"] == "regressed" + assert synthetic_secret not in serialized + assert "REDACTED" in serialized + + +def test_machine_outputs_use_standard_json_for_non_finite_scores(tmp_path) -> None: + report = SleepReport( + night=1, + project=str(tmp_path), + baseline_score=float("nan"), + candidate_score=float("inf"), + gate_no_regression=True, + gate_trials=[ + { + "target": "final", + "baseline_score": None, + "candidate_score": None, + "accepted": False, + "blocked_by_regression": True, + "task_deltas": [], + } + ], + ) + outcome = type("Outcome", (), {"staging_dir": "", "adopted": False})() + + cli_payload = _report_payload(report, outcome) + json.dumps(cli_payload, allow_nan=False) + assert cli_payload["baseline"] is None + assert cli_payload["candidate"] is None + + staging_dir = write_staging( + str(tmp_path), + report=report, + proposed_skill=None, + proposed_memory=None, + live_skill_path="", + live_memory_path="", + report_md="report", + ) + with open(os.path.join(staging_dir, "report.json"), encoding="utf-8") as handle: + staged_payload = json.load(handle) + assert staged_payload["baseline_score"] is None + assert staged_payload["candidate_score"] is None + + evidence_path = tmp_path / "evidence.jsonl" + EvidenceLog(str(evidence_path)).log("gate", "decision", score=float("nan")) + with open(evidence_path, encoding="utf-8") as handle: + evidence_record = json.loads(handle.read()) + assert evidence_record["score"] is None + + +def test_cycle_persists_gate_trials_in_diagnostics(tmp_path) -> None: + project = tmp_path / "project" + project.mkdir() + config = load_config( + invoked_project=str(project), + projects="invoked", + backend="mock", + state_dir=str(tmp_path / "state"), + claude_home=str(tmp_path / ".claude"), + gate_no_regression=True, + auto_adopt=False, + ) + tasks = assign_splits(researcher_persona(), holdout_fraction=0.34, seed=42) + + outcome = run_sleep_cycle( + config, + seed_tasks=tasks, + backend=MockBackend(), + ) + + with open( + os.path.join(outcome.staging_dir, "diagnostics.json"), + encoding="utf-8", + ) as handle: + diagnostics = json.load(handle) + assert diagnostics["gate_no_regression"] is True + assert diagnostics["gate_trials"] + assert diagnostics["gate_trials"] == outcome.report.gate_trials + + +def test_task_delta_artifacts_redact_secret_shaped_metadata(tmp_path) -> None: + project = tmp_path / "project" + project.mkdir() + config = load_config( + invoked_project=str(project), + projects="invoked", + backend="mock", + state_dir=str(tmp_path / "state"), + claude_home=str(tmp_path / ".claude"), + gate_no_regression=True, + auto_adopt=False, + ) + tasks = assign_splits(researcher_persona(), holdout_fraction=0.34, seed=42) + synthetic_secret = "api_key=synthetic-example-123456" + validation_task = next(task for task in tasks if task.split == "val") + validation_task.id = synthetic_secret + validation_task.tags.append(synthetic_secret) + + outcome = run_sleep_cycle( + config, + seed_tasks=tasks, + backend=MockBackend(), + ) + + for filename in ( + "report.md", + "report.json", + "diagnostics.json", + "evidence.jsonl", + ): + with open(os.path.join(outcome.staging_dir, filename), encoding="utf-8") as handle: + persisted = handle.read() + assert synthetic_secret not in persisted + assert "REDACTED" in persisted