From cb8a0e08867661f9591e14bb64e902afbc223ca3 Mon Sep 17 00:00:00 2001 From: Hawthorn Date: Wed, 10 Jun 2026 17:30:47 +0530 Subject: [PATCH 1/6] docs: merge roadmap docs into single hand-off document --- docs/HANDOFF.md | 310 ++++++++++++++++++++++++++++++++++ docs/next-steps.md | 91 ---------- docs/training-integrations.md | 104 ------------ 3 files changed, 310 insertions(+), 195 deletions(-) create mode 100644 docs/HANDOFF.md delete mode 100644 docs/next-steps.md delete mode 100644 docs/training-integrations.md diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md new file mode 100644 index 0000000..9b2522b --- /dev/null +++ b/docs/HANDOFF.md @@ -0,0 +1,310 @@ +# SkillLoop — Hand-Off Document + +**Last updated:** 2026-06-10 +**Repository:** https://github.com/lamenting-hawthorn/skillloop + +This document is the single source of truth for SkillLoop's state, direction, and +outstanding work. Read it at the start of any new session to pick up where we left +off. Update it at the end of every meaningful session before committing. + +--- + +## 1. What SkillLoop Is (Context for New Sessions) + +SkillLoop is a **local autonomous learning sidecar for Hermes**. The user talks to +Hermes normally. SkillLoop runs in the background, watches Hermes's execution +history, and turns good/bad experiences into governed learning artifacts. + +**Critical architecture rule:** Hermes is the source-of-truth runtime substrate. +SkillLoop must NOT duplicate Hermes memory, skills, tracing, cron, tools, gateway, +or subagent systems. SkillLoop is the **learning governor** over those existing +Hermes artifacts. + +Target UX: + +```text +User talks to Hermes normally + -> Hermes stores sessions/traces in state.db + -> SkillLoop runs locally in the background + -> SkillLoop ingests unseen Hermes sessions + -> SkillLoop evaluates, distills, updates datasets, judges readiness + -> User only sees approvals, important summaries, or status +``` + +Target setup UX: one or two commands max. + +```bash +skillloop setup --connect hermes --start +``` + +--- + +## 2. Current System State + +### 2.1 What is implemented (done) + +| Layer | Files | Status | +|-------|-------|--------| +| Trace ingestion | `adapters/hermes.py`, `adapters/generic_jsonl.py` | Can read Hermes `state.db` incrementally (unseen sessions only) and JSONL exports | +| Schema | `schema.py` | `AgentTrace`, `AgentMessage`, `ToolCall`, `Evaluation`, `Proposal` with full provenance fields | +| Persistence | `store.py` | SQLite-backed `.skillloop/skillloop.db` | +| Evaluation | `eval/rubric.py`, `eval/legacy.py`, `eval/registry.py`, `eval/evidence.py` | Deterministic rubric + legacy for benchmarks; structured evidence with tool/file/user feedback tracking | +| Proposal distillation | `distill/memory.py`, `distill/skills.py` | Heuristic memory and skill proposal generation | +| Proposal lifecycle | `review/queue.py`, `apply/filesystem.py` | Review queue (approve/reject/list), local apply under `.skillloop/approved/` | +| Provenance | `provenance.py` | Component provenance with SHA256 hashing; evaluations and proposals carry source hashes | +| Dataset export | `export/sft.py`, `export/dpo.py`, `dataset.py` | SFT JSONL, DPO JSONL (conservative), train/val/test splits, manifests with token estimates | +| Training config | `training_config.py` | Unsloth, TRL, Axolotl config generation with explicit `auto_run=false` safety | +| Benchmarking | `benchmark.py` | Replay benchmark comparing evaluator versions | +| Outer loop | `loop.py` | Schedule, tick, declarative done/stopped/failing conditions | +| Condition engine | `conditions.py` | `score_gte`, `required_tags`, `forbidden_tags`, `max_iterations` | +| Policy | `policy.py` | `SkillLoopPolicy` with ingestion/evaluation/dataset/training sections; default conservative | +| Controller | `controller.py` | `controller_tick()`: ingest->eval->distill->export->report. Incremental Hermes DB ingestion | +| CLI | `cli.py` | Full CLI for all commands | +| Tests | `tests/` | 54 tests, all passing | +| Secret redaction | `sanitize.py` | Pattern-based API key/token/password redaction during ingestion and export | + +### 2.2 Architecture diagram + +```text +Hermes state.db (sessions, messages, tool_calls) + | + v + [adapters/hermes.py] -- read-only, incremental ingestion + | + v + [store.py] -- SQLite persistence of traces, evaluations, proposals + | + v + [eval/registry.py + eval/rubric.py] -- deterministic scoring with evidence + | + v + [distill/memory.py + distill/skills.py] -- proposals with provenance + | + v + [review/queue.py + apply/filesystem.py] -- human-gated approval + | + +--> [export/sft.py, export/dpo.py] -- training datasets + | | + | v + | [dataset.py] -- manifests, splits, token estimates + | | + | v + | [training_config.py] -- Unsloth/TRL/Axolotl configs (auto_run=false) + | + +--> [policy.py + controller.py + loop.py] -- autonomous controller + | + v + [controller_runs/] -- run reports +``` + +--- + +## 3. Short-Term Tasks (P0 — active) + +Goal: make SkillLoop a real local sidecar with setup/start UX. + +### 3.1 Setup/start UX (P0.1) + +**Current state:** Policy and controller code exist but the user must still manually +configure and run commands. + +**Needed:** + +- [ ] `skillloop setup --connect hermes --start` + - Detect `~/.hermes/state.db` + - Create `.skillloop/policy.json` with conservative defaults + - Set ingestion adapter to `hermes-db` + - Run one controller tick + - Install/start background service + +- [ ] `skillloop status` — show current state, last run, pending proposals, dataset stats + +- [ ] Background service runner: + - macOS: launchd plist generation + - Linux: systemd unit or cron job + +**Files likely touched:** `cli.py`, new `install.py` or `setup.py`, new launchd/systemd template files. + +--- + +### 3.2 Controller run history (P0.2) + +**Current state:** Controller writes run reports under `.skillloop/controller_runs/`, but there is no inspection command. + +**Needed:** + +- [ ] `skillloop controller history` — list past runs with summary +- [ ] `skillloop controller show ` — full run detail +- [ ] Store runs in SQLite alongside traces/evaluations + +**Files likely touched:** `controller.py`, `store.py`, `schema.py`, `cli.py`. + +--- + +### 3.3 Auto-export on controller tick (P0.3) + +**Current state:** Controller calls dataset export but only when policy explicitly enables it. The dataset readiness layer does not exist yet. + +**Needed:** + +- [ ] Controller tick should update SFT dataset automatically if `datasets.sft.auto_update: true` +- [ ] Dataset should only include traces that pass eval condition + +--- + +## 4. Long-Term Tasks (P1–P3) + +### 4.1 Dataset Readiness Judge (P1) + +**Goal:** SkillLoop should decide when training data is organized enough to propose training. + +**Needed:** + +- [ ] Minimum record count check +- [ ] Minimum estimated tokens +- [ ] Validation split required +- [ ] Average evaluation score threshold +- [ ] Low-score record cap +- [ ] Duplicate/near-duplicate checks +- [ ] Secret-like content scan +- [ ] Explicit recommendation output: `ready`, `collect_more_data`, `blocked` + +**Files likely:** new `dataset_judge.py`, integration into `controller.py`. + +--- + +### 4.2 Training Planner (P2) + +**Goal:** Create training plans automatically when readiness passes. Keep actual training gated. + +**Needed:** + +- [ ] Training plan object + - Target library: Unsloth, TRL, Axolotl + - Base model + - Dataset manifest + - Hyperparameters + - Expected output paths + - Cost/time/hardware estimates where available + - Approval requirement flag + +- [ ] `skillloop training plan` command + +**Files likely:** new `training_plan.py`, extension of `training_config.py`. + +--- + +### 4.3 Training Runner + Evaluation Harness (P3) + +**Only after P0–P2 are stable.** + +**Needed:** + +- [ ] Training runner + - Manual/approved execution only at first + - Captures logs, exit status, checkpoint paths, runtime, cost placeholder + - Never stores hub tokens or credentials in SkillLoop state + +- [ ] Candidate model registry + - Base model identity + - Adapter/checkpoint path + - Dataset manifest used + - Training config used + - Training run ID + - Provenance hashes + +- [ ] Evaluation harness + - Compare candidate against baseline + - Use held-out validation traces and regression prompts + - Report improvements, regressions, safety failures + +- [ ] Promotion policy + - No auto-promotion initially + - Promotion becomes a reviewed proposal + - Require minimum score improvement and no critical regression + +--- + +## 5. Known Issues / Gaps + +### 5.1 Engineering gaps + +- **Cost tracking:** No per-run or per-trace cost tracking. Must add before LLM evaluators. +- **Error handling:** If one trace errors during loop tick, the whole tick may not recover gracefully. +- **No evaluator-component-change re-ingestion:** If evaluator hash changes, old evaluations are not automatically flagged as stale. +- **DPO conservative only:** DPO export only works when explicit preference pairs already exist in trace metadata. No automatic chosen/rejected generation. +- **Skill distiller basic:** Skill proposal generation uses simple heuristics, not deep trace analysis. +- **No real cron/daemon integration:** The loop exists in code but there is no `skillloop daemon` or automated cron registration. +- **uv.lock exists but pyproject.toml uses setuptools:** uv.lock was accidentally created in the tree; repo does not use uv. + +### 5.2 Non-goals (do NOT implement yet) + +- Do NOT auto-finetune before readiness, cost, evaluation, and promotion gates exist. +- Do NOT auto-promote models into Hermes. +- Do NOT write into global Hermes memory/skills without explicit approval. +- Do NOT duplicate Hermes memory, skills, session tracing, cron, gateway, tools, or subagent systems. +- Do NOT make users manually orchestrate the pipeline. + +--- + +## 6. Architecture Rules (Recurring Principles) + +1. **Hermes is the runtime; SkillLoop is the governor.** Never rebuild Hermes subsystems. +2. **Review-first.** No automatic global mutation. Approvals required for memory/skills/training/promotion. +3. **Provenance everywhere.** Every evaluation, proposal, dataset, and training artifact must carry source hashes. +4. **Conservative DPO.** Only export explicitly provided preference pairs. +5. **Deterministic evaluator first.** LLM evaluators only after cost tracking exists. +6. **Cost-conscious.** Estimate tokens, storage, and compute before scaling. Bounded defaults. +7. **Local-first.** No cloud dependency. State lives in `.skillloop/`. +8. **CLI is admin surface, not user product.** Normal users talk to Hermes. + +--- + +## 7. Repository Layout + +```text +skillloop/ + adapters/ Hermes state.db + generic JSONL ingestion + apply/ Review-approved filesystem exports + distill/ Memory and skill proposal generation + eval/ Evaluator registry, deterministic rubric, legacy, evidence + export/ SFT and DPO dataset exporters + review/ Proposal review queue helpers + benchmark.py Replay benchmark comparing evaluator versions + cli.py Command-line interface (admin/debug surface) + conditions.py Declarative run/done conditions + controller.py Autonomous controller tick + dataset.py Dataset split, manifest, provenance, and stats helpers + loop.py Outer loop scheduling + policy.py Policy schema and defaults + provenance.py Component provenance with SHA256 hashing + sanitize.py Secret redaction + schema.py Trace/Evaluation/Proposal dataclasses + store.py SQLite persistence + training_config.py Unsloth/TRL/Axolotl config generation +tests/ Pytest suite (54 tests) +docs/ + architecture.md System design + cli.md Command reference + safety.md Safety boundaries + trace-schema.md Data format +``` + +--- + +## 8. Session Hand-Off Protocol + +At the end of every session where progress was made: + +1. Update the "What is implemented" table (section 2.1). +2. Check off completed items in short-term/long-term sections. +3. Add any new issues to "Known Issues / Gaps" (section 5). +4. Update the "Last updated" date at the top. +5. Commit with message like `docs: update hand-off document`. + +At the start of every new session: + +1. Read this document top to bottom. +2. Check `git log --oneline -10` for any commits since the last update. +3. Run `python -m pytest -q` to confirm test baseline. +4. Pick up from the first unchecked item in section 3 (short-term). diff --git a/docs/next-steps.md b/docs/next-steps.md deleted file mode 100644 index ab0e459..0000000 --- a/docs/next-steps.md +++ /dev/null @@ -1,91 +0,0 @@ -# SkillLoop Next Steps - -Current product direction: SkillLoop is not a user-operated CLI workflow. It is a local autonomous learning sidecar for Hermes-like agent harnesses. - -Critical architecture rule: Hermes is the source-of-truth runtime substrate. SkillLoop must not duplicate Hermes memory, skills, tracing, cron, tools, gateway, or subagent systems. SkillLoop is the learning governor over those existing Hermes artifacts. - -User-facing model: - -```text -User talks to Hermes normally - -> Hermes emits traces/events - -> SkillLoop runs locally in the background - -> SkillLoop ingests, evaluates, distills, updates datasets, judges readiness - -> User only sees approvals, important summaries, or status -``` - -## P0: Autonomous Controller - -Goal: convert existing CLI primitives into a policy-driven controller that can run unattended locally. - -Tasks: - -1. Add policy schema/config. - - File: `skillloop/policy.py` - - Default policy should be conservative and review-first. - - Policy controls ingestion, evaluation, proposals, datasets, training, promotion. - -2. Add controller tick. - - File: `skillloop/controller.py` - - One tick should execute: ingest configured source -> evaluate -> distill -> update datasets/readiness report. - - Controller should call existing Python functions, not shell out to CLI. - -3. Add run reports. - - Store every autonomous tick summary locally. - - Include actions taken, traces seen, evaluations, proposals, dataset state, readiness result, errors, cost placeholder. - -4. Add Hermes connector. - - First connector reads Hermes `state.db` locally as the canonical execution trace source. - - It must ingest unseen Hermes sessions incrementally, not repeatedly re-ingest only the latest session. - - Preserve Hermes session IDs, message IDs, source/channel, and timestamps as provenance. - - Do not create a parallel memory/skill runtime; read Hermes artifacts and produce governed learning objects. - - Later connector can consume explicit emitted events/webhooks. - -5. Add setup/start UX. - - Target install experience: one or two commands max. - - Example final UX: `skillloop setup --connect hermes --start`. - - On macOS, setup should support launchd; on Linux, systemd or cron. - -## P1: Dataset Readiness - -Goal: SkillLoop should decide when training data is organized enough to propose training. - -Tasks: - -1. Add dataset judge. - - Check min records, validation split, average score, low-score count, duplicates, estimated tokens, secret-like content. - -2. Add readiness result object. - - Output: ready/not ready, reasons, stats, recommendation. - -3. Integrate readiness into controller. - - Controller updates datasets automatically, then writes readiness into run report. - -## P2: Training Planner, Not Auto-Training Yet - -Goal: create training plans automatically when readiness passes, but keep actual training gated. - -Tasks: - -1. Generate training plan from policy + manifest. -2. Estimate hardware/cost/time where possible. -3. Require approval before any expensive training run. - -## P3: Training Runner + Evaluation Harness - -Only after P0-P2 are stable. - -Tasks: - -1. Run Unsloth/TRL/Axolotl jobs under explicit policy gates. -2. Track checkpoints/adapters as model candidates. -3. Evaluate candidate model against baseline. -4. Propose promotion only if candidate wins and no safety regression appears. - -## Non-goals right now - -- Do not make users manually orchestrate the pipeline. -- Do not duplicate Hermes memory, skills, session tracing, cron, gateway, tools, or subagent systems. -- Do not auto-finetune before readiness, cost, evaluation, and promotion gates exist. -- Do not auto-promote models into Hermes. -- Do not write into global Hermes memory/skills without explicit approval. diff --git a/docs/training-integrations.md b/docs/training-integrations.md deleted file mode 100644 index 7291a82..0000000 --- a/docs/training-integrations.md +++ /dev/null @@ -1,104 +0,0 @@ -# SkillLoop Training Integrations Roadmap - -Current stance: SkillLoop should autonomously prepare and judge training data from Hermes's existing execution substrate, but should not automatically run or promote fine-tunes until the controller, readiness, cost, evaluation, and approval gates are implemented. - -Critical architecture rule: Hermes remains the live runtime and source of truth for sessions, tools, memory, skills, cron, gateway, and subagents. SkillLoop should govern learning artifacts derived from Hermes history; it should not rebuild a competing Hermes runtime. - -## Intended autonomous flow - -```text -Hermes state.db sessions/messages/tool calls - -> SkillLoop controller tick - -> eval + condition gates - -> proposal queue - -> SFT/DPO dataset refresh - -> dataset readiness judge - -> training plan proposal - -> approval gate - -> training runner - -> candidate model evaluation - -> promotion proposal -``` - -## What exists today - -- SFT JSONL export. -- Conservative DPO export for explicit preference pairs. -- Train/validation/test split support. -- Dataset manifests with counts, estimated tokens, output files, provenance. -- Training config generation for Unsloth, TRL, and Axolotl. -- Explicit no-auto-training safety flags. - -## What must exist before autonomous training - -1. Dataset readiness judge. - - Minimum record count. - - Minimum estimated tokens. - - Validation split required. - - Average evaluation score threshold. - - Low-score record cap. - - Duplicate/near-duplicate checks. - - Secret-like content scan. - - Explicit recommendation: collect more data, ready to plan, or blocked. - -2. Training plan object. - - Target library: Unsloth, TRL, Axolotl. - - Base model. - - Dataset manifest. - - Hyperparameters. - - Expected output paths. - - Cost/time/hardware estimates where available. - - Approval requirement. - -3. Training runner. - - Manual/approved execution only at first. - - Captures logs, exit status, checkpoint paths, runtime, cost placeholder. - - Never stores hub tokens or credentials in SkillLoop state. - -4. Candidate model registry. - - Base model identity. - - Adapter/checkpoint path. - - Dataset manifest used. - - Training config used. - - Training run ID. - - Provenance hashes. - -5. Evaluation harness. - - Compare candidate against baseline. - - Use held-out validation traces and regression prompts. - - Report improvements, regressions, safety failures. - -6. Promotion policy. - - No auto-promotion initially. - - Promotion becomes a reviewed proposal. - - Require minimum score improvement and no critical regression. - -## Target install/deploy UX - -Final user setup should be one or two commands max: - -```bash -git clone -cd skillloop -./install.sh -``` - -or: - -```bash -skillloop setup --connect hermes --start -``` - -Setup should: - -- Install package/dependencies. -- Create local `.skillloop/` state. -- Detect/connect Hermes profile. -- Write default conservative policy. -- Install background runner using launchd/systemd/cron depending on OS. -- Start controller. -- Print status and next scheduled tick. - -## Product rule - -The normal user talks to Hermes, not SkillLoop. SkillLoop runs in the background and only surfaces approvals, summaries, and important warnings. From 93742eb220bd3d75e651e54c53d1fbf4a5475c89 Mon Sep 17 00:00:00 2001 From: Hawthorn Date: Wed, 10 Jun 2026 18:27:52 +0530 Subject: [PATCH 2/6] docs: add Hermes loop-engineering skills reference (pre-loop-checklist, goal-loop, cron-job-workflows patch) --- docs/HANDOFF.md | 18 ++ docs/analysis/loop-engineering-analysis.md | 192 +++++++++++++++++++++ 2 files changed, 210 insertions(+) create mode 100644 docs/analysis/loop-engineering-analysis.md diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md index 9b2522b..9b98de8 100644 --- a/docs/HANDOFF.md +++ b/docs/HANDOFF.md @@ -308,3 +308,21 @@ At the start of every new session: 2. Check `git log --oneline -10` for any commits since the last update. 3. Run `python -m pytest -q` to confirm test baseline. 4. Pick up from the first unchecked item in section 3 (short-term). + +--- + +## 9. Related Hermes Skills + +These operational skills live in `~/.hermes/skills/system/` and complement +SkillLoop's loop engineering work. They handle the Hermes runtime side +(SkillLoop is the governor, Hermes is the runtime — Rule #1). + +| Skill | Path | Purpose | +|-------|------|---------| +| `pre-loop-checklist` | `system/pre-loop-checklist/` | 4-condition test + 30-second check before creating cron jobs | +| `cron-job-workflows` | `system/cron-job-workflows/` | Cron patterns + STATE.md + Ralph Wiggum detection + Red Flags | +| `goal-loop` | `system/goal-loop/` | /goal primitive: cron jobs that run until objective condition met | + +See `docs/analysis/loop-engineering-analysis.md` section 8 for full details on +what each skill contains and how they were derived from the loop engineering +framework. diff --git a/docs/analysis/loop-engineering-analysis.md b/docs/analysis/loop-engineering-analysis.md new file mode 100644 index 0000000..3a9e1fd --- /dev/null +++ b/docs/analysis/loop-engineering-analysis.md @@ -0,0 +1,192 @@ +# Deep Analysis: Addy Osmani's Loop Engineering vs SkillLoop/Hermes + +**Source:** Addy Osmani (@addyosmani) on X, June 8, 2026. 836K views, 11.5K bookmarks, 4.8K likes. +**Core quote:** "Loop engineering is replacing yourself as the person who prompts the agent. You design the system that does it instead." + +--- + +## 1. What's genuinely good about this post/idea? + +**The paradigm shift is real and important.** + +Addy correctly identifies that the leverage point has moved from "writing better prompts" to "designing the system that generates prompts." This is a genuine phase change in how agentic work gets done — not hype. + +**The outer loop framing is precise.** He distinguishes the inner agentic loop (perceive→reason→act→observe on each turn) from the outer loop (scheduled automation that runs the harness on a timer, spawns helpers, feeds itself). This mental model is actually useful and matches how production agent systems fail. + +**The 5 building blocks + memory are concrete and actionable:** +1. Automations (scheduled triggers) +2. Worktrees (parallel isolation) +3. Skills (project knowledge persistence) +4. Plugins/connectors (tool integration) +5. Sub-agents (maker/verifier split) +6. Memory (durable state outside context window) + +This is a real architecture, not hand-waving. + +**The risks section is honest.** He explicitly calls out that verification is still on you, comprehension debt grows faster, and cognitive surrender is the "comfortable failure." Most AI thought leaders don't caveat their frameworks this directly. + +--- + +## 2. What's well done — what makes it work? + +**It names a real pattern that was already happening.** People were doing loop engineering before Addy named it (Steinberger, Cherny, the HumanLayer folks). Naming it gives practitioners a shared vocabulary and accelerates adoption of the pattern. + +**It positions loop engineering as the third layer in a stack:** +- Prompt engineering → single turn +- Context engineering → conditions around one answer +- Loop engineering → autonomous system across many turns + +This layering makes it clear that loop engineering doesn't replace the previous layers — it wraps them. + +**It's tool-agnostic by design.** Addy explicitly points out the shape is identical across Claude Code and OpenAI Codex. This is a valuable insight: the loop design matters more than which agent tool you use. + +**The maker/verifier sub-agent split is the most important architectural insight.** One agent drafts, a separate agent reviews. This is essentially what SkillLoop's eval→distill→review→apply pipeline does, but at the agent orchestration level rather than the trace learning level. + +--- + +## 3. What can be taken directly into SkillLoop/Hermes? + +**The 6-component taxonomy maps almost 1:1 onto SkillLoop's architecture:** + +| Loop Engineering | SkillLoop/Hermes | +|-----------------|------------------| +| Automation | Hermes cron/scheduler triggers | +| Worktrees | Hermes subagent isolation | +| Skills | Hermes skills (existing) | +| Connectors/MCP | Hermes MCP integrations | +| Sub-agents (maker/verifier) | SkillLoop eval→distill→review pipeline | +| Memory | Hermes memories (existing) | + +**Directly adoptable patterns:** + +1. **Scheduled eval triggers.** Addy mentions "an automation runs every weekday morning on the repo." SkillLoop could add a cron-style eval that runs on a schedule and auto-files issues for traces that drop below quality thresholds. + +2. **The maker/verifier split as a first-class concept.** SkillLoop's eval→distill pipeline is essentially this: one agent (eval) scores, another (distill) proposes, a human reviews. This matches exactly. + +3. **Worktree-style parallel subagent isolation.** Hermes subagents already do this to some degree, but the worktree pattern (isolated git branches per subagent) could make subagent runs safer. + +4. **Memory as a first-class component.** The insight that "the model forgets everything between runs, so the state has to live on disk" is the core insight behind Hermes memories. It's good validation that this is the right architectural choice. + +--- + +## 4. What can be improved by taking inspiration from it? + +**Hermes's outer loop is implicit.** There's no explicit construct for "run this harness on a schedule." The cron integration exists but isn't framed as the primary loop driver. Addy's framing would make this more intentional. + +**SkillLoop's "apply" is manual and local.** Addy describes a loop where connectors auto-update tickets, open PRs, etc. SkillLoop v1 explicitly doesn't write outside `.skillloop/approved/`. This is the right conservative choice for v1, but the architecture should anticipate automated application of approved changes. + +**No explicit "done" condition / eval gate.** Addy mentions /goal in Claude Code ("run until a verifiable condition holds") — SkillLoop could add a declarative "stop condition" to the eval schema: a boolean signal that means "this trace is good enough, stop iterating." + +**The eval→distill coupling is too loose.** In Addy's model, verification happens continuously (the loop checks each result). In SkillLoop, eval and distill are separate commands. The loop concept would tighten this into a continuous pipeline. + +--- + +## 5. If you fully adopt this, what will go WRONG? + +**If SkillLoop/Hermes tried to implement full loop engineering today:** + +**Cost explosion.** Addy explicitly flags "token economics can swing wildly." A naive implementation of "run the eval loop continuously" could burn through context windows and API calls. SkillLoop is designed for offline, local-first evaluation — a full loop would need cost controls that don't exist yet. + +**Premature automation of apply.** Addy describes connectors that "open the PR and update the ticket" — but SkillLoop's conservative boundary (approved artifacts only, no direct Hermes writes) exists for good reason. Full adoption would need security hardening (permissions, audit logging) that the MVP doesn't have. + +**Comprehension debt at the skill level.** Addy: "the faster the loop ships code you didn't write, the bigger the gap." SkillLoop creates skills from traces — if those skills encode patterns the engineer doesn't understand, you get skill-level comprehension debt on top of trace-level debt. + +**Cognitive surrender becomes the default.** If the loop runs and produces artifacts, there's pressure to just accept them. The review step exists in SkillLoop precisely to prevent this — but if reviews become rubber stamps, the whole system degrades. + +**The orchestrator becomes a single point of failure.** If the outer loop has a bug, it runs the wrong thing at scale, faster. With a manual prompting workflow, mistakes are bounded by human speed. + +--- + +## 6. If you adopt it, what will go RIGHT? + +**If SkillLoop/Hermes selectively adopts loop engineering principles:** + +**The eval gate becomes the loop condition.** Instead of "run eval once on demand," you get "run eval continuously and auto-fail traces below threshold." This is exactly what Addy means by "the loop decides whether the result is acceptable." + +**Skills compound correctly.** Addy's insight is that skills should contain "exit criteria" — not vague instructions, but concrete checklists. SkillLoop's distillation could be enhanced to output skill files with explicit exit criteria rather than just general guidance. + +**The memory layer gets validated by production use.** Hermes memories exist as a concept, and loop engineering proves they're load-bearing for long-running agentic systems. SkillLoop's trace→memory pipeline would benefit from explicit "what did the loop learn this week" summaries. + +**Subagent coordination becomes explicit.** The maker/verifier split is Addy's most concrete architectural contribution. SkillLoop already does this internally (eval vs distill), but Hermes could expose this as a first-class pattern for multi-subagent workflows. + +**Evaluation becomes the primary signal.** Addy says "the loop's 'it's done' means something" only if you have a verifier. SkillLoop's deterministic eval rubric is the right foundation for this — it's just not yet wired as a loop condition. + +--- + +## 7. How does this compare to what SkillLoop/Hermes already does better? + +**SkillLoop is already doing the inner loop right.** The eval→distill→review→apply→export pipeline is essentially Addy's maker/verifier pattern applied to trace learning, not code output. SkillLoop just does it at the data layer instead of the agent layer. + +**Hermes already has the memory layer.** Addy calls memory "the trick every long-running agent depends on." Hermes already has memories. This is validation, not a new idea. + +**SkillLoop's conservative boundary is a feature, not a gap.** Addy: "build the loop like someone who intends to stay the engineer." SkillLoop's explicit choice not to auto-apply to Hermes, not to touch existing architecture, not to run fine-tuning automatically — this is exactly the "stay the engineer" posture. + +**Where Hermes/SkillLoop needs work vs. Addy's vision:** + +| Gap | Addy's vision | Current Hermes/SkillLoop | +|-----|--------------|-------------------------| +| Outer loop scheduling | Explicit cron/automation triggers | Implicit (via cron, but not framed as loop) | +| Done conditions | Verifiable stop criteria | Basic score, not declarative conditions | +| Connector layer | MCP-based tool integration | MCP exists, but not loop-aware | +| Apply automation | Auto-PR/ticket updates | Manual approve→apply only | +| Cost awareness | Instrumented token usage | Not in MVP | + +--- + +## Verdict + +Addy Osmani's Loop Engineering is a high-quality, well-named articulation of a real pattern that production agent practitioners have been converging on. It's not revolutionary — it's the right name for work that was already happening (Steinberger, Cherny, HumanLayer, Anthropic's harness design docs). + +**For SkillLoop/Hermes specifically:** The overlap is significant. The 5 building blocks + memory map cleanly onto existing Hermes concepts. The maker/verifier split is already implemented in SkillLoop's pipeline. The main gap is explicit outer-loop scheduling and done conditions — these would tighten the eval gate from "on-demand check" to "continuous condition." + +**The right move:** Don't adopt loop engineering wholesale. Take the taxonomy (it validates what you already built), the done-condition pattern (add this to the eval schema), and the maker/verifier framing (explicitly name this in the architecture docs). Keep the conservative apply boundary. Add cost awareness before adding unattended automation. + +The post's greatest value for SkillLoop is as architecture validation: what you're building is the right thing, and now it has a name that the industry understands. + +--- + +## 8. Implemented Hermes Skills (June 10, 2026) + +Based on this analysis, three operational Hermes skills were built to bridge the +loop engineering concepts into the Hermes runtime. These live in +`~/.hermes/skills/system/` and load into Hermes sessions — they are NOT part of +SkillLoop (architecture rule #1: never rebuild Hermes subsystems). + +### pre-loop-checklist + +`~/.hermes/skills/system/pre-loop-checklist/SKILL.md` + +Gate to load before any `cronjob(action='create')`. Encodes the 4-condition test +(repeats weekly? automated verification? token budget? reproduction environment?) +plus a 30-second tactical checklist, good/bad first loop examples, and a failure +mode reference card. Verdict: most developers don't need a loop yet. + +### cron-job-workflows (patched) + +`~/.hermes/skills/system/cron-job-workflows/SKILL.md` + +Existing skill, patched with three loop-engineering additions: +- **STATE.md template** — the persistent file outside the conversation that holds + progress, lessons learned, and efficiency tracking (cost per accepted change) +- **Ralph Wiggum loop detection** — how to detect loops that fail quietly +- **Red Flags — Never Do These** — 10 concrete prohibitions (soft stop + conditions, self-grading, judgment-call work, etc.) + +### goal-loop + +`~/.hermes/skills/system/goal-loop/SKILL.md` + +Full implementation of the `/goal` primitive as a Hermes cron pattern. Four-part +architecture: cron tick → read STATE.md → checker subagent (different model) → +goal met? report + self-cancel : worker subagent → update STATE.md → loop again. +Includes prompt template, self-cancellation logic, and bail-out signals. + +### Future: SkillLoop loop-engineering evaluator (P1/P2) + +A SkillLoop evaluator that scores traces on loop engineering hygiene — gated on +cost tracking being added first. Would check: +- Was pre-loop-checklist loaded before cron job creation? +- Does the loop use maker/checker split with different models? +- Is STATE.md being maintained? +- Is acceptance rate above 50%? +- Any Ralph Wiggum signals (soft completion, self-grading)? From bf20d7e210ee266ba7e5733deb7236c9d0a4c080 Mon Sep 17 00:00:00 2001 From: Hawthorn Date: Wed, 10 Jun 2026 18:38:26 +0530 Subject: [PATCH 3/6] feat: add loop-engineering Hermes skills (pre-loop-checklist, goal-loop, cron-job-workflows) --- references/hermes-skills/README.md | 48 +++ .../hermes-skills/cron-job-workflows/SKILL.md | 274 ++++++++++++++ references/hermes-skills/goal-loop/SKILL.md | 353 ++++++++++++++++++ .../hermes-skills/pre-loop-checklist/SKILL.md | 231 ++++++++++++ 4 files changed, 906 insertions(+) create mode 100644 references/hermes-skills/README.md create mode 100644 references/hermes-skills/cron-job-workflows/SKILL.md create mode 100644 references/hermes-skills/goal-loop/SKILL.md create mode 100644 references/hermes-skills/pre-loop-checklist/SKILL.md diff --git a/references/hermes-skills/README.md b/references/hermes-skills/README.md new file mode 100644 index 0000000..a02a127 --- /dev/null +++ b/references/hermes-skills/README.md @@ -0,0 +1,48 @@ +# Loop Engineering Skills for Hermes + +These are Hermes Agent skills that implement the loop engineering framework +described in `docs/analysis/loop-engineering-analysis.md`. + +## What's here + +| Skill | Purpose | +|-------|---------| +| `pre-loop-checklist` | Gate before creating any cron job: 4-condition test + 30-second checklist | +| `goal-loop` | The /goal pattern: cron jobs that run until an objective condition is met | +| `cron-job-workflows` | State management, Ralph Wiggum detection, watchdog patterns | + +## How to install + +Copy to your Hermes skills directory: + +```bash +# Install all three +cp -r references/hermes-skills/pre-loop-checklist ~/.hermes/skills/system/ +cp -r references/hermes-skills/goal-loop ~/.hermes/skills/system/ +cp -r references/hermes-skills/cron-job-workflows ~/.hermes/skills/system/ + +# Verify +hermes skills list | grep -E "pre-loop|goal-loop|cron-job" +``` + +Or install individually via Hermes: + +```bash +hermes skills install /path/to/skillloop/references/hermes-skills/pre-loop-checklist/SKILL.md +``` + +## Skill relationship + +``` +pre-loop-checklist (gate: should I build a loop?) + │ + ├──→ cron-job-workflows (how to build and maintain loops) + │ + └──→ goal-loop (the /goal pattern: run until condition met) +``` + +## Origin + +Derived from the [0xCodez loop engineering framework](https://movez.substack.com/p/loop-engineering-the-14-step-roadmap) +(June 2026), Anthropic's evaluator-optimizer pattern, and Addy Osmani's loop +engineering essay. Full analysis: `docs/analysis/loop-engineering-analysis.md`. diff --git a/references/hermes-skills/cron-job-workflows/SKILL.md b/references/hermes-skills/cron-job-workflows/SKILL.md new file mode 100644 index 0000000..2c79260 --- /dev/null +++ b/references/hermes-skills/cron-job-workflows/SKILL.md @@ -0,0 +1,274 @@ +--- +name: cron-job-workflows +description: Patterns and constraints for tasks running as Hermes scheduled cron jobs. Covers restricted tools, output delivery, and proven workarounds. +links: + - references/batch-judge-json-schema.md: Schema used by batch_judge.py result files +metadata: + hermes: + tags: [cron, automation, state, loop-engineering, watchdog] + related_skills: [pre-loop-checklist] +--- + +# Cron Job Workflows + +Use this skill whenever you are running as a scheduled cron job (your conversation starts with "[IMPORTANT: You are running as a scheduled cron job]"). + +## Key restrictions (cron mode) + +These tools/patterns are **blocked** in cron mode — the system cannot prompt for approval without a user present: + +| Pattern | Status | Error you'll see | +|---|---|---| +| `execute_code()` | ❌ Blocked | `BLOCKED: execute_code runs arbitrary local Python... Cron jobs run without a user present to approve it.` | +| `terminal("python3 -c '...'")` | ❌ Blocked | `pending_approval` — matches the `script execution via -e/-c flag` approval pattern | +| `terminal("python3 << 'EOF' ... EOF")` | ❌ Likely blocked | Same inline-execution approval trigger | +| `clarify()` / asking questions | ❌ Not available | "There is no user present — you cannot ask questions, request clarification, or wait for follow-up." | +| `send_message()` | ❌ Not needed | Your final response IS the delivery mechanism | + +## Proven workaround: write_file + terminal + +The reliable pattern for running Python analysis in cron mode: + +1. **Write the script** with `write_file` (always works — no approval needed) +2. **Execute it** with `terminal("python3 ")` (works — shell runs a file, not inline code) + +```python +# Step 1: Write the analysis script +write_file( + path="/path/to/_tmp_report.py", + content='''#!/usr/bin/env python3 +import json, os, glob, statistics + +# ... your analysis logic ... +print(f"Total: {n}, Average: {avg:.2f}") +''' +) + +# Step 2: Execute it +terminal("python3 /path/to/_tmp_report.py") +``` + +### Cleanup +After the task, delete the temp script: +```bash +rm /path/to/_tmp_report.py +``` + +## Output delivery rules + +- Your **final response** is automatically delivered to the configured destination (email, Slack, Telegram, etc.). +- Do NOT call `send_message()` or any delivery tool — let the system handle it. +- If there is genuinely nothing new to report, respond with exactly `[SILENT]` (nothing else) to suppress delivery. +- Never combine `[SILENT]` with content — either report findings normally, or say `[SILENT]` and nothing more. + +## General cron job patterns + +### Reading outputs from long-running commands +If a command times out (exit code 124), the output you got may be partial. Look for: +- Saved output files (logs, result JSONs, CSVs) the script wrote before timeout +- The script's own partial-progress files +- Previous runs' output directories + +Parse those independently rather than re-running the full command. + +### Error reporting +If a command fails (API key issue, network error, binary not found): +- Report the exact error verbatim +- Say what command produced it +- Do NOT retry unless the error was transient (rate limit, timeout) +- For hard failures (bad key, wrong URL), report once and stop + +## STATE.md — The Agent Forgets, the File Does Not + +A loop without persistent state restarts every run. A loop with state resumes. +This is the spine of every working cron job — a markdown file outside the +conversation that holds what's done and what is next. + +### State File Template + +Create a `STATE.md` in the project root or inside a `.hermes/` directory (for +project-scoped jobs) or at a path referenced in the job's `workdir`. The cron +job reads it at the start of each run and writes to it before exiting. + +```markdown +# Loop state · {job-name} + +## Last run +{timestamp} UTC · {summary of what happened} + +## In progress +- {branch/PR} — {status, next step} + +## Completed today +- {branch/PR} → {outcome: merged/closed/escalated} + +## Escalated to humans +- {file/task} — {why it needs a human} + +## Lessons learned (write here, not in chat) +- {date}: {lesson that future runs need to know} + +## Stop conditions met since last review +- /goal "{condition}" achieved on commit {hash} at {time} +``` + +### Where to Store the State File + +- **Markdown in the repo** — `STATE.md` at project root or inside `.hermes/`. + Version-controlled. Simple. Diff-readable. Best for solo or small team work. +- **External system** (Linear, GitHub Issues, a database) — survives across + repos, queryable. Best for production loops where multiple humans need to see + what the loop is doing. + +For long-running loops that risk goal drift, pair the state file with a standing +high-level spec — `AGENTS.md` or `VISION.md` — that the agent rereads each run. +State tells the agent _where it is_. The spec tells it _where to go_. + +### Writing State from a Cron Job + +The cron job writes to STATE.md at the end of each run. Use `write_file` in cron +mode (always works, no approval needed). For append-only updates, read the file +first, append the new section, then write back: + +```python +# Read current state (use read_file tool, not terminal) +# Build updated state with new section appended +# Write back with write_file +``` + +### Cost Per Accepted Change + +Track in STATE.md under an `## Efficiency` section: + +``` +## Efficiency +- Runs this month: _ +- Changes accepted: _ +- Acceptance rate: _% +- Est. token cost: ~$_ +``` + +If acceptance rate is below 50%, the loop is losing money — you're doing review +work the loop was supposed to save you from. + +## Ralph Wiggum Loop Detection + +Named after Geoffrey Huntley's documented failure mode: an agent meant to emit a +completion token _only when finished_ emits it early, and the loop exits on a +half-done job. Without a hard gate, loops fail quietly and keep spending. + +### When You Have a Ralph Wiggum + +- **No real verifier** — just a second agent asked to "review," no objective + signal. Two optimists agreeing. +- **Soft completion conditions** — "done" defined by agent judgment, not by a + test, build, or type check. +- **No hard stops** — loop continues until something external kills it (rate + limit, you noticing) rather than until success is verified. + +### The Fix + +**Something objective that can fail the work.** A test that passes or fails. A +build that compiles or doesn't. A linter that returns zero or non-zero. Not a +verifier that has an opinion. + +### Other Measured Failure Modes + +- **Goal drift over long sessions** — each summarization step is lossy; + constraints disappear at turn 47. Mitigation: standing AGENTS.md reread each + run. +- **Self-preferential bias** — the agent that wrote the code is too nice + grading its own homework. Mitigation: separate verifier subagent with no + exposure to the maker's reasoning (different model). +- **Agentic laziness** — the loop declares "done enough" at partial completion. + Mitigation: objective stop condition checked by a fresh model (/goal pattern). + +## No-Agent Watchdog Pattern (script-only, zero tokens) + +For pure health-check cron jobs that don't need LLM reasoning, use `no_agent=true` with a script. The script's stdout IS the delivery — silent on success, alert on failure. + +### Setup + +```python +cronjob(action="create", + name="Health Watchdog", + schedule="every 5m", + script="my-watchdog.sh", # path under ~/.hermes/scripts/ + no_agent=True, # skip LLM entirely — zero tokens + deliver="origin") # CRITICAL: NOT "local" — failures must reach user +``` + +### Script contract + +``` +Exit 0 + no stdout → SILENT (nothing delivered) ← the healthy case +Exit 0 + stdout → delivered as a normal message +Exit non-zero → delivered as an error alert +``` + +### Script template + +```bash +#!/usr/bin/env bash +set -euo pipefail +ALERTS=() + +# Check 1: ... +if ! some_check; then + ALERTS+=("Thing is broken") +fi + +# Report +if [[ ${#ALERTS[@]} -gt 0 ]]; then + echo "🔴 ALERT ($(date '+%H:%M:%S'))" + for alert in "${ALERTS[@]}"; do + echo " ❌ $alert" + done + exit 1 # Non-zero → delivered as alert +fi +exit 0 # Silent success → nothing delivered +``` + +### Critical: deliver="origin" not "local" + +When `deliver="local"`, output is saved but NEVER delivered to the user — even on failure. A watchdog that fails silently is useless. Always use `deliver="origin"` for watchdogs. + +## Red Flags — Never Do These + +- **Build a cron job without running the pre-loop-checklist.** Most developers + fail at least one condition. Load `pre-loop-checklist` skill and run the + 4-condition test + 30-second checklist before every `cronjob(action='create')`. +- **Use a soft stop condition.** "Done when it looks good" never holds. Use a + test, a type pass, or a passing build. +- **Let the agent grade its own homework.** The maker and the checker must be + different subagents, ideally different models. One agent doing both writing and + verifying = self-preferential bias = "A+" every time. +- **Run a cron job without a token budget cap.** Loops re-read context and retry. + Without a cap, ambitious loops burn 5-10x the tokens you expected. +- **Skip the STATE.md.** Tomorrow's run restarts from zero instead of resuming. + The agent forgets. The file does not. +- **Loop on judgment-call work.** Architecture, auth, payments, vague product + decisions — keep the loop on lint-and-fix, not strategy. +- **Not read the diffs.** Comprehension debt at compound interest. The day you + debug a system no one has read costs more than the tokens ever did. +- **Auto-install community skills in a production loop.** 520 of 17,022 audited + skills leak credentials. Read the source before installing. +- **Run loops on a consumer plan with heavy verification.** Token bill or rate + limit — one of them gets you. +- **Use `deliver="local"` for a watchdog.** A watchdog that keeps its alerts local + is silent when it fails. Always use `deliver="origin"`. + +## Pitfalls + +- **Shell here-docs in terminal**: `python3 << 'EOF' ... EOF` may trigger the same inline-approval pattern as `-c`. Always use `write_file` first. +- **Background processes**: `terminal(background=True)` without `notify_on_complete=True` runs silently and you'll forget to poll. For cron jobs, prefer foreground with a generous timeout (up to 600s). +- **JSON field names**: Batch judge scripts output `overall_score`, not `score` or `grade`. Always check the actual JSON structure before parsing. +- **Temp file collisions**: Use a distinctive name (e.g., `_nightly_report.py`) with an underscore prefix so it sorts visibly and is easy to clean up. +- **Write blocking**: `write_file` is the only guaranteed-unblocked way to create files in cron mode. Do not attempt `echo > file` or `cat > file` heredocs in terminal — they may also trigger approval patterns. + +## Verification + +After writing and running a temp script: +- Confirm the output is correct by inspecting it +- Remove the temp file when done (`terminal("rm ...")`) +- Include cleanup in the same turn diff --git a/references/hermes-skills/goal-loop/SKILL.md b/references/hermes-skills/goal-loop/SKILL.md new file mode 100644 index 0000000..72dfcff --- /dev/null +++ b/references/hermes-skills/goal-loop/SKILL.md @@ -0,0 +1,353 @@ +--- +name: goal-loop +description: "The /goal pattern: cron jobs that run until an objective condition is met, checked by an independent model (maker/checker split). Source: loop engineering framework by 0xCodez / Anthropic evaluator-optimizer pattern." +version: 1.0.0 +author: Hermes Agent +license: MIT +metadata: + hermes: + tags: [loop-engineering, cron, goal, automation, maker-checker, evaluator-optimizer] + related_skills: [pre-loop-checklist, cron-job-workflows] + source: "Adapted from Loop engineering: the 14-step roadmap from prompter to loop designer by 0xCodez (movez.substack.com)" +--- + +# Goal Loop — The /goal Pattern + +The `/goal` primitive: a cron job that runs on a cadence until an **objective +condition** is met, verified by an **independent checker model** — not the agent +that writes the code. This is the maker-vs-checker split applied to the stop +condition itself. + +## When to Use + +- After passing the `pre-loop-checklist` (4-condition test + 30-second check) +- When you have a clear, machine-checkable goal condition +- When the work is iterative (agent tries, fails, learns, retries) +- When verification can be fully automated + +Do NOT use for: +- One-shot tasks (use a single `delegate_task`) +- Vague goals ("make it better") +- Work that can't be verified by a test/build/linter +- Anything where "done" is a judgment call + +## The Pattern (4 Parts) + +``` + ┌──────────────────────┐ + │ Cron job fires on │ + │ cadence (e.g. 30m) │ + └──────────┬───────────┘ + │ + ┌──────────▼───────────┐ + │ Read STATE.md │ + │ (resume, don't │ + │ restart from zero) │ + └──────────┬───────────┘ + │ + ┌──────────▼───────────┐ + │ CHECKER subagent │ + │ (DIFFERENT model) │ + │ Evaluate: is goal │ + │ condition met? │ + └──────────┬───────────┘ + │ + ┌───────────┴───────────┐ + │ │ + ▼ ▼ + ┌──────────┐ ┌──────────┐ + │ GOAL │ │ GOAL │ + │ MET ✓ │ │ NOT MET │ + └────┬─────┘ └────┬─────┘ + │ │ + ▼ ▼ + ┌──────────┐ ┌──────────┐ + │ Report │ │ WORKER │ + │ success │ │ subagent │ + │ Update │ │ (does │ + │ STATE │ │ the │ + │ Option: │ │ work) │ + │ self- │ └────┬─────┘ + │ cancel │ │ + └──────────┘ ▼ + ┌──────────┐ + │ Update │ + │ STATE │ + │ Report │ + │ progress│ + │ (loop │ + │ again │ + │ next │ + │ tick) │ + └──────────┘ +``` + +## Prerequisites + +Before creating a goal loop, ensure: + +1. **A hard gate exists** — test suite, linter, type checker, or build that can + OBJECTIVELY fail bad output. No gate = Ralph Wiggum loop. +2. **A reproduction environment** — the worker can run the code it changes. +3. **STATE.md template ready** — where the loop records progress across runs. + See cron-job-workflows skill for the template. +4. **A different model for the checker** — the model that verifies must NOT be + the same as the model that writes. Configure via `delegation.model` in + config.yaml. Self-preferential bias is real: the maker grades its own + homework and it's always "A+." + +## The Goal Condition + +Must be **objective and machine-verifiable**. The checker evaluates it, not the +worker. + +| Good goal conditions | Bad goal conditions | +|---|---| +| "All tests in test/auth pass" | "The code looks good" | +| "Lint returns zero errors" | "The auth module is done" | +| "Build succeeds on main branch" | "Everything is fixed" | +| "Type check passes with zero errors" | "The performance is acceptable" | +| "CI is green on PR #42" | "It's better than before" | + +## Setup: Creating a Goal Loop + +### Step 1: Pass the pre-loop-checklist + +Load `pre-loop-checklist` skill and run the 4-condition test + 30-second +checklist against the task. If it doesn't pass, do NOT build a goal loop. + +### Step 2: Initialize STATE.md + +Create a STATE.md in the project root (or .hermes/ directory) using the +template from cron-job-workflows skill. The first run reads this to resume. + +### Step 3: Create the cron job + +Use the prompt template below. Replace the bracketed values. + +### Step 4: Monitor the first 3 runs + +Watch the first 3-5 runs manually. Verify: +- The checker is actually checking (not just rubber-stamping) +- The worker is actually working (not looping on the same thing) +- STATE.md is being updated +- The gate is catching bad output + +## Prompt Template + +Copy this template, fill in the brackets, and pass as the `prompt` to +`cronjob(action='create', ...)`. + +``` +You are a /goal loop runner. Your job is to achieve a specific, +machine-checkable goal. You run on a cadence and check progress each +tick. Do NOT do the work yourself — delegate everything. + +## Goal condition +{GOAL_CONDITION — e.g., "All tests in test/auth pass and lint is clean"} + +## Work to do when goal is not met +{WORK_DESCRIPTION — e.g., "Scan src/auth for failing tests, draft fixes +on branch claude/auth-fixes, open draft PR when ready"} + +## State file +Read {PATH_TO_STATE_MD} at the start of every run. Update it at the end. +Use the standard STATE.md format from cron-job-workflows skill. + +## Each tick (every run): + +### 1. Read state +Read {PATH_TO_STATE_MD} to understand current progress and what happened +last run. + +### 2. Check the goal (CHECKER — use a DIFFERENT model) +Dispatch a checker subagent via delegate_task: +- goal: "Verify if goal condition is met: {GOAL_CONDITION}" +- The checker must run the actual verification (tests, lint, build) — NOT + just read files. It must produce an objective PASS or FAIL. +- Use toolsets: ['terminal', 'file'] +- This subagent must use a DIFFERENT model from the worker. Set up + delegation config accordingly. + +### 3. If goal is MET: +- Write final status to STATE.md under "## Stop conditions met" +- Final response: report success with specific evidence (test output, + commit hash, etc.) +- IMPORTANT: After reporting success, call cronjob(action='list') to find + this job, then cronjob(action='remove', job_id=THE_JOB_ID) to self-cancel. + Search for the job by name or look for the one with your prompt text. +- Do NOT continue running after goal is met. + +### 4. If goal is NOT met: +- Dispatch a worker subagent via delegate_task: + - goal: "Make progress toward: {GOAL_CONDITION}" + - context: Include the checker's findings, current state from STATE.md, + and the work description: {WORK_DESCRIPTION} + - toolsets: ['terminal', 'file'] + - The worker changes code, creates branches, runs tests + +- After worker completes, verify: + a. Was anything actually shipped (not just "analyzed")? + b. Did the worker's changes get past the gate (tests pass, lint clean)? + c. If nothing changed or gate failed → record in STATE.md and escalate + +- Update STATE.md: + - "## Last run" — timestamp + summary + - "## In progress" — what's being worked on + - "## Completed today" — what finished + - "## Escalated to humans" — what needs a human + - Update "## Efficiency" counters + +- Final response: report progress (what was done, what remains, when next + tick fires) + +## Hard stops +- Token budget: {MAX_TOKENS_PER_RUN or "stop after 3 worker attempts per tick"} +- Time limit: tick must complete within {TIME_LIMIT} +- If approaching limits, report partial progress and defer to next tick + +## Never do +- Do the work yourself (always delegate to worker subagent) +- Let the checker and worker use the same model (self-preferential bias) +- Skip the STATE.md update +- Continue running after goal is met (self-cancel) +- Accept "done enough" without the gate passing +- Touch {RESTRICTED_PATHS — e.g., "src/payments/, src/auth/, production configs"} +``` + +## Example: CI Triage Goal Loop + +```python +cronjob( + action='create', + name='auth-quality-goal-loop', + schedule='30m', + deliver='origin', + prompt="""You are a /goal loop runner. + +## Goal condition +All tests in test/auth/ pass AND lint returns zero errors on src/auth/ + +## Work to do +Scan CI failures in test/auth/, classify root causes, draft fixes on +branches under claude/auth-fixes/, run tests locally, open draft PRs when +fixes pass locally. + +## State file +Read STATE.md at project root. Update it at the end of every run. + +[Full template follows with checked values...] +""", + workdir='/Users/raghav/myproject', + skills=['pre-loop-checklist', 'cron-job-workflows'], + enabled_toolsets=['terminal', 'file', 'delegation', 'cronjob'], +) +``` + +## Self-Cancellation + +When the goal is met, the cron agent calls: + +```python +# Find this job +jobs = cronjob(action='list') +# Find your job in the list (match by name or prompt content) +# Remove it +cronjob(action='remove', job_id='THE_JOB_ID') +``` + +The `cronjob` tool is enabled in the `enabled_toolsets` so the agent can +self-terminate. Adding a new cron job from a cron run is prohibited, but +removing yourself is fine. + +### Alternative: Keep the job, just go silent + +If you want the loop to remain for future regressions: +- Don't self-cancel +- Instead respond with `[SILENT]` when goal is met and no new work exists +- The job continues to check on cadence but stays quiet until the goal + regresses + +## Monitoring and Bail-Out + +Goal loops can get stuck. Monitor the first 3-5 runs, then spot-check weekly. + +### Bail-out signals (after which you should cancel the job) + +- 3 consecutive ticks with no progress (STATE.md shows no changes) +- Acceptance rate drops below 30% +- The loop starts touching files it shouldn't (permission scope creep) +- Token cost per tick exceeds 3x your estimate +- The checker repeatedly passes but you find bugs manually (gate is rotten) + +### How to bail out + +```python +# Find the job +cronjob(action='list') + +# Cancel it +cronjob(action='remove', job_id='THE_JOB_ID') +``` + +## Pitfalls + +- **Checker and worker use the same model.** This is the #1 failure mode. + The checker must be a different model or at minimum a different subagent + with no access to the worker's reasoning. +- **Goal condition is too vague.** "Make it better" can never be objectively + verified. Use a test, lint, build, or type check. +- **Worker produces no real output.** An "analysis" that changes nothing is + the agent agreeing with itself. Verify the worker actually shipped a + change. +- **Forgetting to update STATE.md.** Tomorrow's run restarts from zero, + re-derives everything, burns tokens re-learning what yesterday already + figured out. +- **No hard stop per tick.** Without one, a single tick can spiral and burn + the entire day's token budget. +- **Using this for judgment-call work.** Architecture, auth, payments — + keep /goal loops on machine-checkable changes only. +- **Not monitoring the first runs.** Goal loops that look correct but fail + silently (Ralph Wiggum) are only caught by human observation. + +## Quick Reference + +``` + ┌───────────────────────────┐ + │ /goal loop decision │ + └───────────┬───────────────┘ + │ + ┌──────────▼──────────┐ + │ pre-loop-checklist │ + │ (4 conditions + │ + │ 30-second test) │ + └──────┬───────┬──────┘ + │ │ + PASS FAIL + │ │ + │ └─────→ Don't build + │ + ┌──────▼──────────────┐ + │ Is goal objective │ + │ and machine- │ + │ checkable? │ + └──────┬───────┬──────┘ + │ │ + YES NO + │ │ + │ └─────→ Don't build + │ + ▼ + ┌──────────────┐ + │ BUILD THE │ + │ GOAL LOOP │ + │ │ + │ 1. STATE.md │ + │ 2. Cron job │ + │ (prompt │ + │ template)│ + │ 3. Monitor │ + │ first 3 │ + │ runs │ + └──────────────┘ +``` diff --git a/references/hermes-skills/pre-loop-checklist/SKILL.md b/references/hermes-skills/pre-loop-checklist/SKILL.md new file mode 100644 index 0000000..b4365c9 --- /dev/null +++ b/references/hermes-skills/pre-loop-checklist/SKILL.md @@ -0,0 +1,231 @@ +--- +name: pre-loop-checklist +description: "Gate before creating any cron job: run the 4-condition test + 30-second checklist to determine if loop engineering is worth the token cost. Source: 0xCodez loop engineering framework (June 2026), Anthropic engineering docs, Addy Osmani." +version: 1.0.0 +author: Hermes Agent +license: MIT +metadata: + hermes: + tags: [loop-engineering, cron, gates, automation, cost-control] + source: "Adapted from Loop engineering: the 14-step roadmap from prompter to loop designer by 0xCodez (movez.substack.com)" +--- + +# Pre-Loop Checklist + +Load this skill BEFORE creating any new cron job. It encodes the decision framework from +loop engineering: should this task be a loop, or should it stay as a manual prompt? + +The honest answer: most developers don't need a loop yet. This checklist saves you from +building loops that cost more than they return. + +## When to Use + +- Before calling `cronjob(action='create', ...)` +- When a user says "schedule this" for a coding/automation task +- Before wrapping a manual task into an unattended loop + +Skip this checklist for: +- Pure watchdog scripts (`no_agent=true`) — those don't use LLM tokens +- One-shot scheduled messages (e.g., "remind me at 9am") — no loop involved +- Non-coding cron jobs (weather reports, news briefs, daily summaries) + +--- + +## Tier 1: The 4-Condition Test (Strategic) + +Run this first. **Miss one condition and the loop costs more than it returns.** + +### Condition 1: The task repeats at least weekly + +"If the work does not recur weekly, you don't have a loop — you have a script you +ran once." + +**Ask:** Will this task fire at least once per week? + +- YES → continue to condition 2 +- NO → keep it as a manual prompt or one-shot. A loop setup cost will never amortize. + +### Condition 2: Verification is automated + +"The loop needs something that can fail the work without you in the room." + +**Ask:** Is there a test suite, type checker, linter, or build that can reject bad output? + +- YES → continue to condition 3 +- NO → STOP. A loop with no real check is the agent agreeing with itself on repeat. + You're back in the chair reading every diff — the exact job the loop was + supposed to remove. + +### Condition 3: Your token budget can absorb the waste + +"Loops re-read context, retry, explore. That burns tokens whether or not the run +ships anything." + +**Ask:** If this loop burns 3-5x more tokens than a single manual run (retries, +re-reading context, exploration), will you still be happy? + +- YES → continue to condition 4 +- NO → STOP. The technique scales with budget. Solo builders on consumer plans + get the token bill before the productivity gain. + +### Condition 4: The agent has a reproduction environment + +"The agent needs the ability to run the code it writes and see what breaks." + +**Ask:** Can the agent actually run the code it changes? (test commands, build +toolchain, dev environment, logs)? + +- YES → all 4 conditions met. Proceed to the 30-second checklist. +- NO → STOP. Without a reproduction environment, the loop iterates blind. + +--- + +## Tier 2: The 30-Second Checklist (Tactical) + +Only run this if all 4 strategic conditions passed. **Miss one box and keep it +as a manual prompt.** + +- [1] **The task happens at least weekly.** Less than weekly → setup cost will + never amortize. +- [2] **A test, type check, build, or linter can reject bad output.** No + automated gate → the agent grades its own homework. +- [3] **The agent can run the code it changes.** No reproduction environment → + iteration is blind. +- [4] **The loop has a hard stop.** Token budget, iteration count, or time limit. + Without one, the loop runs until someone notices the bill. +- [5] **A human reviews before merge, deploy, or dependency changes.** Anything + irreversible needs a human approval gate before action. + +### Verdict + +- All 5 boxes checked → **BUILD.** The task is right for loop engineering. +- 1-2 boxes unchecked → **FIX.** Address the gaps before building. +- 3+ boxes unchecked → **SKIP.** This task should stay as a manual prompt. + +--- + +## Good First Loops vs. Bad First Loops + +### Safe to loop (machine-checkable, bounded scope) + +- **CI failure triage** — nightly, scan failures, classify causes, draft fix PRs +- **Dependency bump PRs** — weekly, scan for updates, test compatibility, open PRs +- **Lint-and-fix passes** — on every PR open event, apply style fixes +- **Flaky test reproduction** — loop until a theory survives the test +- **Issue-to-PR drafts** on code with strong test coverage + +### Do NOT loop (judgment calls, irreversible, vague) + +- Architecture rewrites +- Auth or payments code +- Production deploys +- Vague product work +- Anything where "done" is a judgment call + +--- + +## After You Build: Track Cost Per Accepted Change + +The metric that matters is **cost per accepted change**, not tokens spent or tasks +attempted. If your acceptance rate is below 50%, the loop is losing money. + +Add to the cron job's state file (see cron-job-workflows skill for STATE.md template): + +``` +## Efficiency +- Runs this month: _ +- Changes accepted: _ +- Acceptance rate: _% +``` + +--- + +## Who Wins, Who Loses (Economics) + +### Wins (build loops) + +- Teams with repetitive, machine-checkable work and the budget to run it +- Codebases with strong existing test suites +- Async-first teams with multi-agent patterns already in use + +### Loses (skip it today) + +- **Solo builders on consumer plans** — token bill arrives before productivity +- **No automated verification** — loop with no real check is self-agreement +- **Review capacity is the bottleneck** — loop generates more code, makes queue longer + +For one-off tasks, exploratory work, or anything where "done" is a judgment call, +**a single well-aimed prompt still wins.** Loop engineering is real, and most +developers don't need it yet. + +--- + +## Common Failure Modes (Reference) + +When reviewing existing cron jobs, check for these. See cron-job-workflows skill +for the Ralph Wiggum pattern and mitigation. + +| Failure mode | Signal | Fix | +|---|---|---| +| No real verifier | Second agent asked to "review" with no test/lint/build | Add objective gate | +| Self-preferential bias | Maker grades own homework (same model writes and checks) | Separate verifier subagent, different model | +| Goal drift | Constraints disappear at turn 47 in long sessions | Re-read AGENTS.md or VISION.md each run | +| Agentic laziness | Loop declares "done enough" at partial completion | /goal with objective stop condition checked by fresh model | +| Ralph Wiggum | Loop exits on half-done job, no one notices | Hard gate (test/build/lint), not subjective review | +| Comprehension debt | Code ships faster than anyone reads diffs | Read diffs. Spot-check gates. Block architecture work. | + +--- + +## Pitfalls + +- **Skipping the 4-condition test.** Most developers fail at least one condition. + Step 2 in the article exists for a reason. +- **One agent doing both writing and verifying.** Self-preferential bias. The maker + grades its own homework and it's always "A+." +- **No state file.** Tomorrow's run restarts from zero instead of resuming. +- **Vague stop conditions.** "Done when it looks good" never holds. Use a test, + a type pass, or a passing build. +- **No token budget cap.** Loops re-read context and retry. Without a cap, + ambitious loops burn 5-10x the tokens you expected. +- **Running loops on a consumer plan with heavy verification.** Token bill or + rate limit, one of them gets you. +- **Loops on judgment-call work.** Architecture, auth, payments, vague product + decisions. Keep the loop on lint-and-fix, not strategy. +- **Not reading the diffs.** Comprehension debt at compound interest. The day you + debug a system no one has read costs more than the tokens ever did. + +--- + +## Quick Reference Card + +``` + ┌─────────────────────────┐ + │ Should I loop this? │ + └───────────┬─────────────┘ + │ + ┌────────▼────────┐ + │ 4-condition │ + │ test passed? │ + └────┬───────┬────┘ + │ │ + YES NO + │ │ + │ └──────→ Keep as manual prompt + │ + ┌────────▼────────┐ + │ 30-second │ + │ checklist? │ + └────┬───────┬────┘ + │ │ + ALL NOT + BOXES ALL + │ │ + │ └──────→ Fix gaps or skip + │ + ▼ + ┌─────────┐ + │ BUILD │ + │ THE │ + │ LOOP │ + └─────────┘ +``` From fe38675d937619c4c0e1d8411cd33939c16f7a84 Mon Sep 17 00:00:00 2001 From: Hawthorn Date: Wed, 10 Jun 2026 19:09:57 +0530 Subject: [PATCH 4/6] chore: remove private hand-off document from public repo --- .gitignore | 2 + docs/HANDOFF.md | 328 ------------------------------------------------ 2 files changed, 2 insertions(+), 328 deletions(-) delete mode 100644 docs/HANDOFF.md diff --git a/.gitignore b/.gitignore index 651994c..a13a102 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,5 @@ data/*.jsonl !.env.example .venv/ venv/ +.worktrees/ +docs/HANDOFF.md diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md deleted file mode 100644 index 9b98de8..0000000 --- a/docs/HANDOFF.md +++ /dev/null @@ -1,328 +0,0 @@ -# SkillLoop — Hand-Off Document - -**Last updated:** 2026-06-10 -**Repository:** https://github.com/lamenting-hawthorn/skillloop - -This document is the single source of truth for SkillLoop's state, direction, and -outstanding work. Read it at the start of any new session to pick up where we left -off. Update it at the end of every meaningful session before committing. - ---- - -## 1. What SkillLoop Is (Context for New Sessions) - -SkillLoop is a **local autonomous learning sidecar for Hermes**. The user talks to -Hermes normally. SkillLoop runs in the background, watches Hermes's execution -history, and turns good/bad experiences into governed learning artifacts. - -**Critical architecture rule:** Hermes is the source-of-truth runtime substrate. -SkillLoop must NOT duplicate Hermes memory, skills, tracing, cron, tools, gateway, -or subagent systems. SkillLoop is the **learning governor** over those existing -Hermes artifacts. - -Target UX: - -```text -User talks to Hermes normally - -> Hermes stores sessions/traces in state.db - -> SkillLoop runs locally in the background - -> SkillLoop ingests unseen Hermes sessions - -> SkillLoop evaluates, distills, updates datasets, judges readiness - -> User only sees approvals, important summaries, or status -``` - -Target setup UX: one or two commands max. - -```bash -skillloop setup --connect hermes --start -``` - ---- - -## 2. Current System State - -### 2.1 What is implemented (done) - -| Layer | Files | Status | -|-------|-------|--------| -| Trace ingestion | `adapters/hermes.py`, `adapters/generic_jsonl.py` | Can read Hermes `state.db` incrementally (unseen sessions only) and JSONL exports | -| Schema | `schema.py` | `AgentTrace`, `AgentMessage`, `ToolCall`, `Evaluation`, `Proposal` with full provenance fields | -| Persistence | `store.py` | SQLite-backed `.skillloop/skillloop.db` | -| Evaluation | `eval/rubric.py`, `eval/legacy.py`, `eval/registry.py`, `eval/evidence.py` | Deterministic rubric + legacy for benchmarks; structured evidence with tool/file/user feedback tracking | -| Proposal distillation | `distill/memory.py`, `distill/skills.py` | Heuristic memory and skill proposal generation | -| Proposal lifecycle | `review/queue.py`, `apply/filesystem.py` | Review queue (approve/reject/list), local apply under `.skillloop/approved/` | -| Provenance | `provenance.py` | Component provenance with SHA256 hashing; evaluations and proposals carry source hashes | -| Dataset export | `export/sft.py`, `export/dpo.py`, `dataset.py` | SFT JSONL, DPO JSONL (conservative), train/val/test splits, manifests with token estimates | -| Training config | `training_config.py` | Unsloth, TRL, Axolotl config generation with explicit `auto_run=false` safety | -| Benchmarking | `benchmark.py` | Replay benchmark comparing evaluator versions | -| Outer loop | `loop.py` | Schedule, tick, declarative done/stopped/failing conditions | -| Condition engine | `conditions.py` | `score_gte`, `required_tags`, `forbidden_tags`, `max_iterations` | -| Policy | `policy.py` | `SkillLoopPolicy` with ingestion/evaluation/dataset/training sections; default conservative | -| Controller | `controller.py` | `controller_tick()`: ingest->eval->distill->export->report. Incremental Hermes DB ingestion | -| CLI | `cli.py` | Full CLI for all commands | -| Tests | `tests/` | 54 tests, all passing | -| Secret redaction | `sanitize.py` | Pattern-based API key/token/password redaction during ingestion and export | - -### 2.2 Architecture diagram - -```text -Hermes state.db (sessions, messages, tool_calls) - | - v - [adapters/hermes.py] -- read-only, incremental ingestion - | - v - [store.py] -- SQLite persistence of traces, evaluations, proposals - | - v - [eval/registry.py + eval/rubric.py] -- deterministic scoring with evidence - | - v - [distill/memory.py + distill/skills.py] -- proposals with provenance - | - v - [review/queue.py + apply/filesystem.py] -- human-gated approval - | - +--> [export/sft.py, export/dpo.py] -- training datasets - | | - | v - | [dataset.py] -- manifests, splits, token estimates - | | - | v - | [training_config.py] -- Unsloth/TRL/Axolotl configs (auto_run=false) - | - +--> [policy.py + controller.py + loop.py] -- autonomous controller - | - v - [controller_runs/] -- run reports -``` - ---- - -## 3. Short-Term Tasks (P0 — active) - -Goal: make SkillLoop a real local sidecar with setup/start UX. - -### 3.1 Setup/start UX (P0.1) - -**Current state:** Policy and controller code exist but the user must still manually -configure and run commands. - -**Needed:** - -- [ ] `skillloop setup --connect hermes --start` - - Detect `~/.hermes/state.db` - - Create `.skillloop/policy.json` with conservative defaults - - Set ingestion adapter to `hermes-db` - - Run one controller tick - - Install/start background service - -- [ ] `skillloop status` — show current state, last run, pending proposals, dataset stats - -- [ ] Background service runner: - - macOS: launchd plist generation - - Linux: systemd unit or cron job - -**Files likely touched:** `cli.py`, new `install.py` or `setup.py`, new launchd/systemd template files. - ---- - -### 3.2 Controller run history (P0.2) - -**Current state:** Controller writes run reports under `.skillloop/controller_runs/`, but there is no inspection command. - -**Needed:** - -- [ ] `skillloop controller history` — list past runs with summary -- [ ] `skillloop controller show ` — full run detail -- [ ] Store runs in SQLite alongside traces/evaluations - -**Files likely touched:** `controller.py`, `store.py`, `schema.py`, `cli.py`. - ---- - -### 3.3 Auto-export on controller tick (P0.3) - -**Current state:** Controller calls dataset export but only when policy explicitly enables it. The dataset readiness layer does not exist yet. - -**Needed:** - -- [ ] Controller tick should update SFT dataset automatically if `datasets.sft.auto_update: true` -- [ ] Dataset should only include traces that pass eval condition - ---- - -## 4. Long-Term Tasks (P1–P3) - -### 4.1 Dataset Readiness Judge (P1) - -**Goal:** SkillLoop should decide when training data is organized enough to propose training. - -**Needed:** - -- [ ] Minimum record count check -- [ ] Minimum estimated tokens -- [ ] Validation split required -- [ ] Average evaluation score threshold -- [ ] Low-score record cap -- [ ] Duplicate/near-duplicate checks -- [ ] Secret-like content scan -- [ ] Explicit recommendation output: `ready`, `collect_more_data`, `blocked` - -**Files likely:** new `dataset_judge.py`, integration into `controller.py`. - ---- - -### 4.2 Training Planner (P2) - -**Goal:** Create training plans automatically when readiness passes. Keep actual training gated. - -**Needed:** - -- [ ] Training plan object - - Target library: Unsloth, TRL, Axolotl - - Base model - - Dataset manifest - - Hyperparameters - - Expected output paths - - Cost/time/hardware estimates where available - - Approval requirement flag - -- [ ] `skillloop training plan` command - -**Files likely:** new `training_plan.py`, extension of `training_config.py`. - ---- - -### 4.3 Training Runner + Evaluation Harness (P3) - -**Only after P0–P2 are stable.** - -**Needed:** - -- [ ] Training runner - - Manual/approved execution only at first - - Captures logs, exit status, checkpoint paths, runtime, cost placeholder - - Never stores hub tokens or credentials in SkillLoop state - -- [ ] Candidate model registry - - Base model identity - - Adapter/checkpoint path - - Dataset manifest used - - Training config used - - Training run ID - - Provenance hashes - -- [ ] Evaluation harness - - Compare candidate against baseline - - Use held-out validation traces and regression prompts - - Report improvements, regressions, safety failures - -- [ ] Promotion policy - - No auto-promotion initially - - Promotion becomes a reviewed proposal - - Require minimum score improvement and no critical regression - ---- - -## 5. Known Issues / Gaps - -### 5.1 Engineering gaps - -- **Cost tracking:** No per-run or per-trace cost tracking. Must add before LLM evaluators. -- **Error handling:** If one trace errors during loop tick, the whole tick may not recover gracefully. -- **No evaluator-component-change re-ingestion:** If evaluator hash changes, old evaluations are not automatically flagged as stale. -- **DPO conservative only:** DPO export only works when explicit preference pairs already exist in trace metadata. No automatic chosen/rejected generation. -- **Skill distiller basic:** Skill proposal generation uses simple heuristics, not deep trace analysis. -- **No real cron/daemon integration:** The loop exists in code but there is no `skillloop daemon` or automated cron registration. -- **uv.lock exists but pyproject.toml uses setuptools:** uv.lock was accidentally created in the tree; repo does not use uv. - -### 5.2 Non-goals (do NOT implement yet) - -- Do NOT auto-finetune before readiness, cost, evaluation, and promotion gates exist. -- Do NOT auto-promote models into Hermes. -- Do NOT write into global Hermes memory/skills without explicit approval. -- Do NOT duplicate Hermes memory, skills, session tracing, cron, gateway, tools, or subagent systems. -- Do NOT make users manually orchestrate the pipeline. - ---- - -## 6. Architecture Rules (Recurring Principles) - -1. **Hermes is the runtime; SkillLoop is the governor.** Never rebuild Hermes subsystems. -2. **Review-first.** No automatic global mutation. Approvals required for memory/skills/training/promotion. -3. **Provenance everywhere.** Every evaluation, proposal, dataset, and training artifact must carry source hashes. -4. **Conservative DPO.** Only export explicitly provided preference pairs. -5. **Deterministic evaluator first.** LLM evaluators only after cost tracking exists. -6. **Cost-conscious.** Estimate tokens, storage, and compute before scaling. Bounded defaults. -7. **Local-first.** No cloud dependency. State lives in `.skillloop/`. -8. **CLI is admin surface, not user product.** Normal users talk to Hermes. - ---- - -## 7. Repository Layout - -```text -skillloop/ - adapters/ Hermes state.db + generic JSONL ingestion - apply/ Review-approved filesystem exports - distill/ Memory and skill proposal generation - eval/ Evaluator registry, deterministic rubric, legacy, evidence - export/ SFT and DPO dataset exporters - review/ Proposal review queue helpers - benchmark.py Replay benchmark comparing evaluator versions - cli.py Command-line interface (admin/debug surface) - conditions.py Declarative run/done conditions - controller.py Autonomous controller tick - dataset.py Dataset split, manifest, provenance, and stats helpers - loop.py Outer loop scheduling - policy.py Policy schema and defaults - provenance.py Component provenance with SHA256 hashing - sanitize.py Secret redaction - schema.py Trace/Evaluation/Proposal dataclasses - store.py SQLite persistence - training_config.py Unsloth/TRL/Axolotl config generation -tests/ Pytest suite (54 tests) -docs/ - architecture.md System design - cli.md Command reference - safety.md Safety boundaries - trace-schema.md Data format -``` - ---- - -## 8. Session Hand-Off Protocol - -At the end of every session where progress was made: - -1. Update the "What is implemented" table (section 2.1). -2. Check off completed items in short-term/long-term sections. -3. Add any new issues to "Known Issues / Gaps" (section 5). -4. Update the "Last updated" date at the top. -5. Commit with message like `docs: update hand-off document`. - -At the start of every new session: - -1. Read this document top to bottom. -2. Check `git log --oneline -10` for any commits since the last update. -3. Run `python -m pytest -q` to confirm test baseline. -4. Pick up from the first unchecked item in section 3 (short-term). - ---- - -## 9. Related Hermes Skills - -These operational skills live in `~/.hermes/skills/system/` and complement -SkillLoop's loop engineering work. They handle the Hermes runtime side -(SkillLoop is the governor, Hermes is the runtime — Rule #1). - -| Skill | Path | Purpose | -|-------|------|---------| -| `pre-loop-checklist` | `system/pre-loop-checklist/` | 4-condition test + 30-second check before creating cron jobs | -| `cron-job-workflows` | `system/cron-job-workflows/` | Cron patterns + STATE.md + Ralph Wiggum detection + Red Flags | -| `goal-loop` | `system/goal-loop/` | /goal primitive: cron jobs that run until objective condition met | - -See `docs/analysis/loop-engineering-analysis.md` section 8 for full details on -what each skill contains and how they were derived from the loop engineering -framework. From f7bf18cd613d18d843c4b2c72aeb0f5d40f098a0 Mon Sep 17 00:00:00 2001 From: Hawthorn Date: Sun, 14 Jun 2026 03:27:21 +0530 Subject: [PATCH 5/6] feat: add setup status and controller history --- README.md | 5 ++ docs/cli.md | 33 ++++++++ skillloop/cli.py | 158 +++++++++++++++++++++++++++++++++++++++ skillloop/controller.py | 4 +- skillloop/store.py | 49 ++++++++++++ tests/test_cli.py | 24 ++++++ tests/test_controller.py | 3 + 7 files changed, 275 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cf2cdc0..60124cb 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,8 @@ skillloop --path . ingest generic examples/traces/simple_trace.jsonl ```text skillloop --path init +skillloop --path setup --connect hermes [--start] [--auto-export] +skillloop --path status [--json] skillloop --path ingest generic skillloop --path ingest hermes skillloop --path ingest hermes-db --latest [--db-path ~/.hermes/state.db] @@ -101,6 +103,9 @@ skillloop --path export sft --out [--min-score N] [--split skillloop --path export dpo --out [--min-score N] [--splits train=0.8,validation=0.1,test=0.1] [--manifest-out manifest.json] skillloop --path benchmark [--baseline rubric_legacy] [--candidates rubric] [--out benchmark.json] skillloop --path training-config trl|unsloth|axolotl --dataset-manifest manifest.json --base-model --output-dir --config-dir +skillloop --path controller run +skillloop --path controller history [--limit N] +skillloop --path controller show ``` ## Clean export boundary diff --git a/docs/cli.md b/docs/cli.md index e95119b..01874d9 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -26,6 +26,26 @@ Creates: .skillloop/skillloop.db ``` +## `setup` + +Configures SkillLoop as a local sidecar for Hermes by writing `.skillloop/policy.json` with `hermes-db` ingestion. With `--start`, it immediately runs one controller tick. + +```bash +skillloop --path . setup --connect hermes --start +skillloop --path . setup --connect hermes --db-path ~/.hermes/state.db --max-sessions 20 --auto-export +``` + +This is read-only against Hermes `state.db`; SkillLoop writes only under the selected `--path` root. + +## `status` + +Shows configured policy path, trace/evaluation/proposal counts, and the latest controller run. + +```bash +skillloop --path . status +skillloop --path . status --json +``` + ## `ingest generic` Ingests a generic JSONL trace. @@ -208,6 +228,19 @@ configs/unsloth/unsloth_sft_skeleton.py configs/axolotl/axolotl_config.yml ``` +## `controller run/history/show` + +Runs the autonomous controller once using `.skillloop/policy.json` if present, and inspects prior run reports stored in SQLite. + +```bash +skillloop --path . controller run +skillloop --path . controller history +skillloop --path . controller history --limit 5 +skillloop --path . controller show +``` + +Controller run reports are also mirrored as JSON under `.skillloop/controller_runs/` for easy inspection. + ## Full smoke test ```bash diff --git a/skillloop/cli.py b/skillloop/cli.py index be31f3d..1ecf162 100644 --- a/skillloop/cli.py +++ b/skillloop/cli.py @@ -9,6 +9,7 @@ from skillloop.adapters.hermes import load_hermes_export, load_hermes_state_db from skillloop.apply.filesystem import export_approved from skillloop.benchmark import replay_benchmark, write_benchmark_report +from skillloop.controller import controller_tick from skillloop.dataset import build_manifest, parse_split_spec, split_records, write_jsonl, write_manifest from skillloop.distill.memory import propose_memory_updates from skillloop.distill.skills import propose_skill_updates @@ -17,6 +18,7 @@ from skillloop.export.sft import export_sft_records from skillloop.conditions import LoopCondition from skillloop.loop import LoopSchedule, load_schedule, proposals_with_provenance, run_outer_loop, save_schedule, tick +from skillloop.policy import DatasetPolicy, IngestionPolicy, SkillLoopPolicy from skillloop.schema import AgentTrace, Evaluation, Proposal from skillloop.store import SkillLoopStore from skillloop.training_config import TrainingConfigRequest, generate_training_config @@ -26,6 +28,21 @@ def _store(args: argparse.Namespace) -> SkillLoopStore: return SkillLoopStore(Path(args.path)) +def _policy_path(store: SkillLoopStore) -> Path: + return store.state_dir / "policy.json" + + +def _load_policy(store: SkillLoopStore) -> SkillLoopPolicy: + path = _policy_path(store) + if not path.exists(): + return SkillLoopPolicy.default() + return SkillLoopPolicy.load(path) + + +def _format_count(label: str, count: int) -> str: + return f"{label}: {count}" + + def _resolve_trace(store: SkillLoopStore, trace_ref: str) -> AgentTrace: traces = store.list_traces() if trace_ref == "latest": @@ -54,6 +71,86 @@ def cmd_init(args: argparse.Namespace) -> int: return 0 +def cmd_setup(args: argparse.Namespace) -> int: + store = _store(args) + store.init() + if args.connect != "hermes": + raise SystemExit(f"Unsupported setup connector: {args.connect}") + hermes_db = Path(args.db_path).expanduser().resolve() + if not hermes_db.exists(): + raise SystemExit(f"Hermes state database not found: {hermes_db}") + policy = SkillLoopPolicy.default() + policy.ingestion = IngestionPolicy( + enabled=True, + adapter="hermes-db", + hermes_db_path=str(hermes_db), + latest=False, + max_sessions=args.max_sessions, + ) + policy.dataset = DatasetPolicy(enabled=args.auto_export, kind="sft", out=args.dataset_out, min_score=args.min_score) + policy.evaluation.min_score = args.min_score + policy.evaluation.condition = LoopCondition(score_gte=args.min_score) + policy_path = policy.save(_policy_path(store)) + print(f"Wrote policy to {policy_path}") + if args.start: + report = controller_tick(store, policy) + print(f"Ran controller tick {report.id}: {report.summary}") + return 0 + + +def cmd_status(args: argparse.Namespace) -> int: + store = _store(args) + store.init() + policy_path = _policy_path(store) + traces = store.list_traces() + evaluations = store.list_evaluations() + pending = store.list_proposals(status="pending") + runs = store.list_controller_runs(limit=1) + policy = _load_policy(store) + dataset_manifest = (store.root / policy.dataset.out).resolve().with_suffix(Path(policy.dataset.out).suffix + ".manifest.json") + dataset_stats = None + if dataset_manifest.exists(): + manifest = json.loads(dataset_manifest.read_text(encoding="utf-8")) + dataset_stats = { + "manifest": str(dataset_manifest), + "records": manifest.get("records", 0), + "estimated_tokens": manifest.get("estimated_tokens", 0), + } + status = { + "root": str(store.root), + "state_dir": str(store.state_dir), + "policy": str(policy_path) if policy_path.exists() else None, + "traces": len(traces), + "evaluations": len(evaluations), + "pending_proposals": len(pending), + "dataset": dataset_stats, + "last_controller_run": runs[0] if runs else None, + } + if args.json: + print(json.dumps(status, indent=2, ensure_ascii=False)) + else: + print(f"SkillLoop status for {store.root}") + print(f"state: {store.state_dir}") + print(f"policy: {policy_path if policy_path.exists() else 'not configured'}") + print(_format_count("traces", len(traces))) + print(_format_count("evaluations", len(evaluations))) + print(_format_count("pending proposals", len(pending))) + if dataset_stats: + print( + f"dataset: records={dataset_stats['records']} " + f"estimated_tokens={dataset_stats['estimated_tokens']} manifest={dataset_stats['manifest']}" + ) + else: + print("dataset: none") + if runs: + run = runs[0] + summary = run.get("summary", {}) + print(f"last controller run: {run.get('id')} finished={run.get('finished_at')} errors={summary.get('errors')}") + else: + print("last controller run: none") + return 0 + + def cmd_ingest(args: argparse.Namespace) -> int: store = _store(args) if args.adapter == "generic": @@ -346,6 +443,42 @@ def cmd_loop_tick(args: argparse.Namespace) -> int: return 0 +def cmd_controller_run(args: argparse.Namespace) -> int: + store = _store(args) + policy = _load_policy(store) + report = controller_tick(store, policy) + print(json.dumps(report.to_dict(), indent=2, ensure_ascii=False)) + return 0 + + +def cmd_controller_history(args: argparse.Namespace) -> int: + store = _store(args) + runs = store.list_controller_runs(limit=args.limit) + if not runs: + print("No controller runs found.") + return 0 + for run in runs: + summary = run.get("summary", {}) + print( + f"{run.get('id')}\t{run.get('finished_at') or run.get('started_at')}\t" + f"errors={summary.get('errors', 0)}\t" + f"traces={summary.get('traces_seen', 0)}\t" + f"evaluated={summary.get('traces_evaluated', 0)}\t" + f"review={summary.get('requires_review', 0)}" + ) + return 0 + + +def cmd_controller_show(args: argparse.Namespace) -> int: + store = _store(args) + try: + run = store.get_controller_run(args.run_id) + except KeyError as exc: + raise SystemExit(str(exc)) from exc + print(json.dumps(run, indent=2, ensure_ascii=False)) + return 0 + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="SkillLoop: clean learning/export layer for agent traces") parser.add_argument("--path", default=".", help="Project root for .skillloop state (default: current directory)") @@ -354,6 +487,20 @@ def build_parser() -> argparse.ArgumentParser: p_init = sub.add_parser("init", help="Initialize local .skillloop state") p_init.set_defaults(func=cmd_init) + p_setup = sub.add_parser("setup", help="Configure SkillLoop as a local sidecar") + p_setup.add_argument("--connect", choices=["hermes"], required=True, help="Runtime to connect (currently: hermes)") + p_setup.add_argument("--start", action="store_true", help="Run one controller tick immediately after writing policy") + p_setup.add_argument("--db-path", default=str(Path.home() / ".hermes" / "state.db"), help="Hermes state.db path") + p_setup.add_argument("--max-sessions", type=int, default=20, help="Maximum Hermes sessions to ingest per tick") + p_setup.add_argument("--min-score", type=int, default=70, help="Evaluation/dataset score gate") + p_setup.add_argument("--auto-export", action="store_true", help="Enable controller-managed SFT export") + p_setup.add_argument("--dataset-out", default="data/sft.jsonl", help="Controller-managed SFT output path") + p_setup.set_defaults(func=cmd_setup) + + p_status = sub.add_parser("status", help="Show current SkillLoop state") + p_status.add_argument("--json", action="store_true", help="Emit machine-readable JSON") + p_status.set_defaults(func=cmd_status) + p_ingest = sub.add_parser("ingest", help="Ingest a trace") p_ingest.add_argument("adapter", choices=["generic", "hermes", "hermes-db"]) p_ingest.add_argument("input", nargs="?", help="Input JSONL/JSON path for generic or hermes adapters") @@ -463,6 +610,17 @@ def build_parser() -> argparse.ArgumentParser: p_loop_tick.add_argument("--force", action="store_true", help="Run even if the schedule is not due") p_loop_tick.set_defaults(func=cmd_loop_tick) + p_controller = sub.add_parser("controller", help="Run and inspect autonomous controller ticks") + controller_sub = p_controller.add_subparsers(dest="controller_command", required=True) + p_controller_run = controller_sub.add_parser("run", help="Run one controller tick using .skillloop/policy.json if present") + p_controller_run.set_defaults(func=cmd_controller_run) + p_controller_history = controller_sub.add_parser("history", help="List stored controller run summaries") + p_controller_history.add_argument("--limit", type=int, default=20) + p_controller_history.set_defaults(func=cmd_controller_history) + p_controller_show = controller_sub.add_parser("show", help="Show a stored controller run by full id or unique prefix") + p_controller_show.add_argument("run_id") + p_controller_show.set_defaults(func=cmd_controller_show) + return parser diff --git a/skillloop/controller.py b/skillloop/controller.py index 0114fad..ab3d677 100644 --- a/skillloop/controller.py +++ b/skillloop/controller.py @@ -53,7 +53,9 @@ def save_controller_report(store: SkillLoopStore, report: ControllerRunReport) - report.finish() out = controller_runs_dir(store) / f"{report.id}.json" out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(json.dumps(report.to_dict(), indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + payload = report.to_dict() + out.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + store.save_controller_run(payload) return out diff --git a/skillloop/store.py b/skillloop/store.py index 35b5b3c..2f86265 100644 --- a/skillloop/store.py +++ b/skillloop/store.py @@ -51,6 +51,16 @@ def init(self) -> None: ) """ ) + conn.execute( + """ + CREATE TABLE IF NOT EXISTS controller_runs ( + id TEXT PRIMARY KEY, + started_at TEXT NOT NULL, + finished_at TEXT, + payload TEXT NOT NULL + ) + """ + ) def _connect(self) -> sqlite3.Connection: return sqlite3.connect(self.db_path) @@ -173,3 +183,42 @@ def get_proposal(self, proposal_id: str) -> Proposal: if row is None: raise KeyError(f"proposal not found: {proposal_id}") return Proposal.from_dict(json.loads(row[0])) + + def save_controller_run(self, report: dict) -> str: + self.init() + run_id = str(report["id"]) + payload = json.dumps(report, ensure_ascii=False) + with self._connect() as conn: + conn.execute( + "INSERT OR REPLACE INTO controller_runs (id, started_at, finished_at, payload) VALUES (?, ?, ?, ?)", + (run_id, str(report.get("started_at") or ""), report.get("finished_at"), payload), + ) + return run_id + + def list_controller_runs(self, limit: int | None = None) -> list[dict]: + self.init() + query = "SELECT payload FROM controller_runs ORDER BY started_at DESC" + args: tuple[int, ...] = () + if limit is not None: + query += " LIMIT ?" + args = (int(limit),) + with self._connect() as conn: + rows = conn.execute(query, args).fetchall() + return [json.loads(row[0]) for row in rows] + + def get_controller_run(self, run_id: str) -> dict: + self.init() + with self._connect() as conn: + row = conn.execute("SELECT payload FROM controller_runs WHERE id = ?", (run_id,)).fetchone() + if row is None: + rows = conn.execute( + "SELECT payload FROM controller_runs WHERE id LIKE ? ORDER BY started_at DESC", + (f"{run_id}%",), + ).fetchall() + if len(rows) == 1: + row = rows[0] + elif len(rows) > 1: + raise KeyError(f"controller run id prefix is ambiguous: {run_id}") + if row is None: + raise KeyError(f"controller run not found: {run_id}") + return json.loads(row[0]) diff --git a/tests/test_cli.py b/tests/test_cli.py index c5bbdca..600e67d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -84,6 +84,30 @@ def test_cli_ingests_hermes_state_db_latest(tmp_path, capsys): assert "hermes_state_db" in output +def test_cli_setup_status_and_controller_history(tmp_path, capsys): + db = tmp_path / "state.db" + conn = sqlite3.connect(db) + conn.execute("CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, title TEXT, started_at REAL, ended_at REAL, message_count INTEGER)") + conn.execute("CREATE TABLE messages (id INTEGER PRIMARY KEY, session_id TEXT, role TEXT, content TEXT, tool_calls TEXT, timestamp REAL, active INTEGER)") + conn.execute("INSERT INTO sessions VALUES ('s1', 'cli', 'Session', 1.0, NULL, 2)") + conn.execute("INSERT INTO messages (session_id, role, content, tool_calls, timestamp, active) VALUES ('s1', 'user', 'Remember concise status.', NULL, 1.1, 1)") + conn.execute("INSERT INTO messages (session_id, role, content, tool_calls, timestamp, active) VALUES ('s1', 'assistant', 'Done. Verified with tests.', NULL, 1.2, 1)") + conn.commit() + conn.close() + + assert main(["--path", str(tmp_path), "setup", "--connect", "hermes", "--db-path", str(db), "--start", "--auto-export"]) == 0 + assert (tmp_path / ".skillloop" / "policy.json").exists() + assert main(["--path", str(tmp_path), "status"]) == 0 + assert main(["--path", str(tmp_path), "controller", "history"]) == 0 + + output = capsys.readouterr().out + assert "Wrote policy" in output + assert "Ran controller tick" in output + assert "pending proposals" in output + assert "dataset: records=" in output + assert "errors=0" in output + + def test_cli_export_writes_split_files_and_manifest(tmp_path): for index in range(4): trace_path = tmp_path / f"trace-{index}.jsonl" diff --git a/tests/test_controller.py b/tests/test_controller.py index 2185254..aa4ce07 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -38,6 +38,9 @@ def test_controller_tick_ingests_evaluates_exports_and_records_report(tmp_path): assert len(run_files) == 1 saved = json.loads(run_files[0].read_text()) assert saved["id"] == report.id + stored_runs = store.list_controller_runs() + assert [run["id"] for run in stored_runs] == [report.id] + assert store.get_controller_run(report.id[:8])["id"] == report.id assert saved["summary"]["requires_review"] >= 1 From 1e5bfc9db130d222d297decc1c89c9198efd496d Mon Sep 17 00:00:00 2001 From: Hawthorn Date: Sun, 14 Jun 2026 03:44:12 +0530 Subject: [PATCH 6/6] fix: address CodeRabbit review findings --- references/hermes-skills/goal-loop/SKILL.md | 22 ++++++------ skillloop/cli.py | 37 +++++++++++++++------ skillloop/store.py | 11 ++++-- tests/test_cli.py | 32 ++++++++++++++++++ tests/test_store.py | 19 +++++++++++ 5 files changed, 97 insertions(+), 24 deletions(-) diff --git a/references/hermes-skills/goal-loop/SKILL.md b/references/hermes-skills/goal-loop/SKILL.md index 72dfcff..d76c659 100644 --- a/references/hermes-skills/goal-loop/SKILL.md +++ b/references/hermes-skills/goal-loop/SKILL.md @@ -119,7 +119,9 @@ checklist against the task. If it doesn't pass, do NOT build a goal loop. ### Step 2: Initialize STATE.md Create a STATE.md in the project root (or .hermes/ directory) using the -template from cron-job-workflows skill. The first run reads this to resume. +template from cron-job-workflows skill. After creating the cron job, persist +the exact returned `job_id` in STATE.md so the loop can self-cancel safely. +The first run reads this to resume. ### Step 3: Create the cron job @@ -173,9 +175,9 @@ Dispatch a checker subagent via delegate_task: - Write final status to STATE.md under "## Stop conditions met" - Final response: report success with specific evidence (test output, commit hash, etc.) -- IMPORTANT: After reporting success, call cronjob(action='list') to find - this job, then cronjob(action='remove', job_id=THE_JOB_ID) to self-cancel. - Search for the job by name or look for the one with your prompt text. +- IMPORTANT: The job's STATE.md must contain the exact `job_id` returned + when the loop was created. After reporting success, read that stored ID + and call cronjob(action='remove', job_id=STORED_JOB_ID) to self-cancel. - Do NOT continue running after goal is met. ### 4. If goal is NOT met: @@ -218,7 +220,7 @@ Dispatch a checker subagent via delegate_task: ## Example: CI Triage Goal Loop ```python -cronjob( +job = cronjob( action='create', name='auth-quality-goal-loop', schedule='30m', @@ -238,10 +240,11 @@ Read STATE.md at project root. Update it at the end of every run. [Full template follows with checked values...] """, - workdir='/Users/raghav/myproject', + workdir='', skills=['pre-loop-checklist', 'cron-job-workflows'], enabled_toolsets=['terminal', 'file', 'delegation', 'cronjob'], ) +# Record job['job_id'] in STATE.md before the first tick runs. ``` ## Self-Cancellation @@ -249,11 +252,8 @@ Read STATE.md at project root. Update it at the end of every run. When the goal is met, the cron agent calls: ```python -# Find this job -jobs = cronjob(action='list') -# Find your job in the list (match by name or prompt content) -# Remove it -cronjob(action='remove', job_id='THE_JOB_ID') +# Read the exact job_id persisted in STATE.md when the loop was created. +cronjob(action='remove', job_id=stored_job_id) ``` The `cronjob` tool is enabled in the `enabled_toolsets` so the agent can diff --git a/skillloop/cli.py b/skillloop/cli.py index 1ecf162..8b33689 100644 --- a/skillloop/cli.py +++ b/skillloop/cli.py @@ -79,6 +79,10 @@ def cmd_setup(args: argparse.Namespace) -> int: hermes_db = Path(args.db_path).expanduser().resolve() if not hermes_db.exists(): raise SystemExit(f"Hermes state database not found: {hermes_db}") + if args.max_sessions <= 0: + raise SystemExit("--max-sessions must be positive") + if not 0 <= args.min_score <= 100: + raise SystemExit("--min-score must be between 0 and 100") policy = SkillLoopPolicy.default() policy.ingestion = IngestionPolicy( enabled=True, @@ -110,12 +114,20 @@ def cmd_status(args: argparse.Namespace) -> int: dataset_manifest = (store.root / policy.dataset.out).resolve().with_suffix(Path(policy.dataset.out).suffix + ".manifest.json") dataset_stats = None if dataset_manifest.exists(): - manifest = json.loads(dataset_manifest.read_text(encoding="utf-8")) - dataset_stats = { - "manifest": str(dataset_manifest), - "records": manifest.get("records", 0), - "estimated_tokens": manifest.get("estimated_tokens", 0), - } + try: + manifest = json.loads(dataset_manifest.read_text(encoding="utf-8")) + dataset_stats = { + "manifest": str(dataset_manifest), + "records": manifest.get("records", 0), + "estimated_tokens": manifest.get("estimated_tokens", 0), + } + except (json.JSONDecodeError, OSError) as exc: + dataset_stats = { + "manifest": str(dataset_manifest), + "records": 0, + "estimated_tokens": 0, + "error": f"failed to load manifest: {exc}", + } status = { "root": str(store.root), "state_dir": str(store.state_dir), @@ -136,10 +148,13 @@ def cmd_status(args: argparse.Namespace) -> int: print(_format_count("evaluations", len(evaluations))) print(_format_count("pending proposals", len(pending))) if dataset_stats: - print( - f"dataset: records={dataset_stats['records']} " - f"estimated_tokens={dataset_stats['estimated_tokens']} manifest={dataset_stats['manifest']}" - ) + if dataset_stats.get("error"): + print(f"dataset: error={dataset_stats['error']} manifest={dataset_stats['manifest']}") + else: + print( + f"dataset: records={dataset_stats['records']} " + f"estimated_tokens={dataset_stats['estimated_tokens']} manifest={dataset_stats['manifest']}" + ) else: print("dataset: none") if runs: @@ -453,6 +468,8 @@ def cmd_controller_run(args: argparse.Namespace) -> int: def cmd_controller_history(args: argparse.Namespace) -> int: store = _store(args) + if args.limit is not None and args.limit <= 0: + raise SystemExit("--limit must be positive") runs = store.list_controller_runs(limit=args.limit) if not runs: print("No controller runs found.") diff --git a/skillloop/store.py b/skillloop/store.py index 2f86265..ad121db 100644 --- a/skillloop/store.py +++ b/skillloop/store.py @@ -186,12 +186,16 @@ def get_proposal(self, proposal_id: str) -> Proposal: def save_controller_run(self, report: dict) -> str: self.init() + if not report.get("id"): + raise ValueError("report must have non-empty 'id'") + if not report.get("started_at"): + raise ValueError("report must have non-empty 'started_at'") run_id = str(report["id"]) payload = json.dumps(report, ensure_ascii=False) with self._connect() as conn: conn.execute( "INSERT OR REPLACE INTO controller_runs (id, started_at, finished_at, payload) VALUES (?, ?, ?, ?)", - (run_id, str(report.get("started_at") or ""), report.get("finished_at"), payload), + (run_id, str(report["started_at"]), report.get("finished_at"), payload), ) return run_id @@ -211,9 +215,10 @@ def get_controller_run(self, run_id: str) -> dict: with self._connect() as conn: row = conn.execute("SELECT payload FROM controller_runs WHERE id = ?", (run_id,)).fetchone() if row is None: + escaped_id = run_id.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") rows = conn.execute( - "SELECT payload FROM controller_runs WHERE id LIKE ? ORDER BY started_at DESC", - (f"{run_id}%",), + "SELECT payload FROM controller_runs WHERE id LIKE ? ESCAPE '\\' ORDER BY started_at DESC", + (f"{escaped_id}%",), ).fetchall() if len(rows) == 1: row = rows[0] diff --git a/tests/test_cli.py b/tests/test_cli.py index 600e67d..e3c77eb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,8 @@ import json import sqlite3 +import pytest + from skillloop.cli import main @@ -108,6 +110,36 @@ def test_cli_setup_status_and_controller_history(tmp_path, capsys): assert "errors=0" in output +def test_cli_setup_rejects_invalid_bounds(tmp_path): + db = tmp_path / "state.db" + conn = sqlite3.connect(db) + conn.execute("CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, title TEXT, started_at REAL, ended_at REAL, message_count INTEGER)") + conn.execute("CREATE TABLE messages (id INTEGER PRIMARY KEY, session_id TEXT, role TEXT, content TEXT, tool_calls TEXT, timestamp REAL, active INTEGER)") + conn.commit() + conn.close() + + with pytest.raises(SystemExit, match="--max-sessions must be positive"): + main(["--path", str(tmp_path), "setup", "--connect", "hermes", "--db-path", str(db), "--max-sessions", "0"]) + with pytest.raises(SystemExit, match="--min-score must be between 0 and 100"): + main(["--path", str(tmp_path), "setup", "--connect", "hermes", "--db-path", str(db), "--min-score", "101"]) + + +def test_cli_status_handles_corrupt_dataset_manifest(tmp_path, capsys): + manifest = tmp_path / "data" / "sft.jsonl.manifest.json" + manifest.parent.mkdir(parents=True) + manifest.write_text("{not json", encoding="utf-8") + + assert main(["--path", str(tmp_path), "status"]) == 0 + + output = capsys.readouterr().out + assert "dataset: error=failed to load manifest" in output + + +def test_cli_controller_history_rejects_non_positive_limit(tmp_path): + with pytest.raises(SystemExit, match="--limit must be positive"): + main(["--path", str(tmp_path), "controller", "history", "--limit", "0"]) + + def test_cli_export_writes_split_files_and_manifest(tmp_path): for index in range(4): trace_path = tmp_path / f"trace-{index}.jsonl" diff --git a/tests/test_store.py b/tests/test_store.py index 7f6e4d5..6c213d7 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -1,3 +1,5 @@ +import pytest + from skillloop.adapters.generic_jsonl import load_generic_jsonl from skillloop.schema import AgentMessage, AgentTrace, Evaluation, sha256_text from skillloop.store import SkillLoopStore @@ -58,3 +60,20 @@ def test_store_preserves_raw_trace_and_hashes(tmp_path): assert loaded.raw_trace_sha256 == sha256_text(raw_text) assert loaded.normalized_trace_sha256 == loaded.compute_normalized_sha256() assert store.read_preserved_raw_trace(loaded) == raw_text + + +def test_store_validates_controller_run_report_contract(tmp_path): + store = SkillLoopStore(tmp_path) + + with pytest.raises(ValueError, match="non-empty 'id'"): + store.save_controller_run({"started_at": "2026-01-01T00:00:00+00:00"}) + with pytest.raises(ValueError, match="non-empty 'started_at'"): + store.save_controller_run({"id": "run-1"}) + + +def test_store_controller_run_prefix_search_escapes_like_wildcards(tmp_path): + store = SkillLoopStore(tmp_path) + store.save_controller_run({"id": "a_b", "started_at": "2026-01-01T00:00:00+00:00"}) + store.save_controller_run({"id": "a1b", "started_at": "2026-01-02T00:00:00+00:00"}) + + assert store.get_controller_run("a_")["id"] == "a_b"