From 5eed141bccdf71ee5e6d474377a82424ed6f94f9 Mon Sep 17 00:00:00 2001 From: NoisemakerJon <139656120+Noisemaker111@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:46:29 -0400 Subject: [PATCH 1/3] Establish agents/main integration workflow (#5) * Establish checked agents integration and human-approved main * Record pending main approval enforcement accurately --- .github/workflows/agent-checks.yml | 29 +++++++++++++ AGENTS.md | 64 +++++++++++++++++++++++++++ scripts/integrate_agents.py | 69 ++++++++++++++++++++++++++++++ 3 files changed, 162 insertions(+) create mode 100644 .github/workflows/agent-checks.yml create mode 100644 AGENTS.md create mode 100644 scripts/integrate_agents.py diff --git a/.github/workflows/agent-checks.yml b/.github/workflows/agent-checks.yml new file mode 100644 index 0000000..66d5d91 --- /dev/null +++ b/.github/workflows/agent-checks.yml @@ -0,0 +1,29 @@ +name: Agent checks +on: + pull_request: + branches: [agents, main] + push: + branches: [agents] +permissions: + contents: read +concurrency: + group: agent-checks-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + core: + name: Windows core + runs-on: windows-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: '3.12' + - name: Validate syntax + run: python -m compileall -q scripts experiments/command_specialist + - name: Path binding and saved evidence invariants + run: python -m unittest discover -s experiments/command_specialist -p test_bindings.py -v + - name: Native and PowerShell contract invariants + run: python -m unittest discover -s experiments/command_specialist -p test_contract.py -v diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..6477e37 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,64 @@ +# Agents and main + +`agents` is the development integration branch. `main` is the stable installation +branch. Jon authorized agents-branch automation on September 10, 2026: agents may +prepare owned changes, verify them, open ready PRs against `agents`, and merge them +when the gates below pass. This replaces per-PR human merge requests for `agents`. +Merging into `main`, publication, deployments, model/weight changes, and production +data changes still require explicit approval for the concrete batch. + +Start from fetched `agents` in an owned worktree. Preserve other sessions and reuse +existing PRs for the same work. Run commands from the repository root with Python +3.12. Required checks are: + +``` +python -m compileall -q scripts experiments/command_specialist +python -m unittest discover -s experiments/command_specialist -p test_bindings.py -v +python -m unittest discover -s experiments/command_specialist -p test_contract.py -v +``` + +Also verify the affected user operation through the public CLI and reopen its saved +artifact. Hosted CI has no private model, GPU, or transcripts; it supplements local +product verification. Do not run live model benchmarks concurrently or change the +shared Ollama service. Retain `shell-specialist-pilot` and its existing adapter. + +`.github/workflows/agent-checks.yml` runs the required `Windows core` check on PRs +into both branches and pushes to `agents`. It runs on a hosted Windows runner with +read-only permissions and no persisted checkout credentials. PR code never receives +a privileged merge token. `agents` requires this GitHub Actions check and an up-to-date PR, prevents force +pushes/deletion, and applies protection to administrators. No approving review is +required for `agents`. Main protection is pending Jon's choice of GitHub review +enforcement versus explicit chat approval; do not claim it is server-enforced. + +The local coordinator is `scripts/integrate_agents.py`. After reviewing the diff, +verifying the current head locally, and waiting for CI, run: + +``` +python scripts/integrate_agents.py --pr --verified-head +``` + +It accepts only ready same-repository PRs authored by Noisemaker111 targeting +`agents`; checks the exact head, workflow identity/result, and mergeability; and +merges with GitHub's expected-head condition. It refuses `main`. `--check-only` +checks eligibility without merging. Invoke it from a trusted checkout; passing a +head is the agent's attestation of actual local verification, not proof supplied +by CI. This is agent-operated automation, not an installed unattended scheduler. +The agent continues through merge and merged-revision verification without asking +Jon for routine agents integration approval. + +This repository is a local skill/CLI, with no hosted frontend, backend, database, +queue, production service, or deployment trigger configured here. Dev validation +runs the merged `agents` revision from an isolated worktree with fresh fixtures and +ignored `work/` artifacts. Stable users continue to install `main`; existing local +installations and sessions are not repointed. No automatic host integration is +claimed. Source revision, loaded CLI source, and saved artifact must agree when +reporting dev verification. + +For stable promotion, branch a frozen release candidate from a verified agents +revision and open a ready PR into `main`. Include the full diff, user-facing notes, +verification evidence, and rollback target (the prior main revision). Later agents +changes must not join that candidate silently. Ask Jon to approve that concrete +batch before merging. The agents GitHub PR/check gates are enforced, but this account is shared +by Jon and agents: human authorization is a workflow rule, not an independently +verified GitHub reviewer identity. The coordinator cannot merge stable. No package +or runtime deployment is implied by either merge. diff --git a/scripts/integrate_agents.py b/scripts/integrate_agents.py new file mode 100644 index 0000000..1a25f41 --- /dev/null +++ b/scripts/integrate_agents.py @@ -0,0 +1,69 @@ +"""Integrate a locally verified, CI-green owner PR into agents; never main. + +Run from a trusted agents checkout after inspecting the diff and completing the +real user-operation check. No PR code is downloaded or executed by this tool. +""" +import argparse +import json +import re +import subprocess + +REPO = "Noisemaker111/shell-forensics" +OWNER = "Noisemaker111" + + +def gh(*args): + result = subprocess.run(["gh", *args], check=True, capture_output=True, text=True, encoding="utf-8", timeout=60) + return json.loads(result.stdout) if result.stdout.strip() else None + + +def eligible(pr, verified_head): + if pr["baseRefName"] != "agents": + raise ValueError("Only agents PRs may be integrated; main needs Jon's explicit batch approval") + if pr["state"] != "OPEN" or pr["isDraft"] or pr["isCrossRepository"] or pr["author"]["login"].lower() != OWNER.lower(): + raise ValueError("Require an open, ready, same-repository owner PR") + if pr["headRefOid"] != verified_head: + raise ValueError("Head changed or was not locally verified; recheck the actual user operation") + if pr["mergeable"] != "MERGEABLE" or pr["mergeStateStatus"] != "CLEAN": + raise ValueError("PR must be up to date, mergeable, and pass branch protection") + checks = pr["statusCheckRollup"] + core = [c for c in checks if c.get("name") == "Windows core"] + if not core or any(c.get("status") != "COMPLETED" or c.get("conclusion") != "SUCCESS" for c in core): + raise ValueError("Windows core is missing, pending, or unsuccessful") + if any(c.get("conclusion") not in ("SUCCESS", "NEUTRAL", "SKIPPED") or c.get("status") != "COMPLETED" for c in checks): + raise ValueError("A check is incomplete or unsuccessful") + return core + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--pr", type=int, required=True) + parser.add_argument("--verified-head", required=True, help="Exact head already reviewed and verified through the product") + parser.add_argument("--check-only", action="store_true") + args = parser.parse_args() + fields = "state,isDraft,isCrossRepository,author,baseRefName,headRefOid,mergeable,mergeStateStatus,statusCheckRollup" + pr = gh("pr", "view", str(args.pr), "--repo", REPO, "--json", fields) + checks = eligible(pr, args.verified_head) + for check in checks: + match = re.fullmatch(r"https://github\.com/Noisemaker111/shell-forensics/actions/runs/(\d+)/job/\d+", check.get("detailsUrl", "")) + if not match: + raise ValueError("Required check is not the repository Actions job") + run = gh("api", f"repos/{REPO}/actions/runs/{match[1]}") + if (run["path"] != ".github/workflows/agent-checks.yml" or run["event"] != "pull_request" + or run["head_sha"] != args.verified_head or run["conclusion"] != "success"): + raise ValueError("Required workflow does not validate this PR head") + # Server-side strict checks and the expected-head condition close update races. + latest = gh("pr", "view", str(args.pr), "--repo", REPO, "--json", fields) + eligible(latest, args.verified_head) + if args.check_only: + print(json.dumps({"eligible": True, "pr": args.pr, "base": "agents"})) + return + subprocess.run(["gh", "pr", "merge", str(args.pr), "--repo", REPO, "--squash", "--match-head-commit", args.verified_head], check=True, timeout=60) + receipt = gh("pr", "view", str(args.pr), "--repo", REPO, "--json", "state,baseRefName,mergeCommit,url") + if receipt["state"] != "MERGED" or receipt["baseRefName"] != "agents": + raise RuntimeError("Merge receipt did not confirm agents integration") + print(json.dumps(receipt)) + + +if __name__ == "__main__": + main() From 0f5c7ea6ccac2cf51b039bf1a39e73a7dca8e6db Mon Sep 17 00:00:00 2001 From: NoisemakerJon <139656120+Noisemaker111@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:48:25 -0400 Subject: [PATCH 2/3] Expose handoff runtime limits and preserve selection evidence (#3) --- experiments/command_specialist/README.md | 34 +++++ .../command_specialist/RUNTIME_RESULTS.md | 134 ++++++++++++++++++ experiments/command_specialist/benchmark.py | 19 ++- experiments/command_specialist/run.py | 71 ++++++++-- .../command_specialist/test_bindings.py | 10 +- 5 files changed, 251 insertions(+), 17 deletions(-) create mode 100644 experiments/command_specialist/RUNTIME_RESULTS.md diff --git a/experiments/command_specialist/README.md b/experiments/command_specialist/README.md index cac4cec..8fc9cc1 100644 --- a/experiments/command_specialist/README.md +++ b/experiments/command_specialist/README.md @@ -45,6 +45,40 @@ A caller can hand off a UTF-8 JSON task file: python experiments/command_specialist/run.py --root '' --task-file task.json --backend native ``` +Runtime limits are explicit CLI options and keyword arguments on `inspect_request`. +Defaults remain `--num-ctx 4096 --num-predict 160` for compatibility. To evaluate +more context/output capacity with the same model and adapter: + +```powershell +python experiments/command_specialist/run.py --root '' --task-file task.json --backend native --num-ctx 8192 --num-predict 2048 +``` + +These values reach **both** planning and evidence requests; Modelfile defaults do +not override them. 8192/2048 is an evaluation profile, not a universal capacity +recommendation or a speed claim. `--evidence-max-lines` (100), +`--evidence-max-chars` (8000), and `--packet-max-chars` (2000) independently bound +selection input and compact stdout/stderr. All limits must be positive integers. +The character window includes newline separators. It is not a tokenizer budget: +callers must allow room for their intent, numbered evidence, system instructions, +and generated output within the chosen context. Ollama may truncate overlong +prompts; this pilot does not prove arbitrary inputs fit from character counts. + +A line-limited selection is marked truncated; a character-overflow window skips +selection and asks the caller to inspect the raw artifact. Increasing model context +alone does not enlarge the evidence window. `fallback: "inspect_raw_result"` +identifies bounded or rejected output; it does not trigger an automatic retry or +host action. Raw stdout/stderr remain complete for the executed bounded operation +(the plan's requested line count is still part of that operation). + +Empty stdout deterministically returns empty evidence without a second model call. +For nonempty evidence, the artifact records the requested selection, model response, +selection timing, and returned packet. Planning output stopped at the generation +limit is rejected before execution and saved with its timing and raw response; +evidence output stopped at the limit falls back to the already saved raw result. +The artifact records effective runtime limits, execution time, prompt/decode token +counts and durations, and Ollama's stop reason. No model or service settings change. +See [runtime handoff evaluation](RUNTIME_RESULTS.md) for measured scope and limits. + Or use the Python `run.inspect_request(root, request, targets=...)` boundary. For a small CLI request, `--target 'build=logs/build.log' --request 'Read the last 5 lines of {{build}}.'` supplies the same binding. Existing unbound `--request` calls retain diff --git a/experiments/command_specialist/RUNTIME_RESULTS.md b/experiments/command_specialist/RUNTIME_RESULTS.md new file mode 100644 index 0000000..da0ba59 --- /dev/null +++ b/experiments/command_specialist/RUNTIME_RESULTS.md @@ -0,0 +1,134 @@ +# Public handoff runtime evaluation + +Measured September 10, 2026. The unchanged `shell-specialist-pilot` model is +Qwen2.5-Coder 1.5B Q4_K_M with its existing LoRA adapter. Current checks confirmed +Ollama 0.33.3, RTX 3070 8GB, Ryzen 9 3900X, and GPU execution. Shared Ollama service +configuration, weights, quantization, and adapter were not changed. No deployment +or host integration occurred. + +## Outcome + +The confirmed bug was unnecessary model selection on empty stdout. In the merged +handoff, both no-match cases generated 160 tokens of nonexistent line numbers, +stopped at the output limit, and fell back with invalid JSON. Deterministic empty +selection fixes those cases and removes the second inference call. Nonempty +evidence still uses the model and is returned verbatim. + +The public CLI and callable now expose context/output and evidence/packet window +limits. Defaults remain compatible at 4096/160; 8192/2048 is an opt-in evaluation +profile. Raw artifacts now retain evidence selection/timing, effective settings, +execution timing, stop reasons, and raw model responses. Output-limit planning +failures save diagnostic artifacts and execute nothing. Selection failure keeps +the pre-selection raw artifact. Character-window accounting includes newlines. + +## Final paired measurements + +Three runs used the same saved 16-case synthetic set across all treatments, +with fresh opaque filenames and two registered candidates per case. All five +operations were included, along with longer caller context, evidence selection, +two empty searches, a character-window overflow, compact-output truncation, and +an escaping target rejected before inference. PowerShell reference outputs were +prepared outside measurement. JSON correctness compares values; other operations +compare actual stdout and exit status. Evidence compares requested source lines +and verbatim text; explicit overflow fallback and invalid-input rejection have +separate expected outcomes. Exact plan text is not the correctness definition. + +Each treatment block ran a full-workload warmup and then the same measured cases. +Order was merged/compat/capacity, capacity/compat/merged, then merged/compat/capacity. +Blocks avoid forcing context reload on every case. These are paired by saved case, +with alternating treatment order, not randomized interleaving or independent samples. + +| Run | Source/profile | Checks | CLI median ms | CLI p95 ms | Mean generated tokens | Mean prompt ms | Mean decode ms | Weighted decode tokens/s | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| 1 | Merged PR2, 4096/160 | 14/16 | 289.6 | 1186.1 | 40.94 | 37.6 | 226.8 | 180.53 | +| 1 | Revised, 4096/160 | 16/16 | 278.6 | 450.3 | 20.94 | 29.9 | 116.1 | 180.31 | +| 1 | Revised, 8192/2048 | 16/16 | 292.7 | 418.8 | 20.94 | 30.4 | 119.3 | 175.57 | +| 2 | Merged PR2, 4096/160 | 14/16 | 289.4 | 1166.3 | 40.94 | 33.1 | 221.8 | 184.58 | +| 2 | Revised, 4096/160 | 16/16 | 287.4 | 438.1 | 20.94 | 29.2 | 119.1 | 175.78 | +| 2 | Revised, 8192/2048 | 16/16 | 283.9 | 419.2 | 20.94 | 28.5 | 117.7 | 177.94 | +| 3 | Merged PR2, 4096/160 | 14/16 | 295.7 | 1191.8 | 40.94 | 36.1 | 225.3 | 181.67 | +| 3 | Revised, 4096/160 | 16/16 | 298.6 | 463.3 | 20.94 | 32.7 | 125.4 | 166.96 | +| 3 | Revised, 8192/2048 | 16/16 | 296.7 | 482.3 | 20.94 | 29.5 | 120.5 | 173.76 | + +Whole-handoff time includes Python/CLI startup, task-file reading, binding, model +calls, validation, native execution, real raw-artifact writes, packet output, and +artifact reload. The loopback diagnostic proxy buffers requests/replies in memory; +trace serialization, report writes, and correctness scoring occur after the timer. +Proxy overhead remains included equally. Prompt/decode figures sum both model calls +per handoff, including rejected generations. Means include the pre-inference invalid +case as zero tokens/time. Weighted throughput is total tokens / total decode seconds, +not the arithmetic mean of per-request throughput. No decode improvement is claimed. + +Across 48 measured attempts per treatment, execution was correct on all 45 executable +cases and all three invalid cases were rejected. Evidence/fallback checks passed +42/48 for merged code and 48/48 for each revised profile. The six merged failures +were empty-search evidence selection, not wrong execution or filename resolution. +There were no observed execution/evidence regressions. Nonempty evidence selection +passed throughout this set. Each revised arm had six flagged truncations/fallbacks +(three compact-output windows, three oversized selection windows); merged had 12, +including its six selection failures. Fallback means the caller must inspect the +raw artifact; no automated retry or host fallback is installed. + +Pooled medians/p95 were 291.2/1186.1 ms merged, 288.0/438.1 ms revised compatibility, +and 290.1/418.8 ms revised capacity. Median differences are small and inconsistent +across runs. The lower tail latency and generation count come from removing the +failed empty-selection calls. With only 16 cases per run, nearest-rank p95 is the +maximum observation; do not generalize it to production. No >5% decode-throughput +win, general reliability claim, or whole-agent speed claim is supported. + +## Cache conditions and capacity checks + +The initial unchanged public-CLI smoke returned the correct execution and selected +error/summary lines; its raw artifact was reopened. It took 2379.4 ms including +2001.4 ms model load. This is a cold-load observation, not a warm baseline. + +The final paired table reports warmed repeated workloads. First traversal blocks +were retained separately: median 299.3 ms merged, 284.0 ms compatibility, and +299.2 ms capacity; largest load times were 2866.9, 5.9, and 2891.4 ms respectively. +These are not controlled cold-cache comparisons: context changes can reload the +runner, later arms reuse prompts, and fresh filenames become short references. +No shared-service unload/cache reset was performed. Novel real-world prompt and +cold-start performance remain unestablished. + +Separate final-source public-CLI probes (not pooled into the paired table): + +- 100 error lines: the 160-token selection cap was reached and flagged; raw stdout + survived. At 2048, all 100 source lines were selected verbatim using 296 tokens. +- Longer caller context: Ollama reported 5187 prompt tokens at context 8192; actual + execution and requested error/summary evidence were correct. +- A two-line evidence window omitted the third-line summary with `truncated: true` + and raw-result fallback. A five-character evidence window skipped selection. +- A five-character packet window returned the prefix, flagged truncation, and kept + complete raw stdout. A one-token planning cap exited nonzero, saved the raw model + reply/stop reason, and executed nothing. Zero context was rejected before inference. +- The final public CLI also passed through the actual PowerShell backend, with + readable bound paths, source lines 2/3, and the saved output reopened. + +These probes demonstrate particular capacities, not that arbitrary prompts fit. +Character windows are not token counts; the pilot does not detect all server-side +prompt truncation. The 100-line operation contract remains. Larger context alone +does not enlarge selection windows. Unexpected provider/network error recovery and +arbitrary Unicode/JSON equivalence beyond the existing pilot remain unproven. + +## Evidence and reproduction + +Local evidence remains under this task worktree in `work/runtime-final/`: +`evaluate.py`, immutable `source-before/` and `source-after/`, `manifest.json`, +saved tasks/fixtures and PowerShell references, all 288 warmup/measured records, +per-operation raw artifacts and buffered wire traces, `summary.json`, and nine +CLI probes under `probes/`. SHA-256 source/input checks passed after measurement; +the four executed production modules match the final worktree byte-for-byte. +The exploratory set remains separately in `work/runtime-evaluation/`. The final +set was generated after implementation was frozen and not used to tune the model +or code. Historical development/final-test corpora were never opened. + +To reproduce locally, copy the saved evaluator and both source snapshots into a +new empty result directory, then run `python /evaluate.py` from the +repository. It creates new filenames, freezes inputs, and performs all three runs. +The evaluator is local diagnostic evidence, not a retained product scenario suite. +Raw inputs, model traces, corpora, and weights are excluded from Git. + +Validation: all 10 focused binding/contract invariant tests passed, including actual +native and PowerShell execution, path confinement, revalidation, literal quoting, +configured settings on both model calls, and persisted faithful evidence. This does +not validate automatic OpenCode2 integration; that remains a subsequent bounded step. diff --git a/experiments/command_specialist/benchmark.py b/experiments/command_specialist/benchmark.py index 00f1cfb..338fe31 100644 --- a/experiments/command_specialist/benchmark.py +++ b/experiments/command_specialist/benchmark.py @@ -22,17 +22,30 @@ def post(base: str, path: str, body: dict) -> dict: return json.load(response) -def predict(base: str, model: str, case: dict) -> tuple[dict, dict]: +class PredictionError(ValueError): + def __init__(self, message, timing): + super().__init__(message) + self.timing = timing + + +def predict(base: str, model: str, case: dict, *, num_ctx=4096, num_predict=160) -> tuple[dict, dict]: start = time.perf_counter() response = post(base, "/api/chat", { "model": model, "messages": messages(case), "stream": False, "format": prediction_schema(case), "think": False, "keep_alive": "5m", - "options": {"temperature": 0, "seed": 20260909, "num_ctx": 4096, "num_predict": 160}, + "options": {"temperature": 0, "seed": 20260909, "num_ctx": num_ctx, "num_predict": num_predict}, }) timing = {"inference_wall_ms": (time.perf_counter() - start) * 1000} timing.update({key: response.get(key) for key in ["total_duration", "load_duration", "prompt_eval_count", "prompt_eval_duration", "eval_count", "eval_duration"]}) - return json.loads(response["message"]["content"]), timing + timing["done_reason"] = response.get("done_reason") + timing["raw_response"] = response["message"]["content"] + if response.get("done_reason") == "length": + raise PredictionError("Model output limit reached; no prediction accepted", timing) + try: + return json.loads(response["message"]["content"]), timing + except (ValueError, TypeError) as error: + raise PredictionError(str(error), timing) from error def equivalent_output(case: dict, actual: str, expected: str) -> bool: diff --git a/experiments/command_specialist/run.py b/experiments/command_specialist/run.py index af3de57..6b5665b 100644 --- a/experiments/command_specialist/run.py +++ b/experiments/command_specialist/run.py @@ -9,7 +9,7 @@ import uuid from pathlib import Path -from benchmark import predict +from benchmark import PredictionError, predict from bindings import bind_request from contract import evidence_result, execute_plan @@ -17,27 +17,46 @@ def inspect_request(root: Path, request: str, *, targets: dict[str, str] | None = None, evidence_request: str | None = None, model="shell-specialist-pilot", base_url="http://127.0.0.1:11434", - backend="powershell", artifacts=Path("work/command-specialist/runs")): + backend="powershell", artifacts=Path("work/command-specialist/runs"), + num_ctx=4096, num_predict=160, evidence_max_lines=100, + evidence_max_chars=8000, packet_max_chars=2000): if not isinstance(request, str) or not request.strip(): raise ValueError("request must be a nonempty string") if evidence_request is not None and not isinstance(evidence_request, str): raise ValueError("evidence_request must be a string") + settings = dict(num_ctx=num_ctx, num_predict=num_predict, + evidence_max_lines=evidence_max_lines, evidence_max_chars=evidence_max_chars, + packet_max_chars=packet_max_chars) + if any(type(value) is not int or value < 1 for value in settings.values()): + raise ValueError("runtime limits must be positive integers") root = Path(root).resolve(strict=True) if not root.is_dir(): raise ValueError("root must name a directory") start = time.perf_counter() bound = bind_request(root, request, targets) if targets is not None else None case = bound.case() if bound else {"kind": "plan", "request": request} - prediction, planning = predict(base_url, model, case) + try: + prediction, planning = predict(base_url, model, case, num_ctx=num_ctx, num_predict=num_predict) + except PredictionError as error: + artifacts = Path(artifacts) + artifacts.mkdir(parents=True, exist_ok=True) + artifact = artifacts / (uuid.uuid4().hex + ".json") + artifact.write_text(json.dumps({"request": bound.display_request if bound else request, + "model": model, "runtime": settings, "planning": error.timing, + "planning_error": str(error), "executed": False}, indent=2, + ensure_ascii=False), encoding="utf-8") + raise ValueError(f"{error}; no command executed; raw result: {artifact.resolve()}") from error plan = bound.resolve(prediction) if bound else prediction # Revalidate the resolved path immediately before execution in either backend. + execution_start = time.perf_counter() result = execute_plan(plan, root, backend=backend) + execution_ms = (time.perf_counter() - execution_start) * 1000 artifacts = Path(artifacts) artifacts.mkdir(parents=True, exist_ok=True) artifact = artifacts / (uuid.uuid4().hex + ".json") display_request = bound.display_request if bound else request saved = {"request": display_request, "plan": plan, "result": result, - "model": model, "planning": planning} + "model": model, "planning": planning, "runtime": settings, "execution_ms": execution_ms} if bound: saved["binding_trace"] = {"template": request, "targets": dict(targets), "model_request": bound.model_request, @@ -46,24 +65,44 @@ def inspect_request(root: Path, request: str, *, targets: dict[str, str] | None artifact.write_text(json.dumps(saved, indent=2, ensure_ascii=False), encoding="utf-8") packet = {"request": display_request, "plan": plan, "backend": backend, "exit_code": result["exit_code"], "raw_result": str(artifact.resolve()), - "stderr": result["stderr"][:2000], "stderr_truncated": len(result["stderr"]) > 2000} + "stderr": result["stderr"][:packet_max_chars], "stderr_truncated": len(result["stderr"]) > packet_max_chars} if evidence_request: lines = result["stdout"].splitlines() - chosen = lines[:100] + chosen = lines[:evidence_max_lines] selected_case = {"kind": "evidence", "request": evidence_request, "output_lines": chosen, "exit_code": result["exit_code"], "truncated": len(lines) > len(chosen)} - if sum(len(line) for line in chosen) > 8000: - packet.update({"selection_error": "Output window exceeds pilot context budget; inspect the raw result.", + packet["evidence_window"] = {"total_lines": len(lines), "included_lines": len(chosen), + "max_chars": evidence_max_chars} + saved["evidence_request"] = evidence_request + if not chosen: + # Exact empty output has no selectable evidence; inference adds no information. + saved["selection"] = {"lines": []} + saved["selection_method"] = "empty_output" + packet.update(evidence_result(saved["selection"], selected_case)) + elif len("\n".join(chosen)) > evidence_max_chars: + packet.update({"selection_error": "Output window exceeds evidence_max_chars; inspect the raw result.", "truncated": True}) else: try: - selection, _ = predict(base_url, model, selected_case) + selection, selection_timing = predict(base_url, model, selected_case, + num_ctx=num_ctx, num_predict=num_predict) + saved["selection"] = selection + saved["selection_timing"] = selection_timing packet.update(evidence_result(selection, selected_case)) - except (ValueError, TypeError) as error: + except (ValueError, TypeError, OSError, TimeoutError) as error: + if hasattr(error, "timing"): + saved["selection_timing"] = error.timing packet.update({"selection_error": str(error), "truncated": True}) else: - packet.update({"stdout": result["stdout"][:2000], "truncated": len(result["stdout"]) > 2000}) + packet.update({"stdout": result["stdout"][:packet_max_chars], "truncated": len(result["stdout"]) > packet_max_chars}) + packet["runtime"] = settings + packet["fallback"] = "inspect_raw_result" if packet.get("selection_error") or packet.get("truncated") or packet["stderr_truncated"] else None + saved["packet"] = dict(packet) + # Preserve the pre-selection raw artifact even if this final audit write fails. + final_artifact = artifact.with_suffix(".tmp") + final_artifact.write_text(json.dumps(saved, indent=2, ensure_ascii=False), encoding="utf-8") + final_artifact.replace(artifact) packet["planning_ms"] = planning["inference_wall_ms"] packet["wall_ms"] = (time.perf_counter() - start) * 1000 return packet @@ -81,6 +120,11 @@ def main(): parser.add_argument("--base-url", default="http://127.0.0.1:11434") parser.add_argument("--backend", choices=["powershell", "native"], default="powershell") parser.add_argument("--artifacts", type=Path, default=Path("work/command-specialist/runs")) + parser.add_argument("--num-ctx", type=int, default=4096) + parser.add_argument("--num-predict", type=int, default=160) + parser.add_argument("--evidence-max-lines", type=int, default=100) + parser.add_argument("--evidence-max-chars", type=int, default=8000) + parser.add_argument("--packet-max-chars", type=int, default=2000) args = parser.parse_args() if args.task_file: if args.target or args.evidence_request: @@ -103,7 +147,10 @@ def main(): parser.error("targets must be distinct NAME=RELATIVE_PATH bindings") targets[name] = path packet = inspect_request(args.root, request, targets=targets, evidence_request=evidence_request, - model=args.model, base_url=args.base_url, backend=args.backend, artifacts=args.artifacts) + model=args.model, base_url=args.base_url, backend=args.backend, artifacts=args.artifacts, + num_ctx=args.num_ctx, num_predict=args.num_predict, + evidence_max_lines=args.evidence_max_lines, evidence_max_chars=args.evidence_max_chars, + packet_max_chars=args.packet_max_chars) sys.stdout.reconfigure(encoding="utf-8") print(json.dumps(packet, indent=2, ensure_ascii=False)) diff --git a/experiments/command_specialist/test_bindings.py b/experiments/command_specialist/test_bindings.py index 9bac384..ae23fde 100644 --- a/experiments/command_specialist/test_bindings.py +++ b/experiments/command_specialist/test_bindings.py @@ -69,18 +69,24 @@ def test_readable_packet_saved_binding_and_verbatim_evidence(self): ({'lines':[2,3]}, {'inference_wall_ms':1})] with patch('run.predict', side_effect=predictions) as predict: packet = inspect_request(root, 'Read last 3 lines of {{log}}.', targets={'log':name}, - evidence_request='errors and summary', backend='native', artifacts=root/'runs') + evidence_request='errors and summary', backend='native', artifacts=root/'runs', + num_ctx=8192, num_predict=2048) self.assertEqual(predict.call_args_list[0].args[2]['request'], 'Read last 3 lines of "file_1".') self.assertEqual(packet['plan']['path'], name) self.assertIn(name, packet['request']) self.assertEqual(packet['evidence'], ['ERROR preserve this', 'SUMMARY failed']) self.assertEqual(packet['source_lines'], [2,3]) + self.assertTrue(all(call.kwargs == {'num_ctx':8192, 'num_predict':2048} + for call in predict.call_args_list)) self.assertEqual(packet['exit_code'], 0) self.assertNotIn('binding_trace', packet) saved = json.loads(Path(packet['raw_result']).read_text(encoding='utf-8')) self.assertEqual(saved['plan']['path'], name) self.assertEqual(saved['binding_trace']['references'], {'file_1':name}) self.assertEqual(saved['result']['stdout'], content.rstrip('\n')) + self.assertEqual(saved['selection'], {'lines':[2,3]}) + self.assertEqual(saved['runtime']['num_predict'], 2048) + self.assertEqual(saved['packet']['evidence'], packet['evidence']) def test_invalid_reference_never_executes_and_deleted_target_is_revalidated(self): with tempfile.TemporaryDirectory() as directory: @@ -91,7 +97,7 @@ def test_invalid_reference_never_executes_and_deleted_target_is_revalidated(self with self.assertRaisesRegex(ValueError, 'unknown file reference'): inspect_request(root, 'Read {{log}}.', targets={'log':'a.txt'}, artifacts=root/'runs') execute.assert_not_called() - def removed(*args): + def removed(*args, **kwargs): target.unlink() return {'op':'read_head','path':'file_1','value':'','limit':1}, {} with patch('run.predict', side_effect=removed): From d0b40882e3fb9dbf549afb3794608d3dcb3cacfc Mon Sep 17 00:00:00 2001 From: NoisemakerJon <139656120+Noisemaker111@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:55:23 -0400 Subject: [PATCH 3/3] Reserve main merges for Jon and prepare maintainer release notes (#6) --- .github/workflows/agent-checks.yml | 3 ++ AGENTS.md | 32 +++++++++++----- scripts/check_release.py | 25 ++++++++++++ scripts/integrate_agents.py | 23 ++++++++--- scripts/prepare_release.py | 61 ++++++++++++++++++++++++++++++ 5 files changed, 128 insertions(+), 16 deletions(-) create mode 100644 scripts/check_release.py create mode 100644 scripts/prepare_release.py diff --git a/.github/workflows/agent-checks.yml b/.github/workflows/agent-checks.yml index 66d5d91..58fdce6 100644 --- a/.github/workflows/agent-checks.yml +++ b/.github/workflows/agent-checks.yml @@ -2,6 +2,7 @@ name: Agent checks on: pull_request: branches: [agents, main] + types: [opened, synchronize, reopened, ready_for_review, edited] push: branches: [agents] permissions: @@ -27,3 +28,5 @@ jobs: run: python -m unittest discover -s experiments/command_specialist -p test_bindings.py -v - name: Native and PowerShell contract invariants run: python -m unittest discover -s experiments/command_specialist -p test_contract.py -v + - name: Frozen main candidate and patch notes + run: python scripts/check_release.py diff --git a/AGENTS.md b/AGENTS.md index 6477e37..3c8ca2f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,8 +4,11 @@ branch. Jon authorized agents-branch automation on September 10, 2026: agents may prepare owned changes, verify them, open ready PRs against `agents`, and merge them when the gates below pass. This replaces per-PR human merge requests for `agents`. -Merging into `main`, publication, deployments, model/weight changes, and production -data changes still require explicit approval for the concrete batch. +Only Jon personally merges release PRs into `main`. Agents must never merge main, +enable its auto-merge, or push directly to it, even after chat approval. Publication, +deployments, model/weight changes, and production data changes need separate approval. +This agents automation applies to every agent acting for a maintainer with live +repository write, maintain, or admin permission; no particular model/host is privileged. Start from fetched `agents` in an owned worktree. Preserve other sessions and reuse existing PRs for the same work. Run commands from the repository root with Python @@ -27,8 +30,11 @@ into both branches and pushes to `agents`. It runs on a hosted Windows runner wi read-only permissions and no persisted checkout credentials. PR code never receives a privileged merge token. `agents` requires this GitHub Actions check and an up-to-date PR, prevents force pushes/deletion, and applies protection to administrators. No approving review is -required for `agents`. Main protection is pending Jon's choice of GitHub review -enforcement versus explicit chat approval; do not claim it is server-enforced. +required for `agents`. Main requires the same CI/PR gate; Jon performs the merge +himself. A separate approving reviewer is not required because Jon authors these +PRs through the same account. The platform cannot distinguish Jon from an agent +using his credentials; the human-only merge rule is enforced by this workflow and +the coordinator's unconditional refusal of main. The local coordinator is `scripts/integrate_agents.py`. After reviewing the diff, verifying the current head locally, and waiting for CI, run: @@ -37,14 +43,19 @@ verifying the current head locally, and waiting for CI, run: python scripts/integrate_agents.py --pr --verified-head ``` -It accepts only ready same-repository PRs authored by Noisemaker111 targeting -`agents`; checks the exact head, workflow identity/result, and mergeability; and +It accepts only ready same-repository PRs targeting `agents` when both the PR author +and the acting account have live write/maintain/admin permission; checks the exact head, workflow identity/result, and mergeability; and merges with GitHub's expected-head condition. It refuses `main`. `--check-only` checks eligibility without merging. Invoke it from a trusted checkout; passing a head is the agent's attestation of actual local verification, not proof supplied -by CI. This is agent-operated automation, not an installed unattended scheduler. +by CI. Every maintainer's agent uses this same automatic integration path; it does not +require Jon to run the command. It is agent-operated, not an unattended scheduler. The agent continues through merge and merged-revision verification without asking -Jon for routine agents integration approval. +Jon for routine agents integration approval. Each successful merge automatically +runs `scripts/prepare_release.py`: it generates patch notes from merged change +titles and opens a frozen main release PR. An existing open release batch is +preserved; later agents changes wait for the next batch. Release preparation may +be resumed with `python scripts/prepare_release.py`; this never merges main. This repository is a local skill/CLI, with no hosted frontend, backend, database, queue, production service, or deployment trigger configured here. Dev validation @@ -57,8 +68,9 @@ reporting dev verification. For stable promotion, branch a frozen release candidate from a verified agents revision and open a ready PR into `main`. Include the full diff, user-facing notes, verification evidence, and rollback target (the prior main revision). Later agents -changes must not join that candidate silently. Ask Jon to approve that concrete -batch before merging. The agents GitHub PR/check gates are enforced, but this account is shared +changes must not join that candidate silently. CI checks the candidate identity +and generated patch notes via `scripts/check_release.py`, in addition to core +checks. Ask Jon to merge the CI-green release PR himself; agents stop before main. The agents GitHub PR/check gates are enforced, but this account is shared by Jon and agents: human authorization is a workflow rule, not an independently verified GitHub reviewer identity. The coordinator cannot merge stable. No package or runtime deployment is implied by either merge. diff --git a/scripts/check_release.py b/scripts/check_release.py new file mode 100644 index 0000000..6f12002 --- /dev/null +++ b/scripts/check_release.py @@ -0,0 +1,25 @@ +"""Validate that a main PR is an explicitly pinned release with patch notes.""" +import json +import os +from pathlib import Path +import re + + +def validate(pr): + if pr['base']['ref'] != 'main': + return + if not pr['head']['ref'].startswith('release/agents-'): + raise ValueError('Main accepts a frozen agents release candidate; prepare its release PR') + marker = f"" + body = pr.get('body') or '' + if marker not in body or '## Patch notes' not in body or not re.search(r'^- .+', body, re.MULTILINE): + raise ValueError('Patch notes and exact candidate identity must be present in the release PR') + if not re.search(r'', body): + raise ValueError('Release must record the prior stable source') + + +if __name__ == '__main__': + event = json.loads(Path(os.environ['GITHUB_EVENT_PATH']).read_text(encoding='utf-8')) + if 'pull_request' in event: + validate(event['pull_request']) + print('Release candidate metadata checked; only Jon may merge main.') diff --git a/scripts/integrate_agents.py b/scripts/integrate_agents.py index 1a25f41..a411508 100644 --- a/scripts/integrate_agents.py +++ b/scripts/integrate_agents.py @@ -1,4 +1,4 @@ -"""Integrate a locally verified, CI-green owner PR into agents; never main. +"""Integrate a locally verified, CI-green maintainer PR into agents; never main. Run from a trusted agents checkout after inspecting the diff and completing the real user-operation check. No PR code is downloaded or executed by this tool. @@ -9,7 +9,7 @@ import subprocess REPO = "Noisemaker111/shell-forensics" -OWNER = "Noisemaker111" +MAINTAINER_PERMISSIONS = {"write", "maintain", "admin"} def gh(*args): @@ -17,11 +17,17 @@ def gh(*args): return json.loads(result.stdout) if result.stdout.strip() else None +def require_maintainer(login): + permission = gh("api", f"repos/{REPO}/collaborators/{login}/permission")["permission"] + if permission not in MAINTAINER_PERMISSIONS: + raise ValueError(f"{login} does not have repository write/maintain/admin permission") + + def eligible(pr, verified_head): if pr["baseRefName"] != "agents": - raise ValueError("Only agents PRs may be integrated; main needs Jon's explicit batch approval") - if pr["state"] != "OPEN" or pr["isDraft"] or pr["isCrossRepository"] or pr["author"]["login"].lower() != OWNER.lower(): - raise ValueError("Require an open, ready, same-repository owner PR") + raise ValueError("Only agents PRs may be integrated; only Jon may perform the main merge") + if pr["state"] != "OPEN" or pr["isDraft"] or pr["isCrossRepository"]: + raise ValueError("Require an open, ready, same-repository maintainer PR") if pr["headRefOid"] != verified_head: raise ValueError("Head changed or was not locally verified; recheck the actual user operation") if pr["mergeable"] != "MERGEABLE" or pr["mergeStateStatus"] != "CLEAN": @@ -44,6 +50,8 @@ def main(): fields = "state,isDraft,isCrossRepository,author,baseRefName,headRefOid,mergeable,mergeStateStatus,statusCheckRollup" pr = gh("pr", "view", str(args.pr), "--repo", REPO, "--json", fields) checks = eligible(pr, args.verified_head) + require_maintainer(pr["author"]["login"]) + require_maintainer(gh("api", "user")["login"]) for check in checks: match = re.fullmatch(r"https://github\.com/Noisemaker111/shell-forensics/actions/runs/(\d+)/job/\d+", check.get("detailsUrl", "")) if not match: @@ -62,7 +70,10 @@ def main(): receipt = gh("pr", "view", str(args.pr), "--repo", REPO, "--json", "state,baseRefName,mergeCommit,url") if receipt["state"] != "MERGED" or receipt["baseRefName"] != "agents": raise RuntimeError("Merge receipt did not confirm agents integration") - print(json.dumps(receipt)) + print(json.dumps(receipt), flush=True) + # Prepare the human's next release batch; this helper never merges main. + from prepare_release import prepare + print(json.dumps(prepare(receipt["mergeCommit"]["oid"]))) if __name__ == "__main__": diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py new file mode 100644 index 0000000..70a7616 --- /dev/null +++ b/scripts/prepare_release.py @@ -0,0 +1,61 @@ +"""Generate patch notes and a frozen release PR; only Jon merges main.""" +import argparse +import json +from pathlib import Path +import re +import subprocess + +REPO = "Noisemaker111/shell-forensics" + + +def command(*args): + return subprocess.run(args, check=True, capture_output=True, text=True, encoding="utf-8", timeout=60).stdout.strip() + + +def prepare(candidate=None): + command("git", "fetch", "origin", "agents", "main") + candidate = candidate or command("git", "rev-parse", "origin/agents") + if not re.fullmatch(r"[0-9a-f]{40}", candidate): + raise ValueError("candidate must be a full commit ID") + command("git", "merge-base", "--is-ancestor", candidate, "origin/agents") + stable = command("git", "rev-parse", "origin/main") + command("git", "merge-base", "--is-ancestor", stable, candidate) + if not command("git", "diff", "--name-only", stable, candidate): + return {"release": "no changes"} + existing = json.loads(command("gh", "pr", "list", "--repo", REPO, "--base", "main", "--state", "open", "--json", "headRefName,url")) + releases = [p for p in existing if p["headRefName"].startswith("release/agents-")] + if releases: + return {"release": "existing batch preserved for Jon", "url": releases[0]["url"]} + subjects = command("git", "log", "--first-parent", "--reverse", "--format=%s", f"{stable}..{candidate}").splitlines() + if not subjects: + raise ValueError("No release changes were found") + notes = (f"\n\n" + "## Patch notes\n\n" + "\n".join(f"- {subject}" for subject in subjects) + + f"\n\n[Full release diff](https://github.com/{REPO}/compare/{stable}...{candidate})\n\n" + "## Merge and verification\n\nOnly Jon merges this PR into main. Agents must not enable auto-merge or perform the stable merge. " + "This candidate is frozen; later agents changes belong to a later batch.\n\n" + "Required CI runs the Windows binding/contract checks, syntax validation, and frozen-candidate metadata check. " + "Review the candidate's included PRs for their affected-user-operation evidence; hosted CI does not run the private model. " + "These patch notes are generated from merged change titles and do not imply additional performance or deployment validation.\n\n" + f"Previous stable source / rollback target: `{stable}`. No deployment, package publication, or data migration is performed by this helper.\n") + out = Path("work/releases") / candidate + out.mkdir(parents=True, exist_ok=True) + body = out / "patch-notes.md" + body.write_text(notes, encoding="utf-8") + branch = f"release/agents-{candidate[:12]}" + remote = command("git", "ls-remote", "origin", f"refs/heads/{branch}") + if remote and remote.split()[0] != candidate: + raise ValueError("Release branch already points elsewhere; refusing to move it") + if not remote: + command("git", "push", "origin", f"{candidate}:refs/heads/{branch}") + url = command("gh", "pr", "create", "--repo", REPO, "--base", "main", "--head", branch, + "--title", "Release verified agents changes to main", "--body-file", str(body)) + receipt = {"release": "ready for Jon after CI", "url": url, "candidate": candidate, "notes": str(body.resolve())} + (out / "receipt.json").write_text(json.dumps(receipt, indent=2), encoding="utf-8") + return receipt + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate") + print(json.dumps(prepare(parser.parse_args().candidate)))