diff --git a/.github/workflows/agent-checks.yml b/.github/workflows/agent-checks.yml index 602fcda..0c0eee8 100644 --- a/.github/workflows/agent-checks.yml +++ b/.github/workflows/agent-checks.yml @@ -23,14 +23,14 @@ jobs: with: python-version: '3.12' - name: Validate syntax - run: python -m compileall -q scripts experiments/command_specialist + run: python -m compileall -q scripts skills experiments/command_model - name: Path binding and saved evidence invariants - run: python -m unittest discover -s experiments/command_specialist -p test_bindings.py -v + run: python -m unittest discover -s experiments/command_model -p test_bindings.py -v - name: Native and PowerShell contract invariants - run: python -m unittest discover -s experiments/command_specialist -p test_contract.py -v + run: python -m unittest discover -s experiments/command_model -p test_contract.py -v - name: English delegation evidence and authorization invariants - run: python -m unittest discover -s experiments/command_specialist -p test_delegate.py -v + run: python -m unittest discover -s experiments/command_model -p test_delegate.py -v - name: Frozen main candidate and patch notes run: python scripts/check_release.py - name: Codex comparison evidence invariants - run: python -m unittest discover -s experiments/command_specialist/codex -p test_compare.py -v + run: python -m unittest discover -s experiments/command_model/codex -p test_compare.py -v diff --git a/.gitignore b/.gitignore index 5051bed..6c932a2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,5 @@ work/ *.jsonl .venv/ .worktrees/ -experiments/command_specialist/js_parser/node_modules/ +experiments/command_model/js_parser/node_modules/ *.log diff --git a/AGENTS.md b/AGENTS.md index bd55b5d..f68056e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,4 +1,9 @@ -# Command specialist +# Command Model + +"Our model", "the model we are building", and "Command Model" refer to this +project. Shell Gatherer is only the collection component in skills/shell-gatherer. +Mining, labeling, execution verification, datasets and training are downstream +steps. Do not describe gathered observations as verified training examples. Build a small, fast local model and execution loop that does command/cell work for frontier agents. Accept English intent and known context, perform the mechanical @@ -6,7 +11,7 @@ work, inspect real outcomes, repair ordinary failures, and return compact verifi evidence with retrievable raw output. Each delegation gets a fresh, bounded worker lifecycle. A generated program or claimed success is not completion. -Read [project purpose and evidence](experiments/command_specialist/PURPOSE.md) +Read [project purpose and evidence](experiments/command_model/PURPOSE.md) before choosing the next experiment. The frontier owns reasoning and delegation; the worker owns bounded mechanical implementation and verified evidence. The 486-example inspection/evidence adapter is not an English code-execution-trained @@ -32,6 +37,21 @@ Measure whole handoffs, including repair and fallback; small synthetic probes do not establish general reliability. Preserve held-out data and the current pilot model/adapter; keep private corpora and weights out of Git. -Follow [DEVELOPMENT.md](DEVELOPMENT.md) for checks, isolation, maintainer permissions -and automatic agents integration. Keep release details out of routine reports. -Jon initiates release work. Report observed improvements, failures and limitations. +Follow the agents-and-main workflow in [DEVELOPMENT.md](DEVELOPMENT.md). +Jon explicitly authorizes agents acting for eligible maintainers to prepare, +verify, push, open ready PRs and merge them into `agents` after the exact reviewed +head passes local verification and required CI. Jon reaffirmed this standing +authorization on September 10, 2026: do not ask for per-PR merge permission for +`agents`. This project-specific authorization replaces generic instructions to +ask Jon before every merge into this development branch. Use the coordinator and +its maintainer, expected-head and CI gates; never bypass a failed gate. + +Use `sb agents` / `sb main` only in a checkout you own. Do implementation work in +isolated worktrees and preserve other sessions. Only Jon personally merges +`main`; prepare a release only when he explicitly requests one. Agents integration +does not authorize production deployment, publication or unrelated changes. +If an external approval control rejects an authorized action, report its actual +reason and provide the existing user authorization; do not bypass that control +or describe the project policy itself as requiring new per-PR permission. +Keep release details out of routine reports. Report observed improvements, +failures and limitations. diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 46c2b34..ee397dd 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -4,6 +4,9 @@ branch. Jon authorized agents-branch automation on September 10, 2026: agents may prepare owned changes, verify them, open ready PRs against `agents`, and merge them when the gates below pass. This replaces per-PR human merge requests for `agents`. +Jon reaffirmed that authorization in chat on September 10, 2026 after an external +approval control rejected a merge. The explicit authorization is also recorded +directly in root AGENTS.md so readers need not infer it from this policy link. Only Jon personally merges release PRs into `main`. Agents must never merge main, enable its auto-merge, or push directly to it, even after chat approval. Publication, deployments, model/weight changes, and production data changes need separate approval. @@ -15,9 +18,9 @@ existing PRs for the same work. Run commands from the repository root with Pytho 3.12. Required checks are: ``` -python -m compileall -q scripts experiments/command_specialist -python -m unittest discover -s experiments/command_specialist -p test_bindings.py -v -python -m unittest discover -s experiments/command_specialist -p test_contract.py -v +python -m compileall -q scripts skills experiments/command_model +python -m unittest discover -s experiments/command_model -p test_bindings.py -v +python -m unittest discover -s experiments/command_model -p test_contract.py -v ``` Also verify the affected user operation through the public CLI and reopen its saved diff --git a/README.md b/README.md index c2e1e08..5e3747b 100644 --- a/README.md +++ b/README.md @@ -1,81 +1,81 @@ -# Shell Forensics +# Command Model -A Claude Code skill that reads the transcripts your coding agents already keep on disk (Codex, Claude Code, OpenCode, Cursor), pulls out every shell command they ran, and builds a report: which model does everything through bash or PowerShell or a JavaScript cell, what the shell is used for, how it fails, and what actually fixed it. The report is a deck of screenshot-sized cards, each with a ready-to-post summary. +A small local model and execution loop for command/cell work delegated by a +frontier agent. The frontier supplies English intent and known context; one fresh +local worker performs bounded mechanical work, verifies the result and returns +compact evidence. "Our model" and "the model we are building" refer to this project. -Example output from the first machine it ran on: 38,176 commands, 874 sessions, 48 model IDs. Findings included a 30% failure rate when the model typed POSIX into a PowerShell host versus 11% when the dialect matched, `| head` typed into PowerShell as the single most common concrete failure, `python -c` one-liners failing at 28% versus 10% for the same Python fed through a heredoc, and GPT-5.6 models printing only `r.output` in Codex cells 92% of the time so that non-zero exit codes never reached the transcript. See `reference.md` for the baseline numbers. +Start with [purpose and evidence](experiments/command_model/PURPOSE.md), +[agent instructions](AGENTS.md), and the [Codex handoff prototype](experiments/command_model/codex/README.md). +The original inspection adapter and later execution experiments have different +scopes; the purpose document preserves what each actually demonstrated. -## Install +## Components -Claude Code: +| Component | Responsibility | +| --- | --- | +| Shell Gatherer | Collect recorded shell commands, results and provenance | +| Mining and labeling pipeline | Normalize observations, recover useful candidates and assign supported labels | +| Execution verification and datasets | Check candidate behavior and save examples with frozen held-out partitions | +| Model training | Train for the actual delegation protocol, preserving original adapters | +| Local execution loop and frontier evaluation | Execute grounded English jobs and measure verified whole-operation accuracy, time and tokens | -``` -git clone https://github.com/Noisemaker111/shell-forensics ~/.claude/skills/shell-forensics -``` +The gatherer does not certify training examples. See the +[data pipeline](experiments/command_model/DATA_PIPELINE.md) and +[measured data results](experiments/command_model/DATA_RESULTS.md). -Then in any Claude Code session: +## Repository and local work +```powershell +git clone https://github.com/Noisemaker111/command-model +cd command-model ``` -/shell-forensics -``` - -Codex reads skills from `~/.codex/skills`, so the same clone works there: -``` -git clone https://github.com/Noisemaker111/shell-forensics ~/.codex/skills/shell-forensics -``` +Follow [DEVELOPMENT.md](DEVELOPMENT.md) for owned worktrees, checks and integration. +The model source is in experiments/command_model. Existing shell-specialist model +aliases, frozen dataset seeds and old work/command-specialist data paths retain +their identities for reproducibility; this rename does not retrain weights. -You can also run the scripts by hand without any agent. Python 3.10+, standard library only: +## Shell Gatherer -``` -python scripts/extract.py work # records.jsonl, toolcounts.json -python scripts/analyze.py work # records_annotated.jsonl, summary.json, prints the headline tables -python scripts/scratch.py work # scratch.json: scripts written to disk per model -python scripts/build.py work # shell-forensics.html, page-data.json -``` +The self-contained skill is [skills/shell-gatherer](skills/shell-gatherer/SKILL.md). +Copy that directory to ~/.claude/skills/shell-gatherer or +~/.codex/skills/shell-gatherer to install it. In a fresh session request +"Use Shell Gatherer to collect my local shell-command observations." -Open `work/shell-forensics.html` in a browser. Add `--no-gallery` to `build.py` for a page you can share: the gallery embeds real commands with real paths from your machine. +To run directly with Python 3.10+ and no dependencies: -## What it reads - -| Harness | Path | Parsed | -|---|---|---| -| Codex CLI and desktop | `~/.codex/sessions/**/*.jsonl` | `exec` JS cells; each `tools.exec_command` / `shell_command` inside is one record | -| Claude Code | `~/.claude/projects/**/*.jsonl` | `Bash` and `PowerShell` tool calls joined to their results | -| OpenCode | `~/.local/share/opencode/opencode.db` | `bash`, `shell`, `oc_bash` and `execute` tool parts, v1 and v2 stores | -| Cursor CLI | `~/.cursor/chats/*/*/store.db` | `Shell` tool calls joined to results | - -Read-only. Nothing leaves the machine unless you publish the page. +```powershell +New-Item -ItemType Directory -Force work +python skills/shell-gatherer/scripts/gather.py work +``` -## What the classifiers do +This writes records.jsonl and toolcounts.json. The old scripts/extract.py command +forwards to the same implementation. Recorded output is capped at 700 characters; +full-transcript ingestion is a separate pipeline. No collected commands execute. -Each command is tagged with the dialect the model wrote (POSIX, PowerShell, cmd, neutral, mixed), the host shell that ran it, its job (git, read, search, build/test, and so on), any embedded program (`python -c`, heredoc, `node -e`, here-string, jq, nested pwsh), the failure cause parsed from the output, and whether the harness reported success while the output contained a hard error. For every failure it also looks at the next shell command in the session to see what the model tried and whether that worked. +## Downstream shell analysis -These are regex heuristics checked by hand on samples. Expect a few percent noise per cell. The page says so. +These scripts consume observations; they are not part of gathering: -## Caveats worth repeating +```powershell +python scripts/analyze.py work +python scripts/scratch.py work +python scripts/build.py work --no-gallery +``` -- Harness design dominates. Codex has no read or edit tools, so its shell share is structural. Compare models within one harness. -- Instructions leak in. Claude Code's auto mode tells the model to work through Bash; the page marks those families. -- Host shells differ per harness and per OS, so failure rates across harnesses are not comparable. -- One user, one machine, one task mix per run. Post your numbers and the picture gets better. +The report is work/shell-analysis.html with page-data.json. Analysis uses heuristic +classifications, not verified task-success labels. The optional gallery exposes +recorded commands and paths; keep private data local unless publication is authorized. +[Historical observations](reference.md) preserve the original analysis context. -## Files +## Naming and compatibility -- `SKILL.md` — instructions the agent follows -- `reference.md` — baseline numbers and verified harness facts from the first run -- `scripts/extract.py`, `analyze.py`, `scratch.py`, `build.py`, `template.html` +The repository was formerly shell-forensics and the model experiment was called +command specialist. Current names are **Command Model** (project) and **Shell +Gatherer** (collection). Historical artifacts and installed model aliases may keep +old names. On the development machine, the former repository folder is a +compatibility junction to command-model so existing worktrees, environments and +saved absolute paths remain accessible. New work should use the canonical name. MIT license. - -## Local command model experiment - -The current [command-specialist direction](experiments/command_specialist/PURPOSE.md) -is English delegation of bounded command/cell work from a frontier agent to a -fresh local worker, returning verified evidence. It distinguishes the narrow -trained inspection pilot from the broader runtime and its measured host results. - -The optional [command specialist pilot](experiments/command_specialist/README.md) -audits training-data quality, benchmarks small local models on bounded PowerShell -inspection and evidence-selection tasks, and trains a local LoRA adapter. Its -synthetic capability scores are separate from this repository's observed transcript -statistics and do not establish superiority to frontier models. diff --git a/SKILL.md b/SKILL.md deleted file mode 100644 index 83a13c6..0000000 --- a/SKILL.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -name: shell-forensics -description: Mine the local transcripts of Codex, Claude Code, OpenCode and Cursor for every shell command the models ran, classify dialect, job, embedded programs (python -c, node -e, heredocs), failures and recoveries, and publish a screenshot-ready card deck with a post for each finding. Use when someone asks how their coding agents use bash/PowerShell, why shell calls fail, which model "does everything through bash", or wants the Shell Forensics report for their own machine. ---- - -# Shell Forensics - -Turns the transcript stores that coding agents already keep on disk into one dataset of shell commands, then into a report. Nothing is self-reported by a model; every record is a tool call joined to its result. - -## Sources read (all optional; missing ones are skipped) - -| Harness | Path | What is parsed | -|---|---|---| -| Codex (CLI + desktop) | `~/.codex/sessions/**/*.jsonl`, `~/.codex/archived_sessions` | `exec` JS cells; each `tools.exec_command` / `shell_command` inside is one record. Exit code comes from the `{"exit_code":n}` JSON the cell printed, when it printed it. Model from `turn_context`. | -| Claude Code | `~/.claude/projects/**/*.jsonl` | `Bash` and `PowerShell` tool_use joined to tool_result by id. Flags sessions carrying the auto-mode "work through Bash" instruction. | -| OpenCode | `~/.local/share/opencode/opencode.db` | v1 `part` and v2 `session_message` tool parts (`bash`, `shell`, `oc_bash`, `execute` JS cell), de-duplicated by call id. | -| Cursor CLI | `~/.cursor/chats/*/*/store.db` | `Shell` tool-calls joined to results. | - -Read-only. Nothing is uploaded until the user publishes the page. - -## Run - -Needs Python 3.10+ (stdlib only) and, for the palette check, nothing else. Pick a work directory (the session scratchpad is fine). - -``` -python /scripts/extract.py # -> records.jsonl, toolcounts.json -python /scripts/analyze.py # -> records_annotated.jsonl, summary.json (prints headline tables) -python /scripts/scratch.py # -> scratch.json (scratch scripts written to disk, per model) -python /scripts/build.py # -> shell-forensics.html, page-data.json -``` - -`` is this skill's directory. On Windows run them through the Bash tool with `PYTHONIOENCODING=utf-8`, or through the PowerShell tool as `python `. The Codex parse is the slow step (about 1 minute per 2 GB of sessions). - -Options for `build.py`: -- `--no-gallery` drops the 1,500-command sample and the retry examples from the page. Use it when the page will be shared outside the machine's owner: the gallery embeds real commands with real paths. -- `posts.json` in the work directory overrides any post by key (`lead`, `share`, `corpus`, `jobs`, `dialect`, `head`, `embed`, `form`, `scratch`, `hidden`, `exit`, `causes`, `recovery`, `output`, `rules`, `change`). Write it after reading `summary.json` when the default wording does not match what the data shows; keep each under 280 characters. - -## After it runs - -1. Read the tables `analyze.py` printed (per-family fail rates, dialect × host, interpreter forms, recovery moves). Sanity-check the two classifiers most likely to be wrong on a new machine: `host_shell()` in `analyze.py` (Windows values are inferred per harness; on macOS/Linux it uses `$SHELL`) and the `family()` map of model IDs to display names. Add any model ID the map does not know. -2. Publish `shell-forensics.html` with the Artifact tool (favicon 🐚). Before publishing a page with the gallery, tell the user it contains real commands and paths and offer `--no-gallery`. -3. Tell the user the numbers that changed what they should do, not the method. The page's "What to change" card is written for a Windows host with Git Bash and pwsh 7; on macOS/Linux only the output-cap, heredoc and exit-code rules apply, so say so. - -## What the classifiers mean - -- **dialect** is what the model typed: `posix`, `powershell`, `cmd`, `neutral` (valid anywhere: bare git/bun/gh), `mixed` (cmdlets and Unix tools in one line), `js-only` (a Codex/OpenCode cell that never called the shell). -- **host** is what ran it. Harness tools do not always run what their name says: on Windows, OpenCode's `bash` tool was observed spawning `powershell.exe` 5.1 and its `shell` tool cmd.exe; Codex runs PowerShell unless `shell:"cmd.exe"` is passed; Claude Code's `Bash` is Git Bash. -- **mismatch** is a POSIX command on a Windows shell or a PowerShell/cmd command on a POSIX shell. -- **interp** is an embedded program inside the shell call: `python -c`, python via heredoc/stdin, `node -e`, node via stdin, `bun -e`, bun/npx/tsx scripts, nested `pwsh -Command`, `cmd /c`, heredoc, PowerShell here-string, jq, sqlite3, perl/awk. -- **cause** is the first matching error pattern in the output; `nonzero/other` means a non-zero exit with no recognizable string; `build/test failure` means the tool worked and the code did not. -- **hidden** is a call the harness marked OK whose output still contains a hard error (cmdlet not recognized, command not found, Traceback, syntax error, path not found, access denied). It is a lower bound. -- **recovery** compares each failed command with the next shell command in the same session: identical retry, rewrite in the same dialect, dialect switch, interpreter switch, tool switch, or leaving the shell; and whether that next one worked. - -All of these are regex heuristics checked by hand on samples. Expect a few percent noise per cell and say so on the page (the Method card already does). - -## Caveats to carry into any conclusion - -- Harness design dominates: Codex has no read/edit tools, so its shell share is structural. Compare models within one harness. -- Instructions leak in: auto mode and repo AGENTS.md files push models toward Bash. The page marks instructed families with ⚑. -- Host shells differ per harness, so failure rates across harnesses are not comparable. -- One user, one machine, one task mix. - -## Files - -- `scripts/extract.py` — parsers for the four stores, JS-cell parser for `tools.*` calls. -- `scripts/analyze.py` — classifiers and aggregates. -- `scripts/scratch.py` — census of script files written through Write/patch tools. -- `scripts/build.py` + `scripts/template.html` — the card deck (IBM Plex, validated categorical palette, light and dark). -- `reference.md` — what was found on the first machine this ran on, as a baseline to compare against. diff --git a/experiments/command_specialist/BINDING_RESULTS.md b/experiments/command_model/BINDING_RESULTS.md similarity index 95% rename from experiments/command_specialist/BINDING_RESULTS.md rename to experiments/command_model/BINDING_RESULTS.md index ccf9819..d11a54d 100644 --- a/experiments/command_specialist/BINDING_RESULTS.md +++ b/experiments/command_model/BINDING_RESULTS.md @@ -67,8 +67,8 @@ Those conditions differ from the native no-selection benchmark above. ## Reproduction and limitations ```powershell -python -m unittest discover -s experiments/command_specialist -p 'test_*.py' -v -python experiments/command_specialist/benchmark_bindings.py --out work/command-specialist/new-binding-trial --backend native +python -m unittest discover -s experiments/command_model -p 'test_*.py' -v +python experiments/command_model/benchmark_bindings.py --out work/command-specialist/new-binding-trial --backend native ``` Each new output directory gets new random filenames. The original cases, hashes, diff --git a/experiments/command_specialist/DATA_PIPELINE.md b/experiments/command_model/DATA_PIPELINE.md similarity index 84% rename from experiments/command_specialist/DATA_PIPELINE.md rename to experiments/command_model/DATA_PIPELINE.md index a4e6d34..460fee3 100644 --- a/experiments/command_specialist/DATA_PIPELINE.md +++ b/experiments/command_model/DATA_PIPELINE.md @@ -27,11 +27,11 @@ An optional `legacy` entry can point to an existing `records_annotated.jsonl`. Its records remain separately attributed; they can overlap native transcripts. ```powershell -python experiments/command_specialist/data_pipeline.py --config work/data-sources.json --out work/data-v2 -python experiments/command_specialist/label_data.py --root work/data-v2 --out work/data-v2/frozen-review -python experiments/command_specialist/review_queue.py --frozen work/data-v2/frozen-review --out work/data-v2/review-queue.json -python experiments/command_specialist/verify_labels.py --frozen work/data-v2/frozen-review --out work/data-v2/verified-reads -python -m unittest discover -s experiments/command_specialist -p 'test_*.py' -v +python experiments/command_model/data_pipeline.py --config work/data-sources.json --out work/data-v2 +python experiments/command_model/label_data.py --root work/data-v2 --out work/data-v2/frozen-review +python experiments/command_model/review_queue.py --frozen work/data-v2/frozen-review --out work/data-v2/review-queue.json +python experiments/command_model/verify_labels.py --frozen work/data-v2/frozen-review --out work/data-v2/verified-reads +python -m unittest discover -s experiments/command_model -p 'test_*.py' -v ``` Run ingestion again to process changes. `--refresh` forces source rereads even if @@ -115,7 +115,7 @@ This supplement alone does not justify retraining a broad command model. ## Static recovery and expanded training -Install the exact Acorn dependency from `experiments/command_specialist/js_parser` +Install the exact Acorn dependency from `experiments/command_model/js_parser` using `npm ci --ignore-scripts --no-audit --no-fund`; run `npm test` in that directory. Use a current Node runtime (verified with the installed Node 24.18.0 runtime). Acorn's [parser API](https://github.com/acornjs/acorn/tree/master/acorn) produces a JavaScript @@ -126,11 +126,11 @@ context. Syntactic candidates are never labeled as observed executions. From the repository root, with a previously frozen dataset and fresh output paths: ```powershell -python experiments/command_specialist/recover_code.py --root work/command-specialist/data-v2 --frozen work/command-specialist/data-v2/frozen-review-final --out work/command-specialist/code-recovery-new -python experiments/command_specialist/recover_reads.py --root work/command-specialist/data-v2 --frozen work/command-specialist/data-v2/frozen-review-final --code-recovery work/command-specialist/code-recovery-new --out work/command-specialist/read-recovery-new -python experiments/command_specialist/verify_labels.py --frozen work/command-specialist/data-v2/frozen-review-final --read-recovery work/command-specialist/read-recovery-new --out work/command-specialist/verified-reads-new -python experiments/command_specialist/verify_more_operations.py --out work/command-specialist/verified-operations-new -python experiments/command_specialist/prepare_expanded.py --pilot work/command-specialist --verified work/command-specialist/verified-reads-new --frozen work/command-specialist/data-v2/frozen-review-final --out work/command-specialist/expanded-data-new +python experiments/command_model/recover_code.py --root work/command-specialist/data-v2 --frozen work/command-specialist/data-v2/frozen-review-final --out work/command-specialist/code-recovery-new +python experiments/command_model/recover_reads.py --root work/command-specialist/data-v2 --frozen work/command-specialist/data-v2/frozen-review-final --code-recovery work/command-specialist/code-recovery-new --out work/command-specialist/read-recovery-new +python experiments/command_model/verify_labels.py --frozen work/command-specialist/data-v2/frozen-review-final --read-recovery work/command-specialist/read-recovery-new --out work/command-specialist/verified-reads-new +python experiments/command_model/verify_more_operations.py --out work/command-specialist/verified-operations-new +python experiments/command_model/prepare_expanded.py --pilot work/command-specialist --verified work/command-specialist/verified-reads-new --frozen work/command-specialist/data-v2/frozen-review-final --out work/command-specialist/expanded-data-new ``` The PowerShell recovery stage uses the SDK's @@ -146,7 +146,7 @@ Recovery and export artifacts carry hashes tying them to the frozen training fil The expanded preparer reuses only the pilot's synthetic training rows, excluding its older mined examples. It never reads pilot evaluation files or frozen non-training partition files. Do not manually add historical examples from a different split. -Run `python -m unittest discover -s experiments/command_specialist -p 'test_*.py'` +Run `python -m unittest discover -s experiments/command_model -p 'test_*.py'` after installing the parser dependency. See [RECOVERY_RESULTS.md](RECOVERY_RESULTS.md) for measured coverage, failures, and scope. diff --git a/experiments/command_specialist/DATA_RESULTS.md b/experiments/command_model/DATA_RESULTS.md similarity index 100% rename from experiments/command_specialist/DATA_RESULTS.md rename to experiments/command_model/DATA_RESULTS.md diff --git a/experiments/command_specialist/ENGLISH_HANDOFF.md b/experiments/command_model/ENGLISH_HANDOFF.md similarity index 98% rename from experiments/command_specialist/ENGLISH_HANDOFF.md rename to experiments/command_model/ENGLISH_HANDOFF.md index ee381ef..5344689 100644 --- a/experiments/command_specialist/ENGLISH_HANDOFF.md +++ b/experiments/command_model/ENGLISH_HANDOFF.md @@ -34,7 +34,7 @@ This verifies the stated stdout condition, not arbitrary semantic correctness. From the repository root, invoke Python 3.12: ``` -python experiments/command_specialist/delegate.py --task-file task.json --root scratch --allow-execute +python experiments/command_model/delegate.py --task-file task.json --root scratch --allow-execute ``` Example handoff (scratch must already exist): diff --git a/experiments/command_specialist/ENGLISH_RESULTS.md b/experiments/command_model/ENGLISH_RESULTS.md similarity index 100% rename from experiments/command_specialist/ENGLISH_RESULTS.md rename to experiments/command_model/ENGLISH_RESULTS.md diff --git a/experiments/command_specialist/EVIDENCE_DIAGNOSTICS.md b/experiments/command_model/EVIDENCE_DIAGNOSTICS.md similarity index 100% rename from experiments/command_specialist/EVIDENCE_DIAGNOSTICS.md rename to experiments/command_model/EVIDENCE_DIAGNOSTICS.md diff --git a/experiments/command_specialist/PURPOSE.md b/experiments/command_model/PURPOSE.md similarity index 97% rename from experiments/command_specialist/PURPOSE.md rename to experiments/command_model/PURPOSE.md index a5dc098..307f86a 100644 --- a/experiments/command_specialist/PURPOSE.md +++ b/experiments/command_model/PURPOSE.md @@ -1,4 +1,7 @@ -# Command specialist: purpose and evidence +# Command Model: purpose and evidence + +Command Model is the project name. Shell Gatherer collects observations; separate +mining, labeling and verification processes produce saved datasets. This is the current project direction. Historical reports describe their own experiments; they do not redefine the product. The goal is to reduce frontier diff --git a/experiments/command_specialist/README.md b/experiments/command_model/README.md similarity index 91% rename from experiments/command_specialist/README.md rename to experiments/command_model/README.md index d0a2265..86a7c2b 100644 --- a/experiments/command_specialist/README.md +++ b/experiments/command_model/README.md @@ -1,4 +1,4 @@ -# Command specialist +# Command Model Start with [project purpose and evidence](PURPOSE.md) for the current goal, frontier/worker responsibilities, training scope and next experiment. This page @@ -51,7 +51,7 @@ A caller can hand off a UTF-8 JSON task file: ``` ```powershell -python experiments/command_specialist/run.py --root '' --task-file task.json --backend native +python experiments/command_model/run.py --root '' --task-file task.json --backend native ``` Runtime limits are explicit CLI options and keyword arguments on `inspect_request`. @@ -59,7 +59,7 @@ Defaults remain `--num-ctx 4096 --num-predict 160` for compatibility. To evaluat more context/output capacity with the same model and adapter: ```powershell -python experiments/command_specialist/run.py --root '' --task-file task.json --backend native --num-ctx 8192 --num-predict 2048 +python experiments/command_model/run.py --root '' --task-file task.json --backend native --num-ctx 8192 --num-predict 2048 ``` These values reach **both** planning and evidence requests; Modelfile defaults do @@ -110,7 +110,7 @@ See [binding measurements](BINDING_RESULTS.md). Run the paired live-model experi with new filenames and two reference candidates per task: ```powershell -python experiments/command_specialist/benchmark_bindings.py --out work/command-specialist/binding-trial --backend native +python experiments/command_model/benchmark_bindings.py --out work/command-specialist/binding-trial --backend native ``` The test alternates bound/unbound order and compares executed output against an @@ -119,7 +119,7 @@ final-test observations and does not prove file-discovery accuracy. ## Why this design -The shell-forensics corpus exposes expensive mechanical failures: wrong dialects, +The Shell Gatherer observation corpus exposes expensive mechanical failures: wrong dialects, lost exit codes, nested quoting, and irrelevant output. Removing those failures with templates and parsers is useful even before training. A local model should interpret ambiguous requests when fixed code cannot. Sending it an already exact @@ -165,14 +165,14 @@ the system Python. Training uses the separate environment. The recorded pilot used CUDA PyTorch 2.11.0+cu128, Transformers 4.57.6, PEFT 0.20.0, and Ollama 0.33.3. ```powershell -python -m unittest discover -s experiments/command_specialist -p 'test_*.py' -v -python experiments/command_specialist/prepare.py --corpus '' -python experiments/command_specialist/extract_full.py +python -m unittest discover -s experiments/command_model -p 'test_*.py' -v +python experiments/command_model/prepare.py --corpus '' +python experiments/command_model/extract_full.py ollama pull qwen2.5-coder:1.5b ollama pull qwen3.5:0.8b -python experiments/command_specialist/benchmark.py --model qwen2.5-coder:1.5b -python experiments/command_specialist/benchmark.py --model qwen3.5:0.8b -python experiments/command_specialist/benchmark.py --model rules +python experiments/command_model/benchmark.py --model qwen2.5-coder:1.5b +python experiments/command_model/benchmark.py --model qwen3.5:0.8b +python experiments/command_model/benchmark.py --model rules ``` Do not run inference benchmarks alongside GPU training. Keep models warm for the @@ -187,7 +187,7 @@ python -m venv .venv .venv/Scripts/python.exe -m pip install --no-cache-dir transformers==4.57.6 peft==0.20.0 accelerate==1.15.0 safetensors ollama stop qwen2.5-coder:1.5b ollama stop qwen3.5:0.8b -.venv/Scripts/python.exe experiments/command_specialist/train.py +.venv/Scripts/python.exe experiments/command_model/train.py ``` Training uses response-only loss, rank-16 LoRA on attention projections, BF16 frozen @@ -212,7 +212,7 @@ preserved. The same applies to dataset and adapter output directories. Once the model is created, try a read-only request against an explicitly chosen root: ```powershell -python experiments/command_specialist/run.py --root '' --request 'Read the last 5 lines of "build.log".' +python experiments/command_model/run.py --root '' --request 'Read the last 5 lines of "build.log".' ``` The CLI prints the plan, result, status, and measured latency. It also saves the full @@ -228,8 +228,8 @@ that arbitrary commands can run without a shell. Native Unicode comparison, encoding handling, and JSON behavior are not a complete PowerShell emulation. ```powershell -python experiments/command_specialist/benchmark.py --model shell-specialist-pilot --backend native --label trained-native -python experiments/command_specialist/run.py --backend native --root '' --request 'Read the last 5 lines of "build.log".' +python experiments/command_model/benchmark.py --model shell-specialist-pilot --backend native --label trained-native +python experiments/command_model/run.py --backend native --root '' --request 'Read the last 5 lines of "build.log".' ``` ## What the scores do and do not mean diff --git a/experiments/command_specialist/RECOVERY_RESULTS.md b/experiments/command_model/RECOVERY_RESULTS.md similarity index 100% rename from experiments/command_specialist/RECOVERY_RESULTS.md rename to experiments/command_model/RECOVERY_RESULTS.md diff --git a/experiments/command_specialist/RESEARCH.md b/experiments/command_model/RESEARCH.md similarity index 99% rename from experiments/command_specialist/RESEARCH.md rename to experiments/command_model/RESEARCH.md index 88e456f..6ab27a0 100644 --- a/experiments/command_specialist/RESEARCH.md +++ b/experiments/command_model/RESEARCH.md @@ -1,4 +1,4 @@ -# Starting point: a local command specialist +# Starting point: a local Command Model A useful local specialist is feasible on an 8 GB RTX 3070. Beating a frontier model on a narrow operation is a testable hypothesis. Beating frontier models generally, diff --git a/experiments/command_specialist/RESULTS.md b/experiments/command_model/RESULTS.md similarity index 100% rename from experiments/command_specialist/RESULTS.md rename to experiments/command_model/RESULTS.md diff --git a/experiments/command_specialist/RUNTIME_RESULTS.md b/experiments/command_model/RUNTIME_RESULTS.md similarity index 100% rename from experiments/command_specialist/RUNTIME_RESULTS.md rename to experiments/command_model/RUNTIME_RESULTS.md diff --git a/experiments/command_specialist/benchmark.py b/experiments/command_model/benchmark.py similarity index 100% rename from experiments/command_specialist/benchmark.py rename to experiments/command_model/benchmark.py diff --git a/experiments/command_specialist/benchmark_bindings.py b/experiments/command_model/benchmark_bindings.py similarity index 100% rename from experiments/command_specialist/benchmark_bindings.py rename to experiments/command_model/benchmark_bindings.py diff --git a/experiments/command_specialist/bindings.py b/experiments/command_model/bindings.py similarity index 100% rename from experiments/command_specialist/bindings.py rename to experiments/command_model/bindings.py diff --git a/experiments/command_specialist/codex/README.md b/experiments/command_model/codex/README.md similarity index 90% rename from experiments/command_specialist/codex/README.md rename to experiments/command_model/codex/README.md index 456668b..a715fee 100644 --- a/experiments/command_specialist/codex/README.md +++ b/experiments/command_model/codex/README.md @@ -14,13 +14,13 @@ two local models. It is a Windows, trusted-workspace Python-task prototype. Install into an isolated Python 3.12 environment: ``` -python -m pip install -r experiments/command_specialist/codex/requirements.txt -python experiments/command_specialist/codex/bench.py prepare --case csv --arm delegated +python -m pip install -r experiments/command_model/codex/requirements.txt +python experiments/command_model/codex/bench.py prepare --case csv --arm delegated ``` Open the printed directory's `workspace` folder as a project in Codex, start a fresh chat, and send the text from the adjacent `prompt.txt`. The project-local -`.codex/config.toml` connects `command_specialist.run_python_task`. Trust that +`.codex/config.toml` connects `command_model.run_python_task`. Trust that specific project when Codex requests it. The tool executes generated Python with the account's privileges, so approve only the intended trusted fixture operation. No global configuration is edited. Desktop tool loading must be verified in the @@ -39,8 +39,8 @@ may remain marked running and must never be treated as success. Run each arm separately (never concurrently) using the same model and effort: ``` -python experiments/command_specialist/codex/bench.py run --case csv --arm baseline --model gpt-6-astra --effort low -python experiments/command_specialist/codex/bench.py run --case csv --arm delegated --model gpt-6-astra --effort low +python experiments/command_model/codex/bench.py run --case csv --arm baseline --model gpt-6-astra --effort low +python experiments/command_model/codex/bench.py run --case csv --arm delegated --model gpt-6-astra --effort low ``` `repair` is a second case with an actual broken program. Each run creates an @@ -60,7 +60,7 @@ must not be assigned CLI token/timing figures. Create a comparison from the two saved summaries: ``` -python experiments/command_specialist/codex/compare.py BASELINE/summary.json DELEGATED/summary.json --out work/comparison.json +python experiments/command_model/codex/compare.py BASELINE/summary.json DELEGATED/summary.json --out work/comparison.json ``` The runner checks the saved program on the original input and an alternate input, @@ -123,7 +123,7 @@ sandboxed by the harness. Compare saved runs with: ``` -python experiments/command_specialist/codex/compare_ten.py BASELINE/summary.json CHAINED/summary.json GROUPED/summary.json --out work/ten-stage-comparison.json +python experiments/command_model/codex/compare_ten.py BASELINE/summary.json CHAINED/summary.json GROUPED/summary.json --out work/ten-stage-comparison.json ``` Keep each summary beside its original run.json and fixture.json. Savings require diff --git a/experiments/command_specialist/codex/RESULTS.md b/experiments/command_model/codex/RESULTS.md similarity index 100% rename from experiments/command_specialist/codex/RESULTS.md rename to experiments/command_model/codex/RESULTS.md diff --git a/experiments/command_specialist/codex/TEN-STAGE-PILOT.md b/experiments/command_model/codex/TEN-STAGE-PILOT.md similarity index 100% rename from experiments/command_specialist/codex/TEN-STAGE-PILOT.md rename to experiments/command_model/codex/TEN-STAGE-PILOT.md diff --git a/experiments/command_specialist/codex/bench.py b/experiments/command_model/codex/bench.py similarity index 96% rename from experiments/command_specialist/codex/bench.py rename to experiments/command_model/codex/bench.py index 882be76..64b807a 100644 --- a/experiments/command_specialist/codex/bench.py +++ b/experiments/command_model/codex/bench.py @@ -37,22 +37,22 @@ def prepare(out, case, arm): data = CASES[case] for name, content in data['files'].items(): (root / name).write_text(content, encoding='utf-8', newline='') - # An independent repository prevents inheriting shell-forensics release chores. + # An independent repository prevents inheriting Command Model release chores. subprocess.run(['git', 'init', '--quiet', str(root)], check=True) - instructions = ('This is an isolated command-specialist benchmark fixture, not an implementation project. ' + instructions = ('This is an isolated Command Model benchmark fixture, not an implementation project. ' 'Do only the requested operation; no commits, PRs, dependency installs or unrelated browsing. ' 'Modify only report.py. Read the provided input file. Use Python 3.12 standard library. ' 'Report observed failures and any fallback honestly. Do not fabricate results.\n') - instructions += ('Use normal native shell/file tools. Do not invoke a local model or command-specialist.\n' + instructions += ('Use normal native shell/file tools. Do not invoke a local model or Command Model.\n' if arm == 'baseline' else - 'Call command_specialist.run_python_task once with English intent, exact target report.py, ' + 'Call command_model.run_python_task once with English intent, exact target report.py, ' 'known context and expected stdout. Do not write source or command sequences in that handoff. ' 'If it fails, report failure; do not silently use another executor.\n') (root / 'AGENTS.md').write_text(instructions, encoding='utf-8') prompt = data['task'] + '\n' + data['context'] + '\nExpected stdout: ' + json.dumps(data['stdout']) (directory / 'prompt.txt').write_text(prompt, encoding='utf-8') if arm == 'delegated': - config = ('[mcp_servers.command_specialist]\n' + config = ('[mcp_servers.command_model]\n' f'command = {json.dumps(sys.executable)}\n' f'args = {json.dumps([str(HERE / "server.py"), "--root", str(root), "--artifacts", str(directory / "local"), "--allow-execute"])}\n' 'required = true\nstartup_timeout_sec = 30\ntool_timeout_sec = 330\n' diff --git a/experiments/command_specialist/codex/compare.py b/experiments/command_model/codex/compare.py similarity index 100% rename from experiments/command_specialist/codex/compare.py rename to experiments/command_model/codex/compare.py diff --git a/experiments/command_specialist/codex/compare_ten.py b/experiments/command_model/codex/compare_ten.py similarity index 100% rename from experiments/command_specialist/codex/compare_ten.py rename to experiments/command_model/codex/compare_ten.py diff --git a/experiments/command_specialist/codex/requirements.txt b/experiments/command_model/codex/requirements.txt similarity index 100% rename from experiments/command_specialist/codex/requirements.txt rename to experiments/command_model/codex/requirements.txt diff --git a/experiments/command_specialist/codex/server.py b/experiments/command_model/codex/server.py similarity index 98% rename from experiments/command_specialist/codex/server.py rename to experiments/command_model/codex/server.py index d87bf53..0053057 100644 --- a/experiments/command_specialist/codex/server.py +++ b/experiments/command_model/codex/server.py @@ -19,7 +19,7 @@ def main(): parser.add_argument('--allow-execute', action='store_true') args = parser.parse_args() root = args.root.resolve(strict=True) - server = FastMCP('command-specialist', log_level='WARNING') + server = FastMCP('command-model', log_level='WARNING') lock = asyncio.Lock() @server.tool(annotations=ToolAnnotations(readOnlyHint=False, destructiveHint=True, diff --git a/experiments/command_specialist/codex/ten_step.py b/experiments/command_model/codex/ten_step.py similarity index 94% rename from experiments/command_specialist/codex/ten_step.py rename to experiments/command_model/codex/ten_step.py index 389d235..39275b8 100644 --- a/experiments/command_specialist/codex/ten_step.py +++ b/experiments/command_model/codex/ten_step.py @@ -29,7 +29,7 @@ def prepare(out, arm): if arm == 'baseline': guidance += 'Use normal native shell/file tools and build_packet.py. Batch work as efficiently as you judge useful; there is no required number of shell calls. Do not use a local model.\n' else: - guidance += ('Use command_specialist.run_python_task with English intent, exact script target, relevant context and exact expected stdout. ' + guidance += ('Use command_model.run_python_task with English intent, exact script target, relevant context and exact expected stdout. ' 'Do not write source or command sequences yourself. Call serially; each call is a fresh worker. ' 'After a failure, preserve it and continue later requested groups when possible; do not switch to native execution or silently retry in another worker.\n') guidance += ('Make exactly ten handoffs, one per numbered stage.\n' if arm == 'chained' else 'Make exactly two handoffs: stages 1–5 together, then stages 6–10 together.\n') @@ -48,7 +48,7 @@ def prepare(out, arm): (directory / 'prompt.txt').write_text(task, encoding='utf-8') if arm != 'baseline': (root / '.codex').mkdir() - (root / '.codex/config.toml').write_text('[mcp_servers.command_specialist]\ncommand = '+json.dumps(sys.executable)+'\nargs = '+json.dumps([str(HERE/'server.py'),'--root',str(root),'--artifacts',str(directory/'local'),'--allow-execute'])+'\nrequired = true\nstartup_timeout_sec = 30\ntool_timeout_sec = 330\ndefault_tools_approval_mode = "prompt"\n', encoding='utf-8') + (root / '.codex/config.toml').write_text('[mcp_servers.command_model]\ncommand = '+json.dumps(sys.executable)+'\nargs = '+json.dumps([str(HERE/'server.py'),'--root',str(root),'--artifacts',str(directory/'local'),'--allow-execute'])+'\nrequired = true\nstartup_timeout_sec = 30\ntool_timeout_sec = 330\ndefault_tools_approval_mode = "prompt"\n', encoding='utf-8') write_json(directory/'fixture.json', {'case':'ten-stage-incident','arm':arm,'scripts':scripts, 'input_hashes':{p.relative_to(root).as_posix():hashlib.sha256(p.read_bytes()).hexdigest() for p in (root/'inputs').rglob('*') if p.is_file()}}) return directory @@ -111,7 +111,7 @@ def collect(directory): 'output_tokens':sum(c['response'].get('eval_count',0) for c in r['model_calls']), 'failed_executions':sum(a.get('result',{}).get('exit_code',0)!=0 for a in r['actions'])}) expected_calls={'baseline':0,'chained':10,'grouped':2}[fixture['arm']] - mcp=[i for i in completed if i.get('type')=='mcp_tool_call' and i.get('server')=='command_specialist'] + mcp=[i for i in completed if i.get('type')=='mcp_tool_call' and i.get('server')=='command_model'] inputs_unchanged=all((root/name).exists() and hashlib.sha256((root/name).read_bytes()).hexdigest()==value for name,value in fixture['input_hashes'].items()) success=bool(turns and run_info['exit_code']==0 and not run_info['source_changed_during_run'] and inputs_unchanged and all(c['passed'] for c in checks) and all(c['passed'] for c in alternate['checks']) diff --git a/experiments/command_specialist/codex/ten_step_workload.py b/experiments/command_model/codex/ten_step_workload.py similarity index 100% rename from experiments/command_specialist/codex/ten_step_workload.py rename to experiments/command_model/codex/ten_step_workload.py diff --git a/experiments/command_specialist/codex/test_compare.py b/experiments/command_model/codex/test_compare.py similarity index 100% rename from experiments/command_specialist/codex/test_compare.py rename to experiments/command_model/codex/test_compare.py diff --git a/experiments/command_specialist/contract.py b/experiments/command_model/contract.py similarity index 100% rename from experiments/command_specialist/contract.py rename to experiments/command_model/contract.py diff --git a/experiments/command_specialist/data_pipeline.py b/experiments/command_model/data_pipeline.py similarity index 100% rename from experiments/command_specialist/data_pipeline.py rename to experiments/command_model/data_pipeline.py diff --git a/experiments/command_specialist/delegate.py b/experiments/command_model/delegate.py similarity index 100% rename from experiments/command_specialist/delegate.py rename to experiments/command_model/delegate.py diff --git a/experiments/command_specialist/extract_full.py b/experiments/command_model/extract_full.py similarity index 100% rename from experiments/command_specialist/extract_full.py rename to experiments/command_model/extract_full.py diff --git a/experiments/command_specialist/ingest_adapters.py b/experiments/command_model/ingest_adapters.py similarity index 100% rename from experiments/command_specialist/ingest_adapters.py rename to experiments/command_model/ingest_adapters.py diff --git a/experiments/command_specialist/js_parser/extract.mjs b/experiments/command_model/js_parser/extract.mjs similarity index 100% rename from experiments/command_specialist/js_parser/extract.mjs rename to experiments/command_model/js_parser/extract.mjs diff --git a/experiments/command_specialist/js_parser/extract.test.mjs b/experiments/command_model/js_parser/extract.test.mjs similarity index 100% rename from experiments/command_specialist/js_parser/extract.test.mjs rename to experiments/command_model/js_parser/extract.test.mjs diff --git a/experiments/command_specialist/js_parser/package-lock.json b/experiments/command_model/js_parser/package-lock.json similarity index 100% rename from experiments/command_specialist/js_parser/package-lock.json rename to experiments/command_model/js_parser/package-lock.json diff --git a/experiments/command_specialist/js_parser/package.json b/experiments/command_model/js_parser/package.json similarity index 100% rename from experiments/command_specialist/js_parser/package.json rename to experiments/command_model/js_parser/package.json diff --git a/experiments/command_specialist/label_data.py b/experiments/command_model/label_data.py similarity index 100% rename from experiments/command_specialist/label_data.py rename to experiments/command_model/label_data.py diff --git a/experiments/command_specialist/parse_read_commands.ps1 b/experiments/command_model/parse_read_commands.ps1 similarity index 100% rename from experiments/command_specialist/parse_read_commands.ps1 rename to experiments/command_model/parse_read_commands.ps1 diff --git a/experiments/command_specialist/prepare.py b/experiments/command_model/prepare.py similarity index 100% rename from experiments/command_specialist/prepare.py rename to experiments/command_model/prepare.py diff --git a/experiments/command_specialist/prepare_expanded.py b/experiments/command_model/prepare_expanded.py similarity index 100% rename from experiments/command_specialist/prepare_expanded.py rename to experiments/command_model/prepare_expanded.py diff --git a/experiments/command_specialist/prepare_read_probe.py b/experiments/command_model/prepare_read_probe.py similarity index 100% rename from experiments/command_specialist/prepare_read_probe.py rename to experiments/command_model/prepare_read_probe.py diff --git a/experiments/command_specialist/recover_code.py b/experiments/command_model/recover_code.py similarity index 100% rename from experiments/command_specialist/recover_code.py rename to experiments/command_model/recover_code.py diff --git a/experiments/command_specialist/recover_reads.py b/experiments/command_model/recover_reads.py similarity index 100% rename from experiments/command_specialist/recover_reads.py rename to experiments/command_model/recover_reads.py diff --git a/experiments/command_specialist/review_queue.py b/experiments/command_model/review_queue.py similarity index 100% rename from experiments/command_specialist/review_queue.py rename to experiments/command_model/review_queue.py diff --git a/experiments/command_specialist/run.py b/experiments/command_model/run.py similarity index 100% rename from experiments/command_specialist/run.py rename to experiments/command_model/run.py diff --git a/experiments/command_specialist/test_bindings.py b/experiments/command_model/test_bindings.py similarity index 100% rename from experiments/command_specialist/test_bindings.py rename to experiments/command_model/test_bindings.py diff --git a/experiments/command_specialist/test_contract.py b/experiments/command_model/test_contract.py similarity index 100% rename from experiments/command_specialist/test_contract.py rename to experiments/command_model/test_contract.py diff --git a/experiments/command_specialist/test_data.py b/experiments/command_model/test_data.py similarity index 100% rename from experiments/command_specialist/test_data.py rename to experiments/command_model/test_data.py diff --git a/experiments/command_specialist/test_delegate.py b/experiments/command_model/test_delegate.py similarity index 100% rename from experiments/command_specialist/test_delegate.py rename to experiments/command_model/test_delegate.py diff --git a/experiments/command_specialist/test_expanded.py b/experiments/command_model/test_expanded.py similarity index 100% rename from experiments/command_specialist/test_expanded.py rename to experiments/command_model/test_expanded.py diff --git a/experiments/command_specialist/test_pipeline.py b/experiments/command_model/test_pipeline.py similarity index 100% rename from experiments/command_specialist/test_pipeline.py rename to experiments/command_model/test_pipeline.py diff --git a/experiments/command_specialist/test_recovery.py b/experiments/command_model/test_recovery.py similarity index 100% rename from experiments/command_specialist/test_recovery.py rename to experiments/command_model/test_recovery.py diff --git a/experiments/command_specialist/token_audit.py b/experiments/command_model/token_audit.py similarity index 100% rename from experiments/command_specialist/token_audit.py rename to experiments/command_model/token_audit.py diff --git a/experiments/command_specialist/train.py b/experiments/command_model/train.py similarity index 100% rename from experiments/command_specialist/train.py rename to experiments/command_model/train.py diff --git a/experiments/command_specialist/verify_labels.py b/experiments/command_model/verify_labels.py similarity index 99% rename from experiments/command_specialist/verify_labels.py rename to experiments/command_model/verify_labels.py index af6c371..19f7cac 100644 --- a/experiments/command_specialist/verify_labels.py +++ b/experiments/command_model/verify_labels.py @@ -148,7 +148,7 @@ def export_train(frozen, out, verification, read_recovery=None): "used_partitions": ["train"], "verifier_cases": verification["passed"], "sft_sha256": hashlib.sha256((out / "sft.jsonl").read_bytes()).hexdigest(), "original_task_success_labels": 0, - "scope": "Supplemental synthetic read examples; insufficient alone for a broad command specialist."} + "scope": "Supplemental synthetic read examples; insufficient alone for a broad Command Model."} atomic_json(out / "export.json", report) return report diff --git a/experiments/command_specialist/verify_more_operations.ps1 b/experiments/command_model/verify_more_operations.ps1 similarity index 100% rename from experiments/command_specialist/verify_more_operations.ps1 rename to experiments/command_model/verify_more_operations.ps1 diff --git a/experiments/command_specialist/verify_more_operations.py b/experiments/command_model/verify_more_operations.py similarity index 100% rename from experiments/command_specialist/verify_more_operations.py rename to experiments/command_model/verify_more_operations.py diff --git a/experiments/command_specialist/verify_read_contracts.ps1 b/experiments/command_model/verify_read_contracts.ps1 similarity index 100% rename from experiments/command_specialist/verify_read_contracts.ps1 rename to experiments/command_model/verify_read_contracts.ps1 diff --git a/scripts/build.py b/scripts/build.py index b5e34c3..7579487 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -185,7 +185,7 @@ def ex(r, n=520): tpl = open(os.path.join(SCRIPT, 'template.html'), encoding='utf-8').read() js = json.dumps(data, ensure_ascii=False, default=str).replace(' n -HEAD = 700 +"""Compatibility entry point for Shell Gatherer; preserves the original CLI.""" +from pathlib import Path +import runpy -def head(s, n=HEAD): - if s is None: return '' - if not isinstance(s, str): s = json.dumps(s) - return s[:n] - -# ---------- JS cell parsing (Codex exec / OpenCode execute) ---------- -def scan_calls(js): - """yield (toolname, argtext) for tools.NAME( ... ) with balanced parens, string-aware""" - out = [] - for m in re.finditer(r'tools\.([A-Za-z_][\w]*)\s*\(', js): - name = m.group(1); i = m.end(); depth = 1; j = i; n = len(js) - q = None - while j < n and depth > 0: - ch = js[j] - if q: - if ch == '\\': j += 2; continue - if ch == q: q = None - else: - if ch in '"\'`': q = ch - elif ch in '([{': depth += 1 - elif ch in ')]}': depth -= 1 - j += 1 - out.append((name, js[i:j-1].strip())) - return out - -STR = r'"((?:[^"\\]|\\.)*)"|\'((?:[^\'\\]|\\.)*)\'|`((?:[^`\\]|\\.)*)`' -def js_str(m): - if m.group(1) is not None: - try: return json.loads('"' + m.group(1) + '"') - except Exception: return m.group(1) - if m.group(2) is not None: return m.group(2).replace("\\'", "'") - return m.group(3) - -def parse_args(argtext): - """return dict-ish of interesting fields from a JS object literal""" - d = {} - try: - o = json.loads(argtext) - if isinstance(o, dict): return o - except Exception: pass - for key in ('cmd', 'command', 'shell', 'workdir', 'yield_time_ms', 'timeout_ms', 'code'): - m = re.search(r'(?:^|[{,\s])["\']?' + key + r'["\']?\s*:\s*(' + STR + ')', argtext) - if m: - d[key] = js_str(re.match(STR, m.group(1))) - elif re.search(r'(?:^|[{,\s])' + key + r'\s*(?:,|})', argtext): - d[key] = None; d[key + '_dynamic'] = True - if re.search(r'(?:^|[{,\s])["\']?(cmd|command)["\']?\s*:\s*[A-Za-z_$]', argtext) or re.search(r'\$\{', argtext): - d['_dynamic'] = True - return d - -SHELLISH = {'exec_command', 'shell_command', 'shell', 'bash', 'oc_bash'} - -def cell_records(js, base): - """turn one JS cell into shell records (+ counts of every tools.* call).""" - calls = scan_calls(js) - shell = [] - for name, argtext in calls: - toolcounts[(base['source'], base['model'], 'jscell:' + name)] += 1 - if name in SHELLISH: - a = parse_args(argtext) - cmd = a.get('cmd') if a.get('cmd') is not None else a.get('command') - shell.append({'cmd': cmd, 'shell_param': a.get('shell'), 'workdir': a.get('workdir'), - 'dynamic': bool(a.get('_dynamic') or a.get('cmd_dynamic') or a.get('command_dynamic')), - 'yield_ms': a.get('yield_time_ms')}) - return calls, shell - -def exit_codes_from_output(txt): - return [int(x) for x in re.findall(r'"exit_code"\s*:\s*(-?\d+)', txt)] - -# ---------- CODEX ---------- -def do_codex(): - files = glob.glob(HOME + '/.codex/sessions/**/*.jsonl', recursive=True) + glob.glob(HOME + '/.codex/archived_sessions/**/*.jsonl', recursive=True) - seen_calls = set() - for f in files: - sid = os.path.basename(f)[:-6] - turn_model = {}; meta = {} - pending = {} # call_id -> record stub - try: - lines = open(f, encoding='utf-8', errors='replace').read().splitlines() - except Exception as e: - print('skip', f, e); continue - for line in lines: - try: o = json.loads(line) - except Exception: continue - t = o.get('type'); p = o.get('payload') or {} - ts = o.get('timestamp') - if t == 'session_meta': - meta = {'cwd': p.get('cwd'), 'originator': p.get('originator'), 'cli': p.get('cli_version')} - elif t == 'turn_context': - turn_model[p.get('turn_id')] = p.get('model') - elif t == 'response_item': - pt = p.get('type') - if pt in ('custom_tool_call', 'function_call'): - cid = p.get('call_id') - if cid in seen_calls: continue - seen_calls.add(cid) - tid = (p.get('internal_chat_message_metadata_passthrough') or {}).get('turn_id') - model = turn_model.get(tid) or (list(turn_model.values())[-1] if turn_model else None) - base = {'source': 'codex', 'model': model, 'session': sid, 'ts': ts, 'cwd': meta.get('cwd'), 'originator': meta.get('originator')} - name = p.get('name') - toolcounts[('codex', model, ('exec' if pt == 'custom_tool_call' else 'fn:') + ('' if pt == 'custom_tool_call' else name))] += 1 - if pt == 'custom_tool_call' and name == 'exec': - js = p.get('input') or '' - calls, shell = cell_records(js, base) - pending[cid] = dict(base, kind='jscell', js=js, calls=[c[0] for c in calls], shell=shell) - elif pt == 'function_call' and name in ('shell_command', 'exec_command'): - try: a = json.loads(p.get('arguments') or '{}') - except Exception: a = {} - pending[cid] = dict(base, kind='fn', tool=name, shell=[{'cmd': a.get('cmd') or a.get('command'), 'shell_param': a.get('shell'), 'workdir': a.get('workdir'), 'dynamic': False, 'yield_ms': a.get('yield_time_ms')}], js=None, calls=[name]) - elif pt in ('custom_tool_call_output', 'function_call_output'): - cid = p.get('call_id'); stub = pending.pop(cid, None) - if not stub: continue - outp = p.get('output') - if isinstance(outp, list): txt = '\n'.join(x.get('text', '') for x in outp if isinstance(x, dict)) - else: txt = str(outp or '') - failed = txt.startswith('Script failed') or txt.startswith('Script error') - codes = exit_codes_from_output(txt) - wall = re.search(r'Wall time:? ([\d.]+)', txt) - emit_codex(stub, txt, failed, codes, wall.group(1) if wall else None) - # unmatched pending (no output recorded) - for cid, stub in pending.items(): - emit_codex(stub, '', None, [], None, unmatched=True) - -def emit_codex(stub, txt, failed, codes, wall, unmatched=False): - shell = stub['shell'] - if stub['kind'] == 'jscell' and not shell: - # a pure-JS cell (no shell call): record it as a js-only action - recs.append({'source': 'codex', 'model': stub['model'], 'session': stub['session'], 'ts': stub['ts'], 'cwd': stub['cwd'], - 'tool': 'exec(js-only)', 'via_js_cell': True, 'js_tools': stub['calls'], 'cmd': None, 'js': head(stub['js'], 1200), - 'shell_param': None, 'dynamic': False, 'ok': (None if unmatched else (not failed)), 'exit': None, - 'out': head(txt), 'wall': wall, 'unmatched': unmatched}) - return - for i, s in enumerate(shell): - code = codes[i] if i < len(codes) and len(codes) == len(shell) else (codes[0] if len(codes) == 1 and len(shell) == 1 else None) - if unmatched: ok = None - elif failed: ok = False - elif code is not None: ok = (code == 0) - else: ok = True # script completed; exit code not surfaced by the model - recs.append({'source': 'codex', 'model': stub['model'], 'session': stub['session'], 'ts': stub['ts'], 'cwd': stub['cwd'], - 'tool': ('exec>' + 'exec_command' if stub['kind'] == 'jscell' else stub['tool']), 'via_js_cell': stub['kind'] == 'jscell', - 'js_tools': stub['calls'], 'cmd': s['cmd'], 'js': head(stub['js'], 1200) if stub['kind'] == 'jscell' else None, - 'shell_param': s['shell_param'], 'workdir': s['workdir'], 'dynamic': s['dynamic'], 'yield_ms': s['yield_ms'], - 'ok': ok, 'exit': code, 'exit_visible': code is not None, 'script_failed': failed, - 'out': head(txt), 'wall': wall, 'unmatched': unmatched, 'n_in_cell': len(shell), 'idx_in_cell': i, - 'backgrounded': ('"session_id"' in txt and code is None)}) - -# ---------- CLAUDE CODE ---------- -def do_claude(): - files = glob.glob(HOME + '/.claude/projects/**/*.jsonl', recursive=True) - seen = set() - for f in files: - sid = os.path.basename(f)[:-6] - proj = f.split('projects')[1].split(os.sep)[1] if 'projects' in f else '' - pending = {} - try: instructed = 'Do your work through the Bash tool' in open(f, encoding='utf-8', errors='replace').read() - except Exception: instructed = False - for line in open(f, encoding='utf-8', errors='replace'): - try: o = json.loads(line) - except Exception: continue - m = o.get('message') - if not isinstance(m, dict): continue - cont = m.get('content') - if not isinstance(cont, list): continue - if o.get('type') == 'assistant': - model = m.get('model') - for b in cont: - if not isinstance(b, dict) or b.get('type') != 'tool_use': continue - tid = b.get('id') - if tid in seen: continue - seen.add(tid) - name = b.get('name') - toolcounts[('claude-code', model, name)] += 1 - if name in ('Bash', 'PowerShell'): - inp = b.get('input') or {} - pending[tid] = {'source': 'claude-code', 'model': model, 'session': sid, 'project': proj, 'ts': o.get('timestamp'), 'cwd': o.get('cwd'), - 'tool': name, 'via_js_cell': False, 'cmd': inp.get('command'), 'desc': inp.get('description'), - 'sidechain': bool(o.get('isSidechain')), 'bg': bool(inp.get('run_in_background')), 'timeout': inp.get('timeout'), 'instructed': instructed} - elif o.get('type') == 'user': - for b in cont: - if not isinstance(b, dict) or b.get('type') != 'tool_result': continue - stub = pending.pop(b.get('tool_use_id'), None) - if not stub: continue - cc = b.get('content') - txt = cc if isinstance(cc, str) else '\n'.join(x.get('text', '') for x in cc if isinstance(x, dict) and x.get('type') == 'text') if isinstance(cc, list) else '' - err = bool(b.get('is_error')) - mcode = re.match(r'Exit code (\d+)', txt or '') - code = int(mcode.group(1)) if mcode else (0 if not err else None) - tur = o.get('toolUseResult') or {} - interrupted = bool(tur.get('interrupted')) if isinstance(tur, dict) else False - stub.update({'ok': (not err) and not interrupted, 'exit': code, 'out': head(txt), 'is_error': err, 'interrupted': interrupted, - 'out_len': len(txt or '')}) - recs.append(stub) - for tid, stub in pending.items(): - stub.update({'ok': None, 'exit': None, 'out': '', 'unmatched': True}); recs.append(stub) - -# ---------- OPENCODE ---------- -def oc_tool_record(part, model, sid, cwd, source_tag): - name = part.get('tool') or part.get('name') - st = part.get('state') or {} - toolcounts[('opencode', model, name)] += 1 - inp = st.get('input') or {} - status = st.get('status') - tm = st.get('time') or part.get('time') or {} - start = tm.get('start') or tm.get('created'); end = tm.get('end') or tm.get('completed') - ts = start - err = st.get('error') - if isinstance(err, dict): err = err.get('message') - content = st.get('content') - txt = '' - if isinstance(content, list): txt = '\n'.join(x.get('text', '') for x in content if isinstance(x, dict)) - elif isinstance(st.get('output'), str): txt = st.get('output') - meta = st.get('metadata') or {} - base = {'source': 'opencode', 'model': model, 'session': sid, 'ts': ts, 'cwd': inp.get('workdir') or cwd, 'callid': part.get('callID') or part.get('id'), 'store': source_tag, - 'dur_ms': (end - start) if (isinstance(start, (int, float)) and isinstance(end, (int, float))) else None} - if name in ('bash', 'shell', 'oc_bash'): - code = meta.get('exit') - if code is None: - mm = re.search(r'\[Exit code: (-?\d+)\]', txt or '') or re.search(r'Exit code: (-?\d+)', txt or '') - if mm: code = int(mm.group(1)) - ok = (status == 'completed') and (code in (None, 0)) - recs.append(dict(base, tool=name, via_js_cell=False, cmd=inp.get('command'), desc=inp.get('description'), timeout=inp.get('timeout'), - ok=ok, exit=code, status=status, err=head(err), out=head(txt), interrupted=bool(meta.get('interrupted')) or (err == 'Tool execution aborted'), - out_len=len(txt or ''))) - elif name == 'execute': - js = inp.get('code') or '' - calls, shell = cell_records(js, {'source': 'opencode', 'model': model}) - failed = status == 'error' - codes = exit_codes_from_output(txt or '') - if not shell: - recs.append(dict(base, tool='execute(js-only)', via_js_cell=True, js_tools=[c[0] for c in calls], cmd=None, js=head(js, 1200), ok=(not failed), exit=None, status=status, err=head(err), out=head(txt))) - for i, s in enumerate(shell): - code = codes[i] if len(codes) == len(shell) else None - recs.append(dict(base, tool='execute>' + 'shell', via_js_cell=True, js_tools=[c[0] for c in calls], cmd=s['cmd'], js=head(js, 1200), dynamic=s['dynamic'], - ok=(not failed) and code in (None, 0), exit=code, exit_visible=code is not None, status=status, err=head(err), out=head(txt), n_in_cell=len(shell), idx_in_cell=i)) - -def do_opencode(): - db = HOME + '/.local/share/opencode/opencode.db' - if not os.path.exists(db): print('opencode db not found'); return - con = sqlite3.connect('file:' + db.replace('\\', '/') + '?mode=ro', uri=True) - cur = con.cursor() - sess_dir = {} - for t in ('session', 'session_v2'): - for (i, d) in cur.execute(f'select id, directory from {t}'): sess_dir[i] = d - seen = set() - # v2 - msg_model = {} - for (mid, sid, d) in cur.execute("select id, session_id, data from session_message where type='assistant'"): - try: o = json.loads(d) - except Exception: continue - mm = o.get('model') or {} - model = (mm.get('providerID') or '') + '/' + (mm.get('id') or mm.get('modelID') or '') - msg_model[mid] = model - for part in o.get('content') or []: - if not isinstance(part, dict) or part.get('type') != 'tool': continue - key = part.get('id') or part.get('callID') - if key in seen: continue - seen.add(key) - oc_tool_record(part, model, sid, sess_dir.get(sid), 'v2') - # v1 - for (mid, sid, d) in cur.execute("select id, session_id, data from message"): - try: o = json.loads(d) - except Exception: continue - if o.get('role') == 'assistant': - msg_model[mid] = (o.get('providerID') or '') + '/' + (o.get('modelID') or '') - for (pid, mid, sid, d) in cur.execute("select id, message_id, session_id, data from part"): - try: part = json.loads(d) - except Exception: continue - if part.get('type') != 'tool': continue - key = part.get('callID') or pid - if key in seen: continue - seen.add(key) - oc_tool_record(part, msg_model.get(mid, 'unknown'), sid, sess_dir.get(sid), 'v1') - -# ---------- CURSOR ---------- -def do_cursor(): - for f in glob.glob(HOME + '/.cursor/chats/*/*/store.db'): - sid = f.split(os.sep)[-2] - try: meta = json.load(open(os.path.join(os.path.dirname(f), 'meta.json'))) - except Exception: meta = {} - con = sqlite3.connect('file:' + f + '?mode=ro', uri=True) - model = None; calls = {}; results = {} - rows = list(con.execute('select id, data from blobs')) - for (i, d) in rows: - try: o = json.loads(d) - except Exception: continue - if not isinstance(o, dict): continue - r = o.get('role'); cont = o.get('content') - if r == 'system' and isinstance(cont, str): - m = re.search(r'powered by (.+?)\. ', cont); model = m.group(1) if m else model - if r == 'assistant' and isinstance(cont, list): - for p in cont: - if p.get('type') == 'tool-call': calls[p.get('toolCallId')] = p - if r == 'tool' and isinstance(cont, list): - for p in cont: - if p.get('type') == 'tool-result': results[p.get('toolCallId')] = p - for cid, p in calls.items(): - name = p.get('toolName') - toolcounts[('cursor', model, name)] += 1 - if name in ('Shell', 'AwaitShell'): - res = results.get(cid, {}); rtxt = res.get('result') - if not isinstance(rtxt, str): rtxt = json.dumps(rtxt) if rtxt is not None else '' - mcode = re.search(r'Exit code: (-?\d+)', rtxt) - code = int(mcode.group(1)) if mcode else None - args = p.get('args') or {} - recs.append({'source': 'cursor', 'model': model, 'session': sid, 'ts': meta.get('createdAtMs'), 'cwd': meta.get('cwd'), 'tool': name, 'via_js_cell': False, - 'cmd': args.get('command') if name == 'Shell' else None, 'ok': (code == 0) if code is not None else None, 'exit': code, 'out': head(rtxt), 'out_len': len(rtxt)}) - -for fn in (do_codex, do_claude, do_opencode, do_cursor): - n0 = len(recs) - try: fn() - except Exception as e: print(fn.__name__, 'skipped:', type(e).__name__, str(e)[:200]) - print(fn.__name__, len(recs) - n0, 'records') -if not recs: - print('No transcripts found. Looked in ~/.codex/sessions, ~/.claude/projects, ~/.local/share/opencode/opencode.db, ~/.cursor/chats'); sys.exit(1) - -with open(os.path.join(OUT, 'records.jsonl'), 'w', encoding='utf-8') as w: - for r in recs: w.write(json.dumps(r, ensure_ascii=False) + '\n') -with open(os.path.join(OUT, 'toolcounts.json'), 'w', encoding='utf-8') as w: - json.dump([{'source': k[0], 'model': k[1], 'tool': k[2], 'n': v} for k, v in toolcounts.items()], w, ensure_ascii=False, indent=0) -print('total', len(recs)) +if __name__ == "__main__": + runpy.run_path(str(Path(__file__).resolve().parents[1] / "skills/shell-gatherer/scripts/gather.py"), run_name="__main__") diff --git a/scripts/integrate_agents.py b/scripts/integrate_agents.py index c8b8ae7..5ec1291 100644 --- a/scripts/integrate_agents.py +++ b/scripts/integrate_agents.py @@ -8,7 +8,7 @@ import re import subprocess -REPO = "Noisemaker111/shell-forensics" +REPO = "Noisemaker111/command-model" MAINTAINER_PERMISSIONS = {"write", "maintain", "admin"} @@ -53,7 +53,7 @@ def main(): require_maintainer(pr["author"]["login"]) require_maintainer(gh("api", "user")["login"]) for check in checks: - match = re.fullmatch(r"https://github\.com/Noisemaker111/shell-forensics/actions/runs/(\d+)/job/\d+", check.get("detailsUrl", "")) + match = re.fullmatch(r"https://github\.com/Noisemaker111/command-model/actions/runs/(\d+)/job/\d+", check.get("detailsUrl", "")) if not match: raise ValueError("Required check is not the repository Actions job") run = gh("api", f"repos/{REPO}/actions/runs/{match[1]}") diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index 70a7616..7f3124e 100644 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -5,7 +5,7 @@ import re import subprocess -REPO = "Noisemaker111/shell-forensics" +REPO = "Noisemaker111/command-model" def command(*args): diff --git a/scripts/template.html b/scripts/template.html index 06ae10a..caca842 100644 --- a/scripts/template.html +++ b/scripts/template.html @@ -1,4 +1,4 @@ -Shell Forensics +Shell Analysis