diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..239589a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,81 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: pyproject.toml + + - run: pip install -e ".[dev]" + + # Neither of these touches the network or spends a token: the synthesis loop is + # driven by a scripted client, and the corpus cases carry their implementations. + - name: Unit tests + run: pytest -q + + # The corpus is the one that would notice a gate going quiet: each case carries a + # deliberately planted bug and names which gate has to catch it. A gate that stops + # firing, or starts firing on the correct implementation, fails here. + - name: Corpus selftest + run: jitagent selftest + + # audit_traps reads the committed audit_result.json, so it needs no model. It is + # the check that the recorded audit still says what the README claims it says. + - name: Audit traps (against the committed audit results) + run: python tools/audit_traps.py + + package: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - run: pip install build twine + + - name: Build sdist and wheel + run: python -m build + + # Catches the metadata problems that only surface at upload time — a malformed + # README, a missing license, a bad classifier — rather than at `twine upload`. + - name: Check the built metadata + run: twine check --strict dist/* + + # A wheel that imports but whose console script is broken is a bad release, and + # nothing above would notice: the tests all run against the source tree. + - name: Install the wheel and run it + run: | + python -m venv /tmp/wheeltest + /tmp/wheeltest/bin/pip install --quiet dist/*.whl + /tmp/wheeltest/bin/jitagent --help + /tmp/wheeltest/bin/jitagent selftest + + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..38badec --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,91 @@ +name: Release + +# Publishing is deliberate: it runs when a GitHub Release is *published*, not on every +# tag push. A tag is cheap to create by accident; a release is not, and a version on +# PyPI can never be reused even after it is deleted. +on: + release: + types: [published] + workflow_dispatch: + inputs: + repository: + description: Where to publish + type: choice + default: testpypi + options: [testpypi, pypi] + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - run: pip install -e ".[dev]" + + # Never publish something that has not passed its own checks. This duplicates CI + # on purpose: CI green on a branch says nothing about the commit being released. + - run: pytest -q + - run: jitagent selftest + + # The tag and the version must agree. Getting this wrong means shipping 0.0.1 as + # v0.0.2, which cannot be undone — PyPI does not allow a version to be re-uploaded. + - name: Check the tag matches the version + if: github.event_name == 'release' + run: | + version=$(python -c "import tomllib,pathlib;print(tomllib.loads(pathlib.Path('pyproject.toml').read_text())['project']['version'])") + tag="${GITHUB_REF_NAME#v}" + if [ "$version" != "$tag" ]; then + echo "::error::tag $GITHUB_REF_NAME does not match pyproject version $version" + exit 1 + fi + echo "tag $GITHUB_REF_NAME matches version $version" + + - run: pip install build twine + - run: python -m build + - run: twine check --strict dist/* + + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + publish-testpypi: + needs: build + if: github.event_name == 'workflow_dispatch' && inputs.repository == 'testpypi' + runs-on: ubuntu-latest + environment: + name: testpypi + url: https://test.pypi.org/p/jitagent + permissions: + id-token: write # Trusted Publishing: OIDC, no stored secret + steps: + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + + publish-pypi: + needs: build + if: github.event_name == 'release' || inputs.repository == 'pypi' + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/jitagent + permissions: + id-token: write # Trusted Publishing: OIDC, no stored secret + steps: + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.gitignore b/.gitignore index ee8f745..708dac3 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,9 @@ build/ # the raw capture from tools/capture_demo.sh; docs/demo.txt is the curated one docs/demo.raw.txt + +# build output +dist/ + +# build metadata +src/*.egg-info/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b26d28b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,51 @@ +# Changelog + +This project follows [Semantic Versioning](https://semver.org/). While the major version +is 0, the API may change in a minor release. + +## [0.0.1] — 2026-09-16 + +First release. The loop round-trips: **a sentence goes in, code comes out, and you fetch +it back by name.** + +### The surface + +- `compile_function` / `get_code` / `call_function` / `search_functions` / + `inspect_function`, and an `jitagent` CLI over the same five operations. +- Three synthesis backends behind one method (`complete(system=, user=)`): the local + `claude` CLI, the Anthropic API, and a scripted client that makes the whole loop + testable with no network and no tokens. +- A registry organised around **the test set**, not the code: one requirement can hold + several versions, and the cases belong to the requirement so a regeneration accepts + against everything learned so far. +- Three-level lookup — exact hash, then candidate retrieval with re-verification against + *this run's* examples, then a miss. Retrieval affects the hit rate; re-verification + decides correctness. + +### Correctness + +- `jitagent` writes the test cases out in full **before** the code exists, so the code + cannot shape the cases. Your seed examples anchor them, and a generated case that + contradicts one is dropped. +- A failure that occurs *only* on generated cases is handed back for adjudication rather + than reported as a bug. +- A decision the requirement never made must be declared in an `assumes` field. +- Three gates can reject an implementation: the static check, your cases, and the return + schema inferred from your examples. + +### Verified against an independent oracle + +Six reference implementations, transcribed literally from the requirement text and +written before any generated case or generated code existed: + +- 48/48 generated expectations agreed with them, with 18/18 planted traps exercised +- 24 compiles across three runs, every one on the first attempt +- 4800 random inputs, no disagreement + +### Known limitations + +One model (Claude Haiku 4.5); the requirements and the audit oracle come from the same +person; the sandbox is a **correctness** sandbox, not a security one; +`datetime.strptime` does not work inside it. See the README. + +[0.0.1]: https://github.com/Birfy/jitagent/releases/tag/v0.0.1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..b72e4ce --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,95 @@ +# Contributing + +```bash +pip install -e ".[dev]" +pytest # 102 unit tests — no network, no tokens +jitagent selftest # 6 corpus cases, each with a deliberately planted bug +``` + +Both must pass before anything else is worth discussing. Neither touches the network: the +synthesis loop is driven by `ScriptedClient`, and every corpus case carries its own +implementation. + +## The one rule that matters + +**A gate that fires on correct code costs more than a gate that misses a bug.** Nothing in +a false positive points at the real cause — it took reading two nearly identical +implementations side by side to find that `datetime.strptime` cannot work inside the +sandbox, and that cost a wasted round of synthesis every time it happened. + +So the corpus checks both directions. `01_correct` exists to make sure a correct +implementation sails through; a change that makes it red has failed, however many bugs it +catches elsewhere. + +## Adding a verification gate + +Seven gates were built here and five were removed, because **not one of them ever caught a +mistake a real model made** — everything they caught was a bug planted by hand. They had +been built by reasoning from a document rather than forced into existence by a real +failure. The reasoning is preserved in [docs/correctness.md](docs/correctness.md); it is +what a new gate has to beat. + +A new gate needs: + +1. **A real failure it would have caught** — a model output, not a hypothetical. +2. **A corpus case** in `src/jitagent/corpus/`, whose `case.json` names the gate that must + reject it and why. +3. **Evidence it does not fire on correct code.** `jitagent selftest` covers the corpus; + for anything subtle, `tools/audit_e2e.py` runs real compiles against independent + reference implementations. + +"Is your test set strong enough?" is **advice to the caller, not a verdict.** A gate that +answers that question belongs in a report, not in the pass/fail path. + +## Changing a prompt + +`prompts.py` has one rule at the top: **only promise what is actually checked.** The prompt +once threatened the model with a branch-coverage threshold that had already been deleted. +If you change what can reject an implementation, change the prompt in the same commit. + +Prompt changes are not covered by the unit tests — they are measured. `tools/audit_e2e.py` +is the measurement, and it costs real tokens. + +## The audits + +```bash +python tools/audit_tests.py # are the generated expectations right? (spends tokens) +python tools/audit_traps.py # did the generated cases go after the hard parts? (free) +python tools/audit_e2e.py # is the compiled function right? (spends tokens) +``` + +The protocol in `tools/audit_tests.py` matters more than the numbers: **the reference +implementation has to be written before any generated case is seen.** Write it afterwards +and you start finding reasons why the model's answer was fine, and the audit is worthless. + +`audit_traps.py` is why the agreement rate means anything. A model that only ever writes +`f([]) == []` scores 100% and has tested nothing, so each trap is checked as a predicate +over the generated case's *input* — a note claiming to test a tie cannot pass without a tie +in the data. + +## The demo in the README + +Three steps, and the middle one is yours: + +```bash +tools/capture_demo.sh # runs the CLI for real, writes docs/demo.raw.txt +$EDITOR docs/demo.txt # pick the lines, trim to <= 92 columns +python tools/make_demo_svg.py # docs/demo.txt -> docs/demo.svg +``` + +Everything the animation shows is real captured output. A hand-written demo shows what +someone hoped the tool would do. + +## Style + +Match the file you are editing. Comments here carry the *reasoning* — why a thing is the +way it is, what was tried and rejected, what would make it wrong — because the mechanisms +are easy to read and the trade-offs behind them are not. A comment that restates the code +is noise; one that records a measurement or a discarded alternative is the point. + +## Releasing + +Maintainers only. Bump `version` in `pyproject.toml`, add a `CHANGELOG.md` entry, then +publish a GitHub Release tagged `vX.Y.Z`. The release workflow checks the tag against the +version, re-runs the tests, builds, and publishes to PyPI over Trusted Publishing — there +is no API token stored anywhere. `workflow_dispatch` publishes to TestPyPI for a rehearsal. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..ad62975 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Birfy + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/NEXT.md b/NEXT.md index 3a7da30..17fe1e7 100644 --- a/NEXT.md +++ b/NEXT.md @@ -90,13 +90,13 @@ Not many left; after the cull there are four: ## Read before starting -1. The header of `src/agentjit/propose.py` — one model writing both the code and the tests +1. The header of `src/jitagent/propose.py` — one model writing both the code and the tests that judge it is circular; which part of that each of the three mitigations solves, and which part cannot be solved -2. The header of `src/agentjit/verify.py` — which five gates were cut, why, and which one is +2. The header of `src/jitagent/verify.py` — which five gates were cut, why, and which one is most likely to come back 3. The header of `tools/audit_tests.py` — the audit protocol. The reference implementation has to be written **before** any generated case is seen, or it starts finding reasons why the model's answer was fine -4. The header of `src/agentjit/lookup.py` — why retrieval is allowed to be crude, and where +4. The header of `src/jitagent/lookup.py` — why retrieval is allowed to be crude, and where being crude does not matter diff --git a/README.md b/README.md index 788898a..d3babcc 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,17 @@
-# agentjit +# jitagent **Stop your agent re-deriving the same function 200 times. Compile it once.** +[![CI](https://github.com/Birfy/jitagent/actions/workflows/ci.yml/badge.svg)](https://github.com/Birfy/jitagent/actions/workflows/ci.yml) +[![PyPI](https://img.shields.io/pypi/v/jitagent?color=3775a9&logo=pypi&logoColor=white)](https://pypi.org/project/jitagent/) [![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-3776ab?logo=python&logoColor=white)](https://www.python.org/downloads/) -[![tests](https://img.shields.io/badge/tests-102%20passing-5ac489)](tests/) -[![corpus](https://img.shields.io/badge/corpus-6%2F6-5ac489)](tests/corpus/) +[![corpus](https://img.shields.io/badge/corpus-6%2F6-5ac489)](src/jitagent/corpus/) +[![License: MIT](https://img.shields.io/badge/license-MIT-blue)](LICENSE) [![status](https://img.shields.io/badge/status-working%20prototype-e0af68)](#known-gaps) -agentjit: compile a requirement, call it, list the registry +jitagent: compile a requirement, call it, list the registry
@@ -17,16 +19,16 @@ The same session as text (it is a real transcript — tools/capture_demo.sh produced it) ```console -$ agentjit compile examples/rank.json --name rank +$ jitagent compile examples/rank.json --name rank requirement Rank {name, score} records from highest score to lowest. Records with the same… seeds 5 model claude-haiku-4-5 backend cli cache auto miss: nothing reusable found wrote 8 extra test cases - {"records": [{"name": "alice",… (Three-way tie causes rank to skip from 1 to 4) - {"records": [{"name": "alice",… (Negative scores ranked lower than zero) - ! assumes something the requirement did not say: Scores can be negative and follow st… - {"records": [{"name": "zoe", "… (Names sorted alphabetically within same rank, regard… - {"records": [{"name": "alice",… (Four-way tie causes rank to skip from 1 to 5) + {"records": [{"name": "bob", "… (Two separate tie groups; each sorted by name within… + {"records": [{"name": "z", "sc… (Negative scores rank below positive; highest to lowe… + ! assumes something the requirement did not say: Scores can be floating-point numbers… + {"records": [{"name": "a", "sc… (Zero score is a valid boundary; tied zeros rank belo… + {"records": [{"name": "a", "sc… (Decimal scores compared with exact equality; 85.5 ≠… attempt 1 passed PASS static passed @@ -34,12 +36,12 @@ miss: nothing reusable found PASS examples 13/13 passed PASS return_schema passed - level: VERIFIED took: 47ms + level: VERIFIED took: 65ms -result: ok tokens 3569in/2154out +result: ok tokens 3692in/1680out cache: miss no hit; synthesised a new one -name rank handle fn_69889ea0d4c5 v1 level VERIFIED -$ agentjit call rank '{"records":[{"name":"zoe","score":7},{"name":"amy","score":9}]}' +name rank handle fn_8617fc1e2b62 v1 level VERIFIED +$ jitagent call rank '{"records":[{"name":"zoe","score":7},{"name":"amy","score":9}]}' ok v1 [ { @@ -51,7 +53,7 @@ ok v1 "rank": 2 } ] -$ agentjit list +$ jitagent list NAME VER CASES REQUIREMENT rank v1 13 Rank {name, score} records from highest score to lowest. ``` @@ -70,11 +72,11 @@ group, sum, format — and a function is something you can read, test, version a model's judgement is worth paying for *once*, to write that function. It is not worth paying for two hundred times to re-derive it. -So hand `agentjit` a sentence and a couple of examples. It writes the test cases out in +So hand `jitagent` a sentence and a couple of examples. It writes the test cases out in full, writes the code, runs it against every case in a sandbox, and stores the lot under a name you choose. -| | in the agent | after `agentjit compile` | +| | in the agent | after `jitagent compile` | | --- | --- | --- | | per call | seconds, thousands of tokens | ~30ms, zero tokens | | same input twice | may differ | identical | @@ -92,15 +94,22 @@ regeneration of everything you have. ## Quickstart ```bash -git clone https://github.com/Birfy/agentjit && cd agentjit -pip install -e ".[dev]" +pip install jitagent # the import name and the CLI are `jitagent` +jitagent selftest # 6 corpus cases, each with a deliberately planted bug +``` + +`selftest` ships with the package and spends nothing. It is worth running on a new +machine: how the sandbox contains a memory bomb is platform-dependent, so "does this +behave here?" is a real question. To work on `jitagent` itself: -pytest # 102 unit tests — no network, no tokens -agentjit selftest # 6 corpus cases, each with a deliberately planted bug +```bash +git clone https://github.com/Birfy/jitagent && cd jitagent +pip install -e ".[dev]" +pytest # 102 unit tests — no network, no tokens ``` Synthesis needs a model. If you have [Claude Code](https://claude.ai/code) installed, -`agentjit` shells out to it and uses its authorisation; otherwise set `ANTHROPIC_API_KEY` +`jitagent` shells out to it and uses its authorisation; otherwise set `ANTHROPIC_API_KEY` and pass `--via api`. A requirement is a JSON file: a sentence, plus a few examples that pin down what you mean. @@ -122,24 +131,25 @@ The examples are the spec, not decoration — nothing without them reaches the c } ``` -That is [`examples/rank.json`](examples/rank.json), so this runs as it stands: +That is [`examples/rank.json`](examples/rank.json) — from a checkout, this runs as it +stands: ```bash -agentjit compile examples/rank.json --name rank # write the cases, then the code -agentjit get rank # print the source -agentjit call rank '{"records": [...]}' # run it in the sandbox -agentjit list # what is in the registry -agentjit search "rank records by score" # is there one already? -agentjit inspect rank # cases, versions, verification report +jitagent compile examples/rank.json --name rank # write the cases, then the code +jitagent get rank # print the source +jitagent call rank '{"records": [...]}' # run it in the sandbox +jitagent list # what is in the registry +jitagent search "rank records by score" # is there one already? +jitagent inspect rank # cases, versions, verification report ``` -The registry lives in `~/.agentjit/registry`; `AGENTJIT_HOME` or `--home` moves it. +The registry lives in `~/.jitagent/registry`; `JITAGENT_HOME` or `--home` moves it. ### From Python ```python -from agentjit import compile_function, get_code, call_function, Example -from agentjit.llm import ClaudeCliClient +from jitagent import compile_function, get_code, call_function, Example +from jitagent.llm import ClaudeCliClient compile_function( "Rank {name, score} records from highest score to lowest...", @@ -283,11 +293,11 @@ precise. ## Reuse, and why the lookup is allowed to be crude -Compiling once saves nothing; the saving is in not compiling the second time. `agentjit +Compiling once saves nothing; the saving is in not compiling the second time. `jitagent inspect` shows what is actually stored: ```console -$ agentjit inspect rank +$ jitagent inspect rank -- test cases (13) -- [caller] {"records": [{"name": "alice", "score": 90}, … -> [{"name": "alice", "rank": 1}, … [caller] {"records": []} -> [] @@ -369,7 +379,7 @@ cases it can see; `verify.py` records what would trigger that. ## The code ``` -src/agentjit/ +src/jitagent/ jit.py the product surface: compile / get_code / call / search / inspect propose.py get the model to write the cases out — before the code, separate call prompts.py the two prompts; the requirement sits in an untrusted data region @@ -383,8 +393,9 @@ src/agentjit/ lookup.py three-level lookup — retrieval narrows, re-verification decides runtime.py calling: input guard → sandbox → return guard. A pure read llm.py backends: the API, the local claude CLI, a scripted replay + corpus/ 6 corpus cases, one per gate — shipped, so selftest runs anywhere tools/ the three audits, plus the demo capture and the SVG builder -tests/corpus/ 6 corpus cases, one per gate +tests/ unit tests; every one runs with no network and no tokens ``` | Document | Contents | @@ -393,6 +404,7 @@ tests/corpus/ 6 corpus cases, one per gate | [docs/correctness.md](docs/correctness.md) | correctness and testing — **what decides whether this stands up** | | [docs/tracing-frontend.md](docs/tracing-frontend.md) | the front end that spots repetition and triggers a compile (not started) | | [NEXT.md](NEXT.md) | what to do next, and what is deliberately not being done | +| [CONTRIBUTING.md](CONTRIBUTING.md) | how to run things, and what a new verification gate has to prove | **Where to start reading:** the headers of `propose.py` (why one model judging itself is circular, and which part of that cannot be fixed), `verify.py` (which five gates were cut diff --git a/docs/correctness.md b/docs/correctness.md index f94874f..70e97d2 100644 --- a/docs/correctness.md +++ b/docs/correctness.md @@ -12,7 +12,7 @@ > coverage, fuzzing and mutation testing were all built and then removed, because across > the whole corpus not one of them ever caught a mistake a real model made. See the > "What was cut, and why" section of the [README](../README.md) and the header of -> `src/agentjit/verify.py`. The reasoning here is kept because it is what any future +> `src/jitagent/verify.py`. The reasoning here is kept because it is what any future > addition has to argue against. --- diff --git a/docs/demo.svg b/docs/demo.svg index b238ef2..ad4157f 100644 --- a/docs/demo.svg +++ b/docs/demo.svg @@ -1,22 +1,22 @@ - -agentjit — text in, code out, fetch it by name + +jitagent — text in, code out, fetch it by name -agentjit — text in, code out, fetch it by name +jitagent — text in, code out, fetch it by name $ -agentjit compile examples/rank.json --name rank +jitagent compile examples/rank.json --name rank requirement Rank {name, score} records from highest score to lowest. Records with the same… seeds 5 model claude-haiku-4-5 backend cli cache auto miss: nothing reusable found wrote 8 extra test cases - {"records": [{"name": "alice",… (Three-way tie causes rank to skip from 1 to 4) - {"records": [{"name": "alice",… (Negative scores ranked lower than zero) - ! assumes something the requirement did not say: Scores can be negative and follow st… - {"records": [{"name": "zoe", "… (Names sorted alphabetically within same rank, regard… - {"records": [{"name": "alice",… (Four-way tie causes rank to skip from 1 to 5) + {"records": [{"name": "bob", "… (Two separate tie groups; each sorted by name within… + {"records": [{"name": "z", "sc… (Negative scores rank below positive; highest to lowe… + ! assumes something the requirement did not say: Scores can be floating-point numbers… + {"records": [{"name": "a", "sc… (Zero score is a valid boundary; tied zeros rank belo… + {"records": [{"name": "a", "sc… (Decimal scores compared with exact equality; 85.5 ≠… attempt 1 passed PASS static passed @@ -24,13 +24,13 @@ PASS examples 13/13 passed PASS return_schema passed - level: VERIFIED took: 47ms + level: VERIFIED took: 65ms -result: ok tokens 3569in/2154out +result: ok tokens 3692in/1680out cache: miss no hit; synthesised a new one -name rank handle fn_69889ea0d4c5 v1 level VERIFIED +name rank handle fn_8617fc1e2b62 v1 level VERIFIED $ -agentjit call rank '{"records":[{"name":"zoe","score":7},{"name":"amy","score":9}]}' +jitagent call rank '{"records":[{"name":"zoe","score":7},{"name":"amy","score":9}]}' ok v1 [ @@ -44,7 +44,7 @@ } ] $ -agentjit list +jitagent list NAME VER CASES REQUIREMENT rank v1 13 Rank {name, score} records from highest score to lowest. diff --git a/docs/demo.txt b/docs/demo.txt index 52ddfde..3083f3b 100644 --- a/docs/demo.txt +++ b/docs/demo.txt @@ -1,14 +1,14 @@ -$ agentjit compile examples/rank.json --name rank +$ jitagent compile examples/rank.json --name rank requirement Rank {name, score} records from highest score to lowest. Records with the same… seeds 5 model claude-haiku-4-5 backend cli cache auto ~~~ miss: nothing reusable found wrote 8 extra test cases - {"records": [{"name": "alice",… (Three-way tie causes rank to skip from 1 to 4) - {"records": [{"name": "alice",… (Negative scores ranked lower than zero) - ! assumes something the requirement did not say: Scores can be negative and follow st… - {"records": [{"name": "zoe", "… (Names sorted alphabetically within same rank, regard… - {"records": [{"name": "alice",… (Four-way tie causes rank to skip from 1 to 5) + {"records": [{"name": "bob", "… (Two separate tie groups; each sorted by name within… + {"records": [{"name": "z", "sc… (Negative scores rank below positive; highest to lowe… + ! assumes something the requirement did not say: Scores can be floating-point numbers… + {"records": [{"name": "a", "sc… (Zero score is a valid boundary; tied zeros rank belo… + {"records": [{"name": "a", "sc… (Decimal scores compared with exact equality; 85.5 ≠… attempt 1 passed PASS static passed @@ -16,13 +16,13 @@ miss: nothing reusable found PASS examples 13/13 passed PASS return_schema passed - level: VERIFIED took: 47ms + level: VERIFIED took: 65ms -result: ok tokens 3569in/2154out +result: ok tokens 3692in/1680out cache: miss no hit; synthesised a new one -name rank handle fn_69889ea0d4c5 v1 level VERIFIED +name rank handle fn_8617fc1e2b62 v1 level VERIFIED ~~~ -$ agentjit call rank '{"records":[{"name":"zoe","score":7},{"name":"amy","score":9}]}' +$ jitagent call rank '{"records":[{"name":"zoe","score":7},{"name":"amy","score":9}]}' ok v1 [ { @@ -35,6 +35,6 @@ ok v1 } ] ~~~ -$ agentjit list +$ jitagent list NAME VER CASES REQUIREMENT rank v1 13 Rank {name, score} records from highest score to lowest. diff --git a/pyproject.toml b/pyproject.toml index 8fece87..42d486f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,19 +1,54 @@ [project] -name = "agentjit" -version = "0.1.0" +# `agentjit` is taken on PyPI by a different, actively published project whose wheel +# also installs a top-level `agentjit` package — so sharing the name would not just be +# confusing, the two would silently overwrite each other in site-packages. Everything +# here is `jitagent`: the distribution, the import name, the CLI and the repository. +name = "jitagent" +version = "0.0.1" description = "Compile a plain-text requirement into a reusable, verified, sandboxed function" +readme = "README.md" requires-python = ">=3.11" -dependencies = ["coverage>=7.4", "jsonschema>=4.21"] +license = "MIT" +license-files = ["LICENSE"] +authors = [{ name = "Birfy", email = "birfychen@gmail.com" }] +keywords = ["llm", "agent", "code-generation", "sandbox", "jit", "verification"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Software Development :: Code Generators", + "Topic :: Software Development :: Testing", +] +# jsonschema is the only runtime dependency: it is the yardstick for both the input +# guard and the return-schema gate. `coverage` used to be here for the branch-coverage +# gate; that gate is gone, so the dependency went with it. +dependencies = ["jsonschema>=4.21"] [project.optional-dependencies] -dev = ["pytest>=8"] +dev = ["pytest>=8", "build>=1.2", "twine>=5.1"] + +[project.urls] +Homepage = "https://github.com/Birfy/jitagent" +Repository = "https://github.com/Birfy/jitagent" +Issues = "https://github.com/Birfy/jitagent/issues" +Changelog = "https://github.com/Birfy/jitagent/blob/main/CHANGELOG.md" [project.scripts] -agentjit = "agentjit.cli:main" +jitagent = "jitagent.cli:main" [build-system] -requires = ["setuptools>=68"] +requires = ["setuptools>=77"] build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] where = ["src"] + +# The corpus ships with the package so `jitagent selftest` works from a pip install, +# not only from a git checkout. +[tool.setuptools.package-data] +jitagent = ["corpus/*/*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/src/agentjit/__init__.py b/src/jitagent/__init__.py similarity index 95% rename from src/agentjit/__init__.py rename to src/jitagent/__init__.py index c84c3b2..49827b6 100644 --- a/src/agentjit/__init__.py +++ b/src/jitagent/__init__.py @@ -1,4 +1,4 @@ -"""agent-jit — text in, code out, fetch it back by name. +"""jitagent — text in, code out, fetch it back by name. compile_function(requirement, examples, name="rank") -> synthesise, verify, store get_code("rank") -> source diff --git a/src/agentjit/_child.py b/src/jitagent/_child.py similarity index 82% rename from src/agentjit/_child.py rename to src/jitagent/_child.py index 7ddcf58..1069a10 100644 --- a/src/agentjit/_child.py +++ b/src/jitagent/_child.py @@ -114,13 +114,7 @@ def main(): _apply_limits(job.get("mem_mb", 512), job.get("timeout_ms", 5000) // 1000 + 6) code_path = job["code_path"] - cov = None - if job.get("coverage"): - import coverage - cov = coverage.Coverage(data_file=None, branch=True, include=[code_path]) - cov.start() - - payload = {"ok": True, "results": [], "coverage": None, "load_error": None} + payload = {"ok": True, "results": [], "load_error": None} try: with open(code_path) as fh: source = fh.read() @@ -128,8 +122,6 @@ def main(): exec(compile(source, code_path, "exec"), g) fn = g[job["entry"]] except BaseException: - if cov: - cov.stop() payload["ok"] = False payload["load_error"] = traceback.format_exc(limit=6) _emit(out_fd, payload) @@ -145,26 +137,6 @@ def main(): "tb": traceback.format_exc(limit=6), }) - if cov: - cov.stop() - try: - # json_report wants a path, not a file object. Write it next to the code so - # it is destroyed with the temp directory. - report_path = code_path + ".cov.json" - cov.json_report(outfile=report_path) - with open(report_path) as fh: - report = json.load(fh) - for path, data in report.get("files", {}).items(): - if os.path.samefile(path, code_path): - payload["coverage"] = { - "summary": data["summary"], - "missing_lines": data.get("missing_lines", []), - "missing_branches": data.get("missing_branches", []), - } - break - except Exception as e: - payload["coverage"] = {"error": f"{type(e).__name__}: {e}"} - _emit(out_fd, payload) diff --git a/src/agentjit/cases.py b/src/jitagent/cases.py similarity index 100% rename from src/agentjit/cases.py rename to src/jitagent/cases.py diff --git a/src/agentjit/cli.py b/src/jitagent/cli.py similarity index 92% rename from src/agentjit/cli.py rename to src/jitagent/cli.py index 1788fe9..1cec36b 100644 --- a/src/agentjit/cli.py +++ b/src/jitagent/cli.py @@ -11,7 +11,12 @@ from .types import Example from .verify import Thresholds, verify -CORPUS = Path(__file__).resolve().parents[2] / "tests" / "corpus" +# The corpus ships inside the package. It used to be resolved relative to the source +# tree, which meant `jitagent selftest` — the command the README tells you to run +# straight after installing — only worked from a git checkout. It is also the one +# check worth running on a new machine: how the sandbox contains a memory bomb is +# platform-dependent, so "does it behave here" is a real question. +CORPUS = Path(__file__).with_name("corpus") def _print_details(report, verbose: bool) -> None: @@ -115,7 +120,7 @@ def _generator(via: str, model: str): class _LazyClient: """Build the API client only when synthesis actually needs it. - That way `agentjit compile` still searches the cache on a machine with no + That way `jitagent compile` still searches the cache on a machine with no credentials — a cache hit needs none. Constructing `AnthropicClient` imports the SDK and reads credentials; doing that up front would make "is there one already?" depend on the network too. @@ -137,8 +142,8 @@ def complete(self, **kw): f"Cache miss, and the Anthropic client could not start: {e}\n" " pip install anthropic\n" " # or use the local Claude Code CLI instead:\n" - " agentjit compile --via cli\n" - "To only use functions that already exist: agentjit list / search." + " jitagent compile --via cli\n" + "To only use functions that already exist: jitagent list / search." ) from e return self._real.complete(**kw) @@ -227,7 +232,7 @@ def cmd_get(args) -> int: code = get_code(args.name, registry=Registry(args.home)) if not code: - print(f"no function named {args.name!r} (try: agentjit list)", file=sys.stderr) + print(f"no function named {args.name!r} (try: jitagent list)", file=sys.stderr) return 1 print(code, end="" if code.endswith("\n") else "\n") return 0 @@ -252,17 +257,17 @@ def cmd_call(args) -> int: def main(argv=None) -> int: p = argparse.ArgumentParser( - prog="agentjit", + prog="jitagent", description="Compile a requirement into a verified, reusable sandboxed function.") p.add_argument("-v", "--verbose", action="store_true") p.add_argument("--home", default=None, - help="registry directory (default: $AGENTJIT_HOME/registry " - "or ~/.agentjit/registry)") + help="registry directory (default: $JITAGENT_HOME/registry " + "or ~/.jitagent/registry)") sub = p.add_subparsers(dest="cmd", required=True) c = sub.add_parser("compile", help="look in the cache, synthesise on a miss") c.add_argument("requirement", help="JSON file: {requirement, examples[]}") - c.add_argument("--name", default=None, help="name it, so `agentjit get ` works") + c.add_argument("--name", default=None, help="name it, so `jitagent get ` works") c.add_argument("--model", default=None) c.add_argument("--attempts", type=int, default=3) c.add_argument("-o", "--out", default=None, help="write the source here on success") diff --git a/tests/corpus/01_correct/case.json b/src/jitagent/corpus/01_correct/case.json similarity index 100% rename from tests/corpus/01_correct/case.json rename to src/jitagent/corpus/01_correct/case.json diff --git a/tests/corpus/01_correct/impl.py b/src/jitagent/corpus/01_correct/impl.py similarity index 100% rename from tests/corpus/01_correct/impl.py rename to src/jitagent/corpus/01_correct/impl.py diff --git a/tests/corpus/02_wrong_no_clean/case.json b/src/jitagent/corpus/02_wrong_no_clean/case.json similarity index 100% rename from tests/corpus/02_wrong_no_clean/case.json rename to src/jitagent/corpus/02_wrong_no_clean/case.json diff --git a/tests/corpus/02_wrong_no_clean/impl.py b/src/jitagent/corpus/02_wrong_no_clean/impl.py similarity index 100% rename from tests/corpus/02_wrong_no_clean/impl.py rename to src/jitagent/corpus/02_wrong_no_clean/impl.py diff --git a/tests/corpus/08_malicious/case.json b/src/jitagent/corpus/08_malicious/case.json similarity index 100% rename from tests/corpus/08_malicious/case.json rename to src/jitagent/corpus/08_malicious/case.json diff --git a/tests/corpus/08_malicious/impl.py b/src/jitagent/corpus/08_malicious/impl.py similarity index 100% rename from tests/corpus/08_malicious/impl.py rename to src/jitagent/corpus/08_malicious/impl.py diff --git a/tests/corpus/09_insufficient/case.json b/src/jitagent/corpus/09_insufficient/case.json similarity index 100% rename from tests/corpus/09_insufficient/case.json rename to src/jitagent/corpus/09_insufficient/case.json diff --git a/tests/corpus/09_insufficient/impl.py b/src/jitagent/corpus/09_insufficient/impl.py similarity index 100% rename from tests/corpus/09_insufficient/impl.py rename to src/jitagent/corpus/09_insufficient/impl.py diff --git a/tests/corpus/10_timeout/case.json b/src/jitagent/corpus/10_timeout/case.json similarity index 100% rename from tests/corpus/10_timeout/case.json rename to src/jitagent/corpus/10_timeout/case.json diff --git a/tests/corpus/10_timeout/impl.py b/src/jitagent/corpus/10_timeout/impl.py similarity index 100% rename from tests/corpus/10_timeout/impl.py rename to src/jitagent/corpus/10_timeout/impl.py diff --git a/tests/corpus/11_memory_bomb/case.json b/src/jitagent/corpus/11_memory_bomb/case.json similarity index 100% rename from tests/corpus/11_memory_bomb/case.json rename to src/jitagent/corpus/11_memory_bomb/case.json diff --git a/tests/corpus/11_memory_bomb/impl.py b/src/jitagent/corpus/11_memory_bomb/impl.py similarity index 100% rename from tests/corpus/11_memory_bomb/impl.py rename to src/jitagent/corpus/11_memory_bomb/impl.py diff --git a/src/agentjit/infer.py b/src/jitagent/infer.py similarity index 100% rename from src/agentjit/infer.py rename to src/jitagent/infer.py diff --git a/src/agentjit/jit.py b/src/jitagent/jit.py similarity index 100% rename from src/agentjit/jit.py rename to src/jitagent/jit.py diff --git a/src/agentjit/llm.py b/src/jitagent/llm.py similarity index 98% rename from src/agentjit/llm.py rename to src/jitagent/llm.py index 8595796..300acad 100644 --- a/src/agentjit/llm.py +++ b/src/jitagent/llm.py @@ -103,7 +103,7 @@ def complete(self, *, system: str, user: str, max_tokens: int = 16000) -> LLMRes # independent of the prompt content. Measured with an empty "repeat this sentence" task: # about 22.2k input tokens. Any token accounting based on the CLI has to subtract it, # or synthesis looks an order of magnitude more expensive than it is — that overhead -# belongs to Claude Code, not to agentjit. +# belongs to Claude Code, not to jitagent. CLI_OVERHEAD_TOKENS = 22_200 @@ -134,7 +134,7 @@ class ClaudeCliClient: def _argv(self, system: str) -> list[str]: return [ self.binary, "-p", "--model", self.model, - # Replace Claude Code's own system prompt with agentjit's. Without this the + # Replace Claude Code's own system prompt with jitagent's. Without this the # model receives two sets of instructions that contradict each other. "--system-prompt", system, "--exclude-dynamic-system-prompt-sections", diff --git a/src/agentjit/lookup.py b/src/jitagent/lookup.py similarity index 100% rename from src/agentjit/lookup.py rename to src/jitagent/lookup.py diff --git a/src/agentjit/prompts.py b/src/jitagent/prompts.py similarity index 100% rename from src/agentjit/prompts.py rename to src/jitagent/prompts.py diff --git a/src/agentjit/propose.py b/src/jitagent/propose.py similarity index 99% rename from src/agentjit/propose.py rename to src/jitagent/propose.py index 165d7ef..5276764 100644 --- a/src/agentjit/propose.py +++ b/src/jitagent/propose.py @@ -1,4 +1,4 @@ -"""Let agentjit write the test cases out in full. +"""Let jitagent write the test cases out in full. **This step violates the warning in [correctness.md §1](../../docs/correctness.md)**, so let's say it diff --git a/src/agentjit/registry.py b/src/jitagent/registry.py similarity index 99% rename from src/agentjit/registry.py rename to src/jitagent/registry.py index aebf715..7bd1678 100644 --- a/src/agentjit/registry.py +++ b/src/jitagent/registry.py @@ -2,7 +2,7 @@ Layout: - $AGENTJIT_HOME/registry// + $JITAGENT_HOME/registry// spec.json name, the requirement text, the inferred schemas, the entry point tests.json the test cases — correctness rests on these, so they are the main asset here @@ -43,7 +43,7 @@ def home() -> Path: - return Path(os.environ.get("AGENTJIT_HOME") or Path.home() / ".agentjit") + return Path(os.environ.get("JITAGENT_HOME") or Path.home() / ".jitagent") def _canon(value: Any) -> str: diff --git a/src/agentjit/runtime.py b/src/jitagent/runtime.py similarity index 100% rename from src/agentjit/runtime.py rename to src/jitagent/runtime.py diff --git a/src/agentjit/sandbox.py b/src/jitagent/sandbox.py similarity index 96% rename from src/agentjit/sandbox.py rename to src/jitagent/sandbox.py index 06de0f9..21a119b 100644 --- a/src/agentjit/sandbox.py +++ b/src/jitagent/sandbox.py @@ -37,7 +37,6 @@ class CallResult: class RunResult: ok: bool # did the process itself finish normally results: list[CallResult] = field(default_factory=list) - coverage: dict[str, Any] | None = None load_error: str = "" # the code failed to load or compile timed_out: bool = False killed: str = "" # "timeout" | "memory" | "signal:SIGxxx" @@ -112,19 +111,17 @@ def run( entry: str, calls: list[dict[str, Any]], *, - coverage: bool = False, timeout_ms: int = 5000, mem_mb: int = 512, ) -> RunResult: t0 = time.perf_counter() - with tempfile.TemporaryDirectory(prefix="agentjit-") as td: + with tempfile.TemporaryDirectory(prefix="jitagent-") as td: code_path = Path(td) / "generated.py" code_path.write_text(source) job = json.dumps({ "code_path": str(code_path), "entry": entry, "calls": calls, - "coverage": coverage, "timeout_ms": timeout_ms, "mem_mb": mem_mb, }) @@ -172,7 +169,6 @@ def run( results=[CallResult(**r) if r["ok"] else CallResult(ok=False, error=r.get("error", ""), tb=r.get("tb", "")) for r in payload["results"]], - coverage=payload.get("coverage"), load_error=payload.get("load_error") or "", wall_ms=wall, ) diff --git a/src/agentjit/static_check.py b/src/jitagent/static_check.py similarity index 100% rename from src/agentjit/static_check.py rename to src/jitagent/static_check.py diff --git a/src/agentjit/synth.py b/src/jitagent/synth.py similarity index 100% rename from src/agentjit/synth.py rename to src/jitagent/synth.py diff --git a/src/agentjit/types.py b/src/jitagent/types.py similarity index 100% rename from src/agentjit/types.py rename to src/jitagent/types.py diff --git a/src/agentjit/verify.py b/src/jitagent/verify.py similarity index 100% rename from src/agentjit/verify.py rename to src/jitagent/verify.py diff --git a/tests/test_cli.py b/tests/test_cli.py index 15f6a58..e27e648 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -11,10 +11,10 @@ import pytest -from agentjit import Example, Registry, Sandbox -from agentjit.cli import main -from agentjit.llm import ScriptedClient -from agentjit.synth import compile_function +from jitagent import Example, Registry, Sandbox +from jitagent.cli import main +from jitagent.llm import ScriptedClient +from jitagent.synth import compile_function REQ = ("Rank {name, score} records from highest score to lowest. " "Records with the same score share a rank.") diff --git a/tests/test_cli_client.py b/tests/test_cli_client.py index 9505022..af311b3 100644 --- a/tests/test_cli_client.py +++ b/tests/test_cli_client.py @@ -2,13 +2,13 @@ What is under test is how the CLI's JSON becomes an LLMResponse, and in particular whether the fixed harness overhead gets subtracted. Actually driving a synthesis is -`agentjit compile --via cli`; that costs tokens and does not belong in a unit test. +`jitagent compile --via cli`; that costs tokens and does not belong in a unit test. """ import json import pytest -from agentjit.llm import CLI_OVERHEAD_TOKENS, ClaudeCliClient, Refused +from jitagent.llm import CLI_OVERHEAD_TOKENS, ClaudeCliClient, Refused class FakeProc: @@ -41,7 +41,7 @@ def run(argv, **kw): def test_the_harness_overhead_is_subtracted(fake): """The CLI carries Claude Code's own system prompt and tool definitions on every call. Without subtracting them, synthesis looks an order of magnitude more expensive - than it is — that overhead belongs to Claude Code, not to agentjit.""" + than it is — that overhead belongs to Claude Code, not to jitagent.""" r = ClaudeCliClient().complete(system="S", user="U") raw = 12 + 22_000 + 400 diff --git a/tests/test_endtoend.py b/tests/test_endtoend.py index 58917d0..2ef0b30 100644 --- a/tests/test_endtoend.py +++ b/tests/test_endtoend.py @@ -5,9 +5,9 @@ """ import pytest -from agentjit import Example, Registry, Runtime, Sandbox, get_code -from agentjit.llm import ScriptedClient -from agentjit.synth import compile_function +from jitagent import Example, Registry, Runtime, Sandbox, get_code +from jitagent.llm import ScriptedClient +from jitagent.synth import compile_function REQ = ("Group rows by type and sum the amounts, returning {type: total}. " "An amount may carry a currency symbol and thousands separators; " diff --git a/tests/test_lookup.py b/tests/test_lookup.py index aaf2f11..c4159aa 100644 --- a/tests/test_lookup.py +++ b/tests/test_lookup.py @@ -12,12 +12,12 @@ """ import pytest -from agentjit import Example, Level, Registry, Report, Sandbox, Spec -from agentjit.jit import NeedsClient, compile_function, search_functions -from agentjit.llm import ScriptedClient -from agentjit.lookup import find, schema_compatible, similarity -from agentjit.synth import spec_for -from agentjit.types import GateResult +from jitagent import Example, Level, Registry, Report, Sandbox, Spec +from jitagent.jit import NeedsClient, compile_function, search_functions +from jitagent.llm import ScriptedClient +from jitagent.lookup import find, schema_compatible, similarity +from jitagent.synth import spec_for +from jitagent.types import GateResult SUM_REQ = ("Group CSV rows by the type field and sum the amount, returning " "{type: total}. The amount may carry a currency symbol and thousands " diff --git a/tests/test_propose.py b/tests/test_propose.py index eed0dbc..f6944ce 100644 --- a/tests/test_propose.py +++ b/tests/test_propose.py @@ -9,10 +9,10 @@ import pytest -from agentjit import Example, Registry, Sandbox -from agentjit.jit import compile_function -from agentjit.llm import LLMResponse, ScriptedClient -from agentjit.propose import propose_tests +from jitagent import Example, Registry, Sandbox +from jitagent.jit import compile_function +from jitagent.llm import LLMResponse, ScriptedClient +from jitagent.propose import propose_tests REQ = ("Rank {name, score} records from highest score to lowest. " "Records with the same score share a rank.") @@ -63,8 +63,8 @@ def test_assumes_is_parsed_and_marked(): def test_assumes_survives_a_round_trip_to_disk(tmp_path): """An assumption is stored with its case. A year from now, the only thing that can answer "why is this the expected value" is this field.""" - from agentjit import Level, Report, Registry, Spec - from agentjit.types import GateResult + from jitagent import Level, Report, Registry, Spec + from jitagent.types import GateResult reg = Registry(tmp_path / "r") report = Report(level=Level.VERIFIED, gates=[GateResult("static", True, "passed")]) diff --git a/tests/test_registry.py b/tests/test_registry.py index feb98ef..849dfe9 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -8,13 +8,13 @@ import pytest -from agentjit import Example, Level, Report, Spec +from jitagent import Example, Level, Report, Spec # TestSet is aliased to Suite: pytest collects any class named Test* and then # complains that it has an __init__ -from agentjit.registry import NotCacheable, Registry -from agentjit.registry import TestSet as Suite -from agentjit.registry import spec_hash -from agentjit.types import GateResult +from jitagent.registry import NotCacheable, Registry +from jitagent.registry import TestSet as Suite +from jitagent.registry import spec_hash +from jitagent.types import GateResult REQ = "Group rows by type and sum the amount" CODE = "def solve(params, ctx):\n return {'n': len(params['rows'])}\n" @@ -124,7 +124,7 @@ def test_examples_dedup_by_input(): def test_get_by_name(reg): """The handle is for machines, the name is for people. The way this actually gets used is "what was that ranking function called?".""" - from agentjit.jit import get_code + from jitagent.jit import get_code fn = reg.put(REQ, SPEC, CODE, report(), EXAMPLES, name="group_sum") assert reg.get("group_sum").spec_hash == fn.spec_hash @@ -136,7 +136,7 @@ def test_get_by_name(reg): def test_a_name_points_at_exactly_one_function(reg): """A name pointing at two functions is no index at all. Better to fail here than to let the result of get("rank") depend on directory traversal order.""" - from agentjit.registry import NameTaken + from jitagent.registry import NameTaken reg.put(REQ, SPEC, CODE, report(), EXAMPLES, name="rank") other = Spec(intent="something else", param_schema={"type": "object"}, diff --git a/tests/test_runtime.py b/tests/test_runtime.py index edc82ee..514bf01 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -6,9 +6,9 @@ """ import pytest -from agentjit import Example, Level, Registry, Report, Runtime, Sandbox, Spec -from agentjit.runtime import UnknownHandle -from agentjit.types import GateResult +from jitagent import Example, Level, Registry, Report, Runtime, Sandbox, Spec +from jitagent.runtime import UnknownHandle +from jitagent.types import GateResult REQ = ("Rank {name, score} records by descending score. " "Records with the same score share a rank.") diff --git a/tests/test_synth.py b/tests/test_synth.py index 7139f41..077b567 100644 --- a/tests/test_synth.py +++ b/tests/test_synth.py @@ -7,9 +7,9 @@ """ import pytest -from agentjit import Example, Sandbox, Thresholds -from agentjit.llm import ScriptedClient -from agentjit.synth import compile_function +from jitagent import Example, Sandbox, Thresholds +from jitagent.llm import ScriptedClient +from jitagent.synth import compile_function REQ = ("Group rows by type and sum the amounts, returning {type: total}. " "An amount may carry a currency symbol and thousands separators; " diff --git a/tests/test_units.py b/tests/test_units.py index f0bb5f0..c8cfe4e 100644 --- a/tests/test_units.py +++ b/tests/test_units.py @@ -1,10 +1,10 @@ -"""Component-level unit tests. The corpus cases (`agentjit selftest`) check end-to-end +"""Component-level unit tests. The corpus cases (`jitagent selftest`) check end-to-end behaviour; this file checks the parts.""" import pytest -from agentjit import Example, Sandbox, Spec, verify -from agentjit.static_check import check -from agentjit.types import deep_equal +from jitagent import Example, Sandbox, Spec, verify +from jitagent.static_check import check +from jitagent.types import deep_equal CODE = "def solve(params, ctx):\n return {'n': len(params['rows'])}\n" @@ -127,7 +127,7 @@ def test_strptime_is_known_broken_and_the_prompt_says_so(sb): holds, the prompt has to say so. The day the sandbox can run strptime this goes red — and that is the signal to delete the passage from the prompt. """ - from agentjit.prompts import SYSTEM + from jitagent.prompts import SYSTEM src = ('def solve(params, ctx):\n' ' return {"v": datetime.datetime.strptime(params["s"], "%Y-%m-%d").year}\n') diff --git a/tools/audit_e2e.py b/tools/audit_e2e.py index 5fddbd6..4fa9e94 100644 --- a/tools/audit_e2e.py +++ b/tools/audit_e2e.py @@ -1,4 +1,4 @@ -"""The end-to-end audit: **is the function agentjit produces actually correct?** +"""The end-to-end audit: **is the function jitagent produces actually correct?** The other script (audit_tests.py) checks whether the generated cases are right. This one checks the final product, against a yardstick **the pipeline has never seen**: @@ -8,7 +8,7 @@ existed. Take that as the answer key and run 200 random inputs through the compiled code. This is -the question that actually matters -- **"is the function agentjit produced correct"**, +the question that actually matters -- **"is the function jitagent produced correct"**, not "does the function it produced pass the tests it wrote itself". Passing its own tests but failing here = the generated cases are too weak and missed a @@ -29,10 +29,10 @@ sys.path.insert(0, "src") sys.path.insert(0, "tools") -from agentjit import Registry, Sandbox # noqa: E402 -from agentjit.jit import compile_function # noqa: E402 -from agentjit.llm import ClaudeCliClient # noqa: E402 -from agentjit.types import deep_equal # noqa: E402 +from jitagent import Registry, Sandbox # noqa: E402 +from jitagent.jit import compile_function # noqa: E402 +from jitagent.llm import ClaudeCliClient # noqa: E402 +from jitagent.types import deep_equal # noqa: E402 from audit_tests import TASKS # noqa: E402 N_RANDOM = 200 @@ -63,7 +63,7 @@ def one(task, sb, seed) -> dict: t0 = time.time() r = compile_function(task.requirement, task.examples(), client=ClaudeCliClient(model="haiku"), - registry=Registry(f"/tmp/agentjit-e2e-4/{task.key}-{seed}"), + registry=Registry(f"/tmp/jitagent-e2e-4/{task.key}-{seed}"), sandbox=sb, cache="force_new", name=None or "", gen_tests=8) row = {"key": task.key, "shape": task.shape, "seed": seed, "compiled": r.ok, "attempts": len(r.synth.attempts) if r.synth else 0, diff --git a/tools/audit_tests.py b/tools/audit_tests.py index c6359ab..7aafe02 100644 --- a/tools/audit_tests.py +++ b/tools/audit_tests.py @@ -1,4 +1,4 @@ -"""Audit the cases agentjit writes: **are the expected values actually right?** +"""Audit the cases jitagent writes: **are the expected values actually right?** The whole design rests on one assumption — that the expectations the model writes into its cases are correct. A case with a wrong expected value condemns a correct @@ -35,9 +35,9 @@ sys.path.insert(0, "src") -from agentjit import Example # noqa: E402 -from agentjit.llm import ClaudeCliClient # noqa: E402 -from agentjit.propose import propose_tests # noqa: E402 +from jitagent import Example # noqa: E402 +from jitagent.llm import ClaudeCliClient # noqa: E402 +from jitagent.propose import propose_tests # noqa: E402 @dataclass diff --git a/tools/capture_demo.sh b/tools/capture_demo.sh index 9bfcd30..d0bb030 100755 --- a/tools/capture_demo.sh +++ b/tools/capture_demo.sh @@ -16,7 +16,7 @@ set -euo pipefail cd "$(dirname "$0")/.." -HOME_DIR="${AGENTJIT_DEMO_HOME:-/tmp/agentjit-demo}" +HOME_DIR="${JITAGENT_DEMO_HOME:-/tmp/jitagent-demo}" OUT="${1:-docs/demo.raw.txt}" PY="${PYTHON:-python3}" RAW="$(mktemp -d)" @@ -31,12 +31,19 @@ show() { for a in "$@"; do case "$a" in *[\ \{\}\"]*) out="$out '$a'" ;; *) out="$out $a" ;; esac done - printf '$ agentjit%s\n' "$out" >> "$RAW/log" + printf '$ jitagent%s\n' "$out" >> "$RAW/log" } run() { show "$@" - AGENTJIT_HOME="$HOME_DIR" "$PY" -m agentjit.cli "$@" 2>&1 | tee -a "$RAW/log" >/dev/null + # The CLI's output goes to the transcript, not to the terminal — but if it fails, the + # operator is left with a bare exit code and the trap then deletes the log. A synthesis + # call can fail transiently, so say what happened. + if ! JITAGENT_HOME="$HOME_DIR" "$PY" -m jitagent.cli "$@" >>"$RAW/log" 2>&1; then + echo "jitagent $* failed:" >&2 + tail -n 20 "$RAW/log" >&2 + exit 1 + fi printf '~~~\n' >> "$RAW/log" } diff --git a/tools/make_demo_svg.py b/tools/make_demo_svg.py index 333aa4e..a46e6fc 100644 --- a/tools/make_demo_svg.py +++ b/tools/make_demo_svg.py @@ -220,7 +220,7 @@ def main(argv: list[str]) -> int: f"what the case pins down.", file=sys.stderr) return 1 - dst.write_text(render(lines, total, "agentjit — text in, code out, fetch it by name")) + dst.write_text(render(lines, total, "jitagent — text in, code out, fetch it by name")) print(f"{dst} {len(lines)} lines, {total:.1f}s loop, {dst.stat().st_size // 1024}KB") return 0