From be810d138b9d947537914c042b435fe7a32c6780 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Sep 2026 09:14:55 +0000 Subject: [PATCH 01/42] Genesis: a fifth domain that grows a product rather than a library `fly`: a Drosophila brain, three assays, an HTTP backend and the page a person watches it learn on. The four domains before it grow a library; this one is the first asked for a frontend, so the specification alone has to carry the design. It is also the first shipped with no reference implementation, and the cost is stated rather than hidden: no offline actor, so `--domain fly` without a model now says so instead of raising AttributeError, and no kill_report, so nothing in the repository checks what this suite rejects. What stands in its place is that every number in the specification was measured on a throwaway implementation while the suite was being written -- 91 assertions passing, and three of the specification's least obvious paragraphs are there because that implementation failed them first: a ring attractor sharpened with any pointwise nonlinearity quantises heading to the nearest wedge; an avoidance assay with the punished source in the middle of a walled arena measures the arena, not the animal; and `src/learn/` beside a public `learn()` is two things called `src.learn`, where the submodule import rebinds the name. That last one is the same collision this port was bitten by twice from the other direction, which is why the specification names it and test_integration.py asserts it. 93 tasks over 8 test files, 10 of them an audit set no agent can read. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/genesis/_fly.py | 1339 ++++++++++++++++++ examples/genesis/genesis_recursive_worlds.py | 18 +- 2 files changed, 1355 insertions(+), 2 deletions(-) create mode 100644 examples/genesis/_fly.py diff --git a/examples/genesis/_fly.py b/examples/genesis/_fly.py new file mode 100644 index 00000000..6acb6254 --- /dev/null +++ b/examples/genesis/_fly.py @@ -0,0 +1,1339 @@ +"""The fifth formation domain: a fruit fly, its brain, and a web page to watch it on. + +The four domains before this one grow a *library*. This one grows a **product**: a +simulated animal, the tasks it is set, the dopamine that changes what it does next, an +HTTP backend and the page a person actually looks at -- and it is the first where the +specification alone had to carry the design, because nothing in this port has ever been +asked for a frontend before. + +It is also the first domain shipped **without a reference implementation**. The others +keep one beside them for two jobs that are not scoring: driving the offline rule-based +actor, and letting a test prove the suite is passable at all. Here there is none, and +the cost is stated rather than hidden. There is no `--domain fly` run without a model, +and no `kill_report`: nothing in the repository checks what this suite *rejects*, which +is the check that caught a molecular-dynamics run reporting 1.000 for a force field +that was identically zero. What stands in its place is that every number in the +specification was measured on a throwaway implementation while the suite was being +written -- 91 assertions passing, and three of the specification's least obvious +paragraphs are there because that implementation failed them first: + +* a ring attractor sharpened with any *pointwise* nonlinearity quantises heading to the + nearest wedge, so the lateral term has to be linear; +* an avoidance assay with the punished source in the middle of a walled arena measures + the arena, not the animal -- the model's valence had correctly reversed by tick 40 and + it still took 61 shocks an episode, because the walls kept reflecting it back through + the gradient; +* and `src/learn/` beside a public `learn()` is two things called `src.learn`, where the + submodule import rebinds the name and every caller after it gets `TypeError: 'module' + object is not callable`. + +The last of those is the same collision this port had already been bitten by twice from +the other direction, which is why the specification names it and `test_integration.py` +asserts it. +""" + +from __future__ import annotations + +from typing import Dict + +from ._suite import PYTHON_MODULE_SKILL, TEST_FAILURE, TestSuite, reward_test +from ._suite import llm_executor as _llm_executor +from ._suite import llm_manager as _llm_manager +from ._world import SKILLS_DIR + +__all__ = ["CASE_NOUN", "CONTRACTS", "FLY", "FROZEN", "GROUP_NOUN", "HELD_OUT_FRAC", + "OBJECTIVE", "REQUIRES_MODEL", "SCORING", "build_tasks", "initial_files", + "llm_executor", "llm_manager", "make_runner", "reward", "suite_failures", + "suite_review"] + +#: The specification, the suite and the driver. Human-supplied, refused to every +#: proposal, and restored pristine before scoring. +FROZEN = ("spec/**", "tests/**", "fly.py") + +#: What is pushed into every brief, in this order. +CONTRACTS = ("spec/**", "fly.py") + +SCORING = "frozen test suite, no reference implementation anywhere in the port" +CASE_NOUN = "test functions" +GROUP_NOUN = "test files" + +ENTRY = "src/__init__.py" + +#: There is no offline actor for this domain, and pretending otherwise would mean +#: shipping the decomposition the run is meant to invent. See the module docstring. +REQUIRES_MODEL = True + +OBJECTIVE = ( + "Build the Drosophila model the frozen driver `fly.py` and the frozen test suite " + "both import: a Python package rooted at `src/`, whose `src/__init__.py` exposes " + "exactly `make_brain`, `step_brain`, `sense`, `act`, `learn`, `make_world`, " + "`step_world`, `run_episode`, `train` and `handle`. It simulates a fly's olfactory " + "brain -- antennal lobe, mushroom body, lateral horn, central complex -- walking in " + "an arena, being punished or rewarded, and changing what it does next; and it " + "serves the whole thing as a web page a person can watch it learn on. " + "`spec/CONTEXT.md` pins those signatures, the circuit and the assays; everything " + "below `src/` -- how many modules, what they are called, who owns what -- is yours " + "to decide. Pure Python, no third-party packages." +) + +#: The specification. The only design a person wrote. +_SPEC = r'''# Specification — a Drosophila brain you can watch learn + +Build the library the frozen driver `fly.py` and the frozen suite in `tests/` import: a +pure-Python package rooted at `src/` that simulates a fruit fly — its olfactory brain, +its body in an arena, the tasks it is set, and the dopamine that changes what it does +next — and serves the whole thing as a web page. + +Reduced units throughout: one body length, one second, one arbitrary unit of odour. +Pure Python, no third-party packages, no numpy. Determinism is a requirement, not a +convenience: the same seed must give the same trajectory on any machine. + +## What the animal is + +Four structures, and the division of labour between them is the point of the model. + +**Antennal lobe.** 24 glomeruli. Olfactory receptor neurons respond across orders of +magnitude of concentration and the projection neurons they drive do not: lateral +inhibition divides each glomerulus by the activity of the whole lobe, so the PN +*pattern* says which odour this is and carries almost nothing about how much of it +there is. Use `r^1.5 / (sigma^1.5 + sum_j r_j^1.5)` with `sigma = 12`. + +**Mushroom body.** 400 Kenyon cells, each sampling 6 glomeruli at random, fixed for the +life of the animal. Feedback inhibition (the APL neuron) leaves the most strongly driven +**5%** active — a rank, not a fixed threshold, so the code stays sparse for a faint +odour and a strong one alike. Two output neurons: MBON 0 is *approach*, MBON 1 is +*avoid*, each reading the active cells through plastic weights that start at 1.0. + +**Learning is depression, and this is the part that is usually modelled backwards.** A +Kenyon cell active at the same moment as a dopaminergic neuron has its synapse onto that +DAN's MBON *weakened*, floored at zero — `w -= 0.5 * kc * da`. Punishment therefore does +not build an avoid response; it removes the approach response for that odour and lets +the untouched avoid pathway win. Nothing is learned about an odour that was absent, +because its cells were silent and the rule is a coincidence rule. + +**Lateral horn.** The projection neurons fork. One branch goes to the mushroom body, +where experience can change what an odour means; the other goes to the lateral horn, +which is stereotyped and does not learn. A naive fly walks towards food odour without +ever having met it. Innate valence is `1.0 * sum(projection)` — positive, so every +odour here starts attractive. + +Keeping these two apart is what lets learning show up as a **reversal**: the lateral +horn says approach, the mushroom body after punishment subtracts more than that, the sum +changes sign, and the animal turns around. Fuse them into one number and there is +nothing left to reverse. + +**Central complex.** A ring of 16 wedges holding one bump of activity whose position is +the heading. There is no compass: the bump is *shifted* by the animal's own turning, so +heading is dead reckoning and it runs in the dark. Two properties, and both are tested. +Turning by an angle and back must return the bump where it started. And the bump must +neither smear flat nor collapse onto a single wedge — hold it with a **linear** filter, +`(1+2a)` on the wedge against `a` on each neighbour at `a = 0.1`. Any *pointwise* +nonlinearity — squaring, thresholding — snaps the bump toward the nearest wedge and +quantises every heading; measured over four full turns, no lateral term drifts 0.45 rad, +a pointwise power loses the heading entirely, and the linear filter drifts 0.04. + +## What the animal has for a body + +A square arena, half-width 20, with **reflecting** walls: a fly in a dish has edges, and +an animal that learns to leave the arena has learned nothing. Odour sources each carry a +receptor profile and a concentration falling off as `exp(-distance / 6)`. + +**Two antennae**, 0.6 apart, and the animal steers on the difference between them: +`turn = wanted * 12 * (left - right) / (left + right)`, where `wanted` is `+1` when +valence is positive and `-1` when it is negative. That sign is the entire behavioural +consequence of learning. + +Bilateral steering has a blind spot exactly where it matters most — pointed straight at +a source both antennae read the same thing — so the temporal gradient covers the head-on +case: when the odour is changing the way the animal does *not* want, it turns, and which +way is one bit of memory it carries between ticks. The relative change is about 2.5% a +tick, so that term needs a gain around 200 to be worth anything. + +Walking speed is `4 * (1 + 0.6 * tanh(valence))`, turning is capped at 3 rad/s, the tick +is 0.05 s and an episode is at most 400 ticks. + +## Three assays + +| task | arena | paid | +|---|---|---| +| `taxis` | one safe source at the centre | `+1` on arrival, which ends the episode | +| `avoid` | one punished source, off centre | `-1` every tick within 5 of it | +| `choice` | a punished source and a safe one, opposite ends | both of the above | + +`avoid` and `choice` put the punished source **off centre on purpose**. With it in the +middle of a walled arena there is nowhere to go: the animal learns the odour is bad, +turns away, hits the wall, is reflected back through the gradient and is punished again +— 61 shocks an episode from a model whose valence had correctly reversed by tick 40. An +avoidance assay has to contain somewhere safe or it measures the arena. + +Reward is paid **per tick**, not per episode, because the plasticity rule is a +coincidence rule: it can only act on what the animal was smelling when the dopamine +arrived. + +## `src/__init__.py` exposes exactly these ten names + +```python +make_brain(config=None) -> dict + # config keys: seed, glomeruli, kenyon_cells, wedges, heading. All optional. + +step_brain(brain, sensory, dt=0.05) -> activity # mutates brain +sense(world) -> sensory +act(activity) -> (forward, turn) +learn(brain, activity, reward) -> brain # mutates brain + +make_world(task="taxis", seed=0, glomeruli=24) -> world +step_world(world, action, dt=0.05) -> (world, reward, done) + +run_episode(brain, task="taxis", seed=0, dt=0.05, learning=True) -> result +train(task="taxis", episodes=20, seed=0) -> {"task", "episodes": [...], "brain"} + +handle(method, path, body=None) -> (status, content_type, body) +``` + +Each MUST import its layer **lazily**, inside the function body, so a missing layer does +not stop the layers before it from passing their tests. + +**No module below `src/` may be named after one of these ten.** A package laid out with +`src/learn/` beside a public `learn()` has two things called `src.learn`; the moment +anything inside imports the submodule, the name on the package is rebound to the module +and every caller gets `TypeError: 'module' object is not callable`. Tests that call the +function first still pass. The one that calls `train()` first does not. + +### `sensory` +`odor` (one float per glomerulus), `odor_left`, `odor_right`, `angular_velocity`. + +### `activity` +`projection` (24 floats), `kenyon` (one 0.0/1.0 per cell), `mbon` (2 floats, approach +then avoid), `ring` (one float per wedge, summing to 1), `heading` (the compass, radians +in `(-pi, pi]`), `innate`, `learned` (= `mbon[0] - mbon[1]`), `valence` (= `innate + +learned`), `delta_drive`, `tumble_sign`, `odor_left`, `odor_right`. + +### `world` +`task`, `seed`, `glomeruli`, `t`, `step`, `position` (x, y), `heading`, +`angular_velocity`, `sources` (each `name`, `position`, `profile`, `strength`, +`punished`), `shocked`. + +### `result` from an episode +`reward`, `shocks`, `steps`, `nearest`, `distance`, `reached`. + +## The web page + +`handle` is a **pure function** — it opens no socket. `fly.py` wraps it in +`http.server` and does nothing else, so the part worth testing is testable without one. + +| route | does | +|---|---| +| `GET /` | the page: one HTML file, no build step, no CDN, no network | +| `GET /api/tasks` | `{"tasks": [...]}` | +| `GET /api/state` | the arena, the brain, and the episode history | +| `POST /api/reset` | `{task, seed, brain}` — `brain: "new"` forgets what was learned | +| `POST /api/step` | `{steps, learning}` — advance the live animal | +| `POST /api/train` | `{task, episodes}` — run episodes, append to the history | + +Anything else is `404` with a JSON body; a body that is not JSON is `400`. + +The page must draw, from `/api/state` and nothing else: the **arena** with the animal, +the sources and the punished region; the **Kenyon-cell code** as a raster with the +active cells marked; the **ellipsoid body** as a ring with the bump and the compass +heading; the **valence** panel breaking the number into innate, approach, avoid and net; +and the **learning curve**, shocks per episode. Controls for task, play/pause, step, +train, and a fresh brain. It must work at phone width and follow the viewer's colour +scheme. + +## How this is scored + +A frozen suite in `tests/`, and no reference implementation anywhere in the loop. +One task per test function; the prompt is that test's own source. + +| file | asserts | +|---|---| +| `test_antennal_lobe.py` | bounds, compression in the range the arena uses, order preserved under rescaling | +| `test_mushroom_body.py` | sparsity at every concentration, the same odour lights the same cells, separation, depression only, compartment specificity | +| `test_central_complex.py` | one bump, neither flat nor collapsed, reversible, accurate over four turns, runs in the dark | +| `test_body.py` | two antennae, the sign of steering, walls, containment, determinism | +| `test_tasks.py` | each assay's shape, that `avoid` has a safe region, what is paid and when | +| `test_learning.py` | valence reverses, other odours keep theirs, shocks fall, and it transfers to arenas it never saw | +| `test_server.py` | every route, every JSON shape, the page, 404 and 400 | +| `test_integration.py` | the loop closes by hand, and no module shadows an entry point | + +A further set of assertions is **held out**: the same requirements asked differently, in +files that are not in the repository and that no agent can read, injected only at +scoring time. That is the honest number, because an executor is shown the source of the +test it is failing and could otherwise write to that one assertion. +''' + +#: The driver: a program you can run, rather than a package nobody can invoke. +_DRIVER = r'''#!/usr/bin/env python3 +"""fly -- a Drosophila brain you can watch learn. + +Human-supplied, frozen, and never written by an agent: the model under ``src/`` is what +the run grows, and this is the shell around it, so the result is a program you can run +rather than a package nobody can invoke. + + python fly.py train --task avoid --episodes 20 + python fly.py probe --task avoid --episodes 10 --trials 5 + python fly.py serve --port 8000 + +It touches the library only through the entry points ``spec/CONTEXT.md`` pins, so it +runs against any implementation that passes ``tests/``, whatever the internal layout +turns out to be. Pure Python and no third-party packages: this has to run anywhere. +""" + +import argparse +import json +import sys + + +def cmd_train(args): + from src import train + result = train(args.task, args.episodes, args.seed) + rows = result["episodes"] + print(f"{'ep':>4} {'shocks':>7} {'reward':>8} {'steps':>6} nearest") + for row in rows: + print(f"{row['episode']:>4} {row['shocks']:>7} {row['reward']:>8.1f} " + f"{row['steps']:>6} {row['nearest']}") + head = sum(r["shocks"] for r in rows[:3]) / max(1, len(rows[:3])) + tail = sum(r["shocks"] for r in rows[-3:]) / max(1, len(rows[-3:])) + print(f"\nfirst 3 episodes: {head:.1f} shocks last 3: {tail:.1f}") + return 0 + + +def cmd_probe(args): + """Train one animal, then test it and a naive one with plasticity switched off. + + The honest measurement of learning: same arenas, seeds neither animal was trained + on, and nothing changing during the test. + """ + from src import make_brain, run_episode, train + trained = train(args.task, args.episodes, args.seed)["brain"] + naive = make_brain({"seed": args.seed}) + out = {} + for label, brain in (("naive", naive), ("trained", trained)): + shocks = [run_episode(brain, args.task, 1000 + i, learning=False)["shocks"] + for i in range(args.trials)] + out[label] = sum(shocks) + print(f"{label:>8}: {sum(shocks):>5} shocks over {args.trials} unseen arenas {shocks}") + print(f"\nlearned reduction: {out['naive'] - out['trained']} shocks") + return 0 + + +def cmd_serve(args): + from http.server import BaseHTTPRequestHandler, HTTPServer + + from src import handle + + class Handler(BaseHTTPRequestHandler): + def _respond(self, method): + length = int(self.headers.get("content-length") or 0) + body = self.rfile.read(length).decode("utf-8") if length else None + status, content_type, payload = handle(method, self.path, body) + data = payload.encode("utf-8") + self.send_response(status) + self.send_header("content-type", content_type) + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_GET(self): + self._respond("GET") + + def do_POST(self): + self._respond("POST") + + def log_message(self, *a): + pass + + server = HTTPServer((args.host, args.port), Handler) + print(f"http://{args.host}:{args.port}/") + sys.stdout.flush() + server.serve_forever() + return 0 + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = parser.add_subparsers(dest="cmd", required=True) + for name, fn in (("train", cmd_train), ("probe", cmd_probe), ("serve", cmd_serve)): + p = sub.add_parser(name) + p.set_defaults(fn=fn) + if name in ("train", "probe"): + p.add_argument("--task", default="avoid") + p.add_argument("--episodes", type=int, default=20) + p.add_argument("--seed", type=int, default=0) + if name == "probe": + p.add_argument("--trials", type=int, default=5) + if name == "serve": + p.add_argument("--host", default="127.0.0.1") + p.add_argument("--port", type=int, default=8000) + args = parser.parse_args(argv) + return args.fn(args) + + +if __name__ == "__main__": + raise SystemExit(main()) +''' + +#: The suite. One task per test function; the prompt is the test's own source. +_TESTS = { + 'tests/test_antennal_lobe.py': r'''"""The antennal lobe: what survives a change of concentration, and what does not.""" + +from src import make_brain, step_brain + +GLOMERULI = 24 + + +def odour(seed, n=GLOMERULI): + """A reproducible receptor pattern. Squared uniforms: a few glomeruli dominate.""" + import random + rng = random.Random(seed) + return [rng.random() ** 2 for _ in range(n)] + + +def present(brain, receptors, scale=1.0): + return step_brain(brain, {"odor": [r * scale for r in receptors]}) + + +def test_projection_is_one_rate_per_glomerulus(): + brain = make_brain({"seed": 0}) + activity = present(brain, odour(1)) + assert len(activity["projection"]) == GLOMERULI + + +def test_projection_rates_are_bounded_below_one(): + """Divisive normalisation: a glomerulus is a share of the lobe, never more than it.""" + brain = make_brain({"seed": 0}) + for scale in (0.01, 1.0, 100.0, 10000.0): + activity = present(brain, odour(2), scale) + assert all(0.0 <= p < 1.0 for p in activity["projection"]) + assert sum(activity["projection"]) < 1.0 + + +def test_a_silent_antenna_gives_a_silent_lobe(): + brain = make_brain({"seed": 0}) + activity = step_brain(brain, {"odor": [0.0] * GLOMERULI}) + assert all(p == 0.0 for p in activity["projection"]) + + +def test_the_lobe_compresses_concentration_where_the_arena_puts_it(): + """Ten times the odour is nowhere near ten times the response -- in range. + + The range matters and the test states it. Far below saturation the lobe is if + anything expansive; what the arena actually delivers at a source is an order of + magnitude above that, and there ten times the odour buys a few per cent. + """ + brain = make_brain({"seed": 0}) + weak = sum(present(brain, odour(3), 10.0)["projection"]) + strong = sum(present(brain, odour(3), 100.0)["projection"]) + assert weak < strong < 1.5 * weak + + +def test_more_odour_is_never_less_response(): + brain = make_brain({"seed": 0}) + totals = [sum(present(brain, odour(7), s)["projection"]) + for s in (0.1, 1.0, 10.0, 100.0, 1000.0)] + assert totals == sorted(totals) + + +def test_the_strongest_glomerulus_stays_the_strongest(): + """Normalisation rescales the pattern; it must not reorder it.""" + brain = make_brain({"seed": 0}) + receptors = odour(4) + loudest = max(range(GLOMERULI), key=lambda g: receptors[g]) + for scale in (0.5, 5.0, 50.0): + projection = present(brain, receptors, scale)["projection"] + assert max(range(GLOMERULI), key=lambda g: projection[g]) == loudest + + +def test_two_odours_give_two_different_patterns(): + brain = make_brain({"seed": 0}) + a = present(brain, odour(5))["projection"] + b = present(brain, odour(6))["projection"] + assert any(abs(x - y) > 1e-6 for x, y in zip(a, b)) +''', + 'tests/test_body.py': r'''"""The body: two antennae, a pair of legs, and walls that do not let go.""" + +import math + +from src import act, make_world, sense, step_world + +SIZE = 20.0 + + +def test_sensing_gives_one_reading_per_glomerulus_and_one_per_antenna(): + world = make_world("taxis", 0) + sensory = sense(world) + assert len(sensory["odor"]) == world["glomeruli"] + assert sensory["odor_left"] >= 0.0 and sensory["odor_right"] >= 0.0 + + +def test_the_antennae_read_more_odour_nearer_the_source(): + near = make_world("taxis", 0) + near["position"] = (2.0, 0.0) + far = make_world("taxis", 0) + far["position"] = (18.0, 0.0) + assert sum(sense(near)["odor"]) > 10.0 * sum(sense(far)["odor"]) + + +def test_the_antenna_facing_the_source_reads_more(): + """Bilateral comparison is the only thing that tells the animal which way to turn.""" + world = make_world("taxis", 0) + world["position"] = (10.0, 0.0) + # Facing +y from x=+10 with the source at the origin puts it to the animal's left. + world["heading"] = math.pi / 2.0 + assert sense(world)["odor_left"] > sense(world)["odor_right"] + # Turn it around and the same source is on the other side. + world["heading"] = -math.pi / 2.0 + assert sense(world)["odor_right"] > sense(world)["odor_left"] + + +def test_an_attractive_odour_turns_the_animal_toward_the_stronger_antenna(): + left = act({"mbon": [1.0, 0.0], "valence": 1.0, "odor_left": 2.0, + "odor_right": 1.0, "delta_drive": 0.0})[1] + right = act({"mbon": [1.0, 0.0], "valence": 1.0, "odor_left": 1.0, + "odor_right": 2.0, "delta_drive": 0.0})[1] + assert left > 0.0 > right + + +def test_an_aversive_odour_turns_the_animal_the_other_way(): + """The same gradient, the opposite sign -- and that is all learning changes.""" + attracted = act({"mbon": [1.0, 0.0], "valence": 1.0, "odor_left": 2.0, + "odor_right": 1.0, "delta_drive": 0.0})[1] + repelled = act({"mbon": [0.0, 1.0], "valence": -1.0, "odor_left": 2.0, + "odor_right": 1.0, "delta_drive": 0.0})[1] + assert attracted > 0.0 > repelled + + +def test_the_animal_walks_faster_toward_something_it_likes(): + liked = act({"mbon": [1.0, 0.0], "valence": 1.0, "odor_left": 1.0, + "odor_right": 1.0, "delta_drive": 0.0})[0] + disliked = act({"mbon": [0.0, 1.0], "valence": -1.0, "odor_left": 1.0, + "odor_right": 1.0, "delta_drive": 0.0})[0] + assert liked > disliked > 0.0 + + +def test_the_animal_stays_in_the_arena(): + world = make_world("taxis", 0) + for _ in range(600): + world, _, _ = step_world(world, (8.0, 0.3)) + assert abs(world["position"][0]) <= SIZE + 1e-6 + assert abs(world["position"][1]) <= SIZE + 1e-6 + + +def test_walking_into_a_wall_turns_the_animal_around(): + world = make_world("taxis", 0) + world["position"] = (SIZE - 0.1, 0.0) + world["heading"] = 0.0 + world, _, _ = step_world(world, (8.0, 0.0)) + assert math.cos(world["heading"]) < 0.0 + + +def test_the_world_is_a_function_of_its_seed(): + one = make_world("taxis", 3) + two = make_world("taxis", 3) + assert one["position"] == two["position"] + assert make_world("taxis", 4)["position"] != one["position"] + + +def test_a_step_reports_reward_and_whether_the_episode_is_over(): + world = make_world("taxis", 0) + world, reward, done = step_world(world, (0.0, 0.0)) + assert isinstance(reward, float) and isinstance(done, bool) +''', + 'tests/test_central_complex.py': r'''"""The ellipsoid body: one bump, and what it costs to move it accurately.""" + +import math + +from src import make_brain, step_brain + +SILENT = {"odor": [0.0] * 24} + + +def turn(brain, rate, seconds, dt=0.05): + """Spin the animal at `rate` rad/s for `seconds` and return the last activity.""" + activity = None + for _ in range(int(round(seconds / dt))): + activity = step_brain(brain, dict(SILENT, angular_velocity=rate), dt) + return activity + + +def wrapped(angle): + return math.atan2(math.sin(angle), math.cos(angle)) + + +def test_the_ring_is_a_distribution(): + brain = make_brain({"seed": 0}) + ring = step_brain(brain, SILENT)["ring"] + assert len(ring) >= 8 + assert all(a >= 0.0 for a in ring) + assert abs(sum(ring) - 1.0) < 1e-6 + + +def test_there_is_exactly_one_bump(): + brain = make_brain({"seed": 0}) + ring = turn(brain, 0.8, 1.3)["ring"] + n = len(ring) + peaks = sum(1 for i in range(n) if ring[i] > ring[i - 1] and ring[i] >= ring[(i + 1) % n]) + assert peaks == 1, ring + + +def test_the_bump_neither_smears_flat_nor_collapses_to_one_wedge(): + """Both failures look fine on a reversibility test and neither is a compass. + + Flat is what happens with no lateral term at all; one wedge is what a pointwise + nonlinearity gives, and it quantises every heading to a multiple of 22.5 degrees. + """ + brain = make_brain({"seed": 0}) + ring = turn(brain, 1.0, 2.0)["ring"] + uniform = 1.0 / len(ring) + assert max(ring) > 2.0 * uniform + assert max(ring) < 0.6 + + +def test_turning_back_returns_the_bump_where_it_started(): + """The integration is reversible, which is the property a dead-reckoner needs.""" + brain = make_brain({"seed": 0}) + start = step_brain(brain, SILENT)["heading"] + turn(brain, 1.0, 1.0) + back = turn(brain, -1.0, 1.0)["heading"] + assert abs(wrapped(back - start)) < 0.05 + + +def test_the_heading_follows_the_turn(): + brain = make_brain({"seed": 0}) + start = step_brain(brain, SILENT)["heading"] + after = turn(brain, 1.0, 1.0)["heading"] + assert wrapped(after - start) > 0.5 + back = turn(brain, -2.0, 1.0)["heading"] + assert wrapped(back - after) < -0.5 + + +def test_the_compass_survives_several_full_turns(): + """Four turns of dead reckoning, and the error stays a fraction of a wedge.""" + brain = make_brain({"seed": 0}) + start = step_brain(brain, SILENT)["heading"] + total = 4.0 * 2.0 * math.pi + after = turn(brain, 1.0, total)["heading"] + assert abs(wrapped(after - start - total)) < 0.4 + + +def test_standing_still_does_not_move_the_compass(): + brain = make_brain({"seed": 0}) + start = step_brain(brain, SILENT)["heading"] + after = turn(brain, 0.0, 5.0)["heading"] + assert abs(wrapped(after - start)) < 1e-6 + + +def test_the_compass_runs_in_the_dark(): + """Nothing external drives it: with no odour at all the bump still tracks turning.""" + brain = make_brain({"seed": 0}) + after = turn(brain, 1.5, 1.0)["heading"] + assert abs(wrapped(after - 1.5)) < 0.3 +''', + 'tests/test_integration.py': r'''"""The whole animal, and the one way a package can break itself.""" + +import json +import random + +from src import (act, handle, learn, make_brain, make_world, run_episode, sense, + step_brain, step_world, train) + + +def test_every_public_name_is_callable(): + """A module is not a function. This catches the one import mistake that hides. + + A package laid out so that `src/learn/` sits beside a public `learn()` has two + things called `src.learn`, and the moment anything inside imports the submodule the + name on the package is rebound to the module. Every test that calls the function + first still passes; the one that calls `train()` first gets `TypeError: 'module' + object is not callable`. No module below `src/` may be named after an entry point. + """ + for name in (act, handle, learn, make_brain, make_world, run_episode, sense, + step_brain, step_world, train): + assert callable(name), name + + +def test_training_first_does_not_break_the_other_entry_points(): + """The order that catches it: touch the deep layers, then use the shallow names.""" + train("taxis", episodes=1, seed=0) + handle("GET", "/api/state") + brain = make_brain({"seed": 0}) + rng = random.Random(0) + activity = step_brain(brain, {"odor": [rng.random() for _ in range(24)]}) + learn(brain, activity, -1.0) + world = make_world("taxis", 0) + step_world(world, act(activity)) + run_episode(brain, "taxis", 0, learning=False) + + +def test_the_loop_closes_by_hand(): + """Sense, think, act, move, be paid, learn -- in that order, with nothing hidden.""" + brain = make_brain({"seed": 0}) + world = make_world("avoid", 0) + total = 0.0 + for _ in range(120): + activity = step_brain(brain, sense(world)) + world, reward, done = step_world(world, act(activity)) + total += reward + if reward: + learn(brain, activity, reward) + if done: + break + assert world["step"] > 0 + assert total <= 0.0 + + +def test_the_animal_the_server_shows_is_the_animal_that_learns(): + handle("POST", "/api/reset", json.dumps({"task": "avoid", "seed": 0, "brain": "new"})) + handle("POST", "/api/step", json.dumps({"steps": 1})) + before = json.loads(handle("GET", "/api/state")[2])["brain"]["learned"] + handle("POST", "/api/train", json.dumps({"task": "avoid", "episodes": 4})) + handle("POST", "/api/step", json.dumps({"steps": 1})) + after = json.loads(handle("GET", "/api/state")[2])["brain"]["learned"] + assert before == 0.0 > after + + +def test_a_brain_is_plain_data(): + """No classes required, and nothing that cannot be written to a file.""" + brain = make_brain({"seed": 0}) + assert isinstance(brain, dict) + + +def test_the_model_scales_down(): + """The sizes are configuration, not constants baked into the wiring.""" + small = make_brain({"seed": 0, "kenyon_cells": 60, "wedges": 8}) + activity = step_brain(small, {"odor": [1.0] * 24}) + assert len(activity["kenyon"]) == 60 + assert len(activity["ring"]) == 8 +''', + 'tests/test_learning.py': r'''"""What the animal does differently after it has been punished.""" + +import random + +from src import learn, make_brain, run_episode, step_brain, train + +GLOMERULI = 24 + + +def odour(seed, n=GLOMERULI): + rng = random.Random(seed) + return [rng.random() ** 2 for _ in range(n)] + + +def present(brain, receptors, scale=10.0): + return step_brain(brain, {"odor": [r * scale for r in receptors]}) + + +def test_a_naive_animal_finds_every_odour_attractive(): + """Innate valence comes from the lateral horn, and it is there before experience.""" + brain = make_brain({"seed": 0}) + for seed in range(5): + assert present(brain, odour(seed))["valence"] > 0.0 + + +def test_valence_is_innate_plus_learned_and_nothing_else(): + brain = make_brain({"seed": 0}) + activity = present(brain, odour(30)) + assert abs(activity["valence"] - (activity["innate"] + activity["learned"])) < 1e-9 + + +def test_a_naive_animal_has_learned_nothing(): + brain = make_brain({"seed": 0}) + assert abs(present(brain, odour(31))["learned"]) < 1e-9 + + +def test_punishment_reverses_an_odour_the_animal_used_to_approach(): + """The headline: the sign of valence flips, and the innate part never moved.""" + brain = make_brain({"seed": 0}) + before = present(brain, odour(32)) + for _ in range(8): + learn(brain, present(brain, odour(32)), -1.0) + after = present(brain, odour(32)) + assert before["valence"] > 0.0 > after["valence"] + assert abs(after["innate"] - before["innate"]) < 1e-9 + assert after["learned"] < -0.5 + + +def test_an_odour_that_was_never_punished_keeps_its_valence(): + brain = make_brain({"seed": 0}) + before = present(brain, odour(34))["valence"] + for _ in range(8): + learn(brain, present(brain, odour(33)), -1.0) + assert present(brain, odour(34))["valence"] > 0.8 * before + + +def test_training_reduces_the_shocks_an_animal_takes(): + result = train("avoid", episodes=12, seed=0) + shocks = [e["shocks"] for e in result["episodes"]] + first = sum(shocks[:3]) / 3.0 + last = sum(shocks[-3:]) / 3.0 + assert first > 0.0 + assert last < 0.5 * first + + +def test_what_was_learned_transfers_to_arenas_it_never_saw(): + """Plasticity off, seeds it was not trained on: the only honest test of learning.""" + trained = train("avoid", episodes=10, seed=0)["brain"] + naive = make_brain({"seed": 0}) + unseen = range(1000, 1005) + before = sum(run_episode(naive, "avoid", s, learning=False)["shocks"] for s in unseen) + after = sum(run_episode(trained, "avoid", s, learning=False)["shocks"] for s in unseen) + assert before > 0 + assert after < 0.25 * before + + +def test_training_reports_one_row_per_episode(): + result = train("taxis", episodes=4, seed=0) + assert len(result["episodes"]) == 4 + for i, row in enumerate(result["episodes"]): + assert row["episode"] == i + for key in ("shocks", "reward", "steps", "nearest", "distance"): + assert key in row + + +def test_training_is_reproducible(): + one = train("avoid", episodes=5, seed=0)["episodes"] + two = train("avoid", episodes=5, seed=0)["episodes"] + assert [e["shocks"] for e in one] == [e["shocks"] for e in two] + + +def test_an_episode_with_learning_off_changes_nothing(): + brain = make_brain({"seed": 0}) + run_episode(brain, "avoid", 0, learning=False) + assert abs(present(brain, odour(35))["learned"]) < 1e-9 + + +def test_a_naive_animal_walks_to_an_unpunished_source(): + """Innate approach plus bilateral steering is enough; nothing is learned here.""" + brain = make_brain({"seed": 0}) + reached = sum(run_episode(brain, "taxis", s, learning=False)["reached"] + for s in range(5)) + assert reached >= 4 +''', + 'tests/test_mushroom_body.py': r'''"""Kenyon cells: a sparse code, the same every time, and what dopamine does to it.""" + +from src import learn, make_brain, step_brain + +GLOMERULI = 24 + + +def odour(seed, n=GLOMERULI): + import random + rng = random.Random(seed) + return [rng.random() ** 2 for _ in range(n)] + + +def present(brain, receptors, scale=10.0): + return step_brain(brain, {"odor": [r * scale for r in receptors]}) + + +def active(code): + return {i for i, c in enumerate(code) if c} + + +def test_the_code_is_sparse(): + """A few per cent of the population, which is what makes odours separable.""" + brain = make_brain({"seed": 0}) + for seed in range(8): + code = present(brain, odour(seed))["kenyon"] + fraction = len(active(code)) / len(code) + assert 0.01 <= fraction <= 0.15, fraction + + +def test_the_code_is_sparse_at_every_concentration(): + """Feedback inhibition sets a rank, not a fixed threshold.""" + brain = make_brain({"seed": 0}) + for scale in (0.1, 1.0, 10.0, 1000.0): + code = present(brain, odour(9), scale)["kenyon"] + assert 0.01 <= len(active(code)) / len(code) <= 0.15 + + +def test_the_same_odour_always_lights_the_same_cells(): + brain = make_brain({"seed": 0}) + first = active(present(brain, odour(10))["kenyon"]) + for _ in range(5): + assert active(present(brain, odour(10))["kenyon"]) == first + + +def test_concentration_does_not_change_which_cells_fire(): + """Identity is the pattern; the projection neurons have already removed the rest.""" + brain = make_brain({"seed": 0}) + weak = active(present(brain, odour(11), 1.0)["kenyon"]) + strong = active(present(brain, odour(11), 20.0)["kenyon"]) + overlap = len(weak & strong) / max(1, len(weak)) + assert overlap > 0.8, overlap + + +def test_two_odours_barely_share_cells(): + brain = make_brain({"seed": 0}) + a = active(present(brain, odour(12))["kenyon"]) + b = active(present(brain, odour(13))["kenyon"]) + assert len(a & b) / max(1, len(a)) < 0.4 + + +def test_two_brains_wire_their_kenyon_cells_differently(): + """The claw map is drawn per animal. Two flies do not share a code.""" + one = active(present(make_brain({"seed": 1}), odour(14))["kenyon"]) + two = active(present(make_brain({"seed": 2}), odour(14))["kenyon"]) + assert one != two + + +def test_punishment_only_ever_removes_drive(): + """Depression, not potentiation: no MBON rate rises because of learning.""" + brain = make_brain({"seed": 0}) + before = present(brain, odour(15))["mbon"] + learn(brain, present(brain, odour(15)), -1.0) + after = present(brain, odour(15))["mbon"] + assert all(a <= b + 1e-9 for a, b in zip(after, before)) + + +def test_punishment_depresses_approach_and_leaves_avoid_alone(): + brain = make_brain({"seed": 0}) + before = present(brain, odour(16))["mbon"] + for _ in range(6): + learn(brain, present(brain, odour(16)), -1.0) + after = present(brain, odour(16))["mbon"] + assert after[0] < before[0] - 0.1 + assert abs(after[1] - before[1]) < 1e-9 + + +def test_reward_depresses_avoid_and_leaves_approach_alone(): + brain = make_brain({"seed": 0}) + before = present(brain, odour(17))["mbon"] + for _ in range(6): + learn(brain, present(brain, odour(17)), 1.0) + after = present(brain, odour(17))["mbon"] + assert after[1] < before[1] - 0.1 + assert abs(after[0] - before[0]) < 1e-9 + + +def test_learning_is_specific_to_the_odour_that_was_present(): + """A coincidence rule can only reach cells that fired, so another odour barely moves. + + Barely, not never: sparse codes drawn at random do share the occasional cell, and + that shared cell is generalisation rather than a defect. Measured on the reference, + eight pairings take the trained odour's approach drive to zero and leave the other + at 0.95 -- the one cell in twenty the two codes have in common. + """ + brain = make_brain({"seed": 0}) + trained_before = present(brain, odour(18))["mbon"][0] + other_before = present(brain, odour(19))["mbon"][0] + for _ in range(8): + learn(brain, present(brain, odour(18)), -1.0) + trained_after = present(brain, odour(18))["mbon"][0] + other_after = present(brain, odour(19))["mbon"][0] + assert trained_after < 0.1 * trained_before + assert other_after > 0.8 * other_before + + +def test_nothing_is_learned_without_a_signal(): + brain = make_brain({"seed": 0}) + before = present(brain, odour(20))["mbon"] + for _ in range(5): + learn(brain, present(brain, odour(20)), 0.0) + assert present(brain, odour(20))["mbon"] == before + + +def test_mbon_rates_never_go_negative(): + """A synapse can be removed. It cannot be run backwards.""" + brain = make_brain({"seed": 0}) + for _ in range(200): + learn(brain, present(brain, odour(21)), -1.0) + assert all(m >= 0.0 for m in present(brain, odour(21))["mbon"]) +''', + 'tests/test_server.py': r'''"""The backend, as a function. No socket is opened anywhere in this file. + +`handle(method, path, body)` is the whole API surface. The frozen `fly.py` wraps it in +`http.server`, and a shell around a pure function is the part not worth testing. +""" + +import json + +from src import handle + + +def get(path): + status, content_type, body = handle("GET", path) + return status, content_type, body + + +def post(path, payload=None): + status, content_type, body = handle("POST", path, + json.dumps(payload) if payload is not None else None) + return status, content_type, body + + +def state(): + status, _, body = post("/api/reset", {"task": "taxis", "seed": 0, "brain": "new"}) + assert status == 200 + return json.loads(body) + + +def test_the_root_serves_a_page(): + status, content_type, body = get("/") + assert status == 200 + assert "text/html" in content_type + assert body.lstrip().lower().startswith(" before["step"] + assert after["position"] != before["position"] + + +def test_resetting_puts_the_animal_back(): + state() + moved = json.loads(post("/api/step", {"steps": 30})[2]) + assert moved["step"] > 0 + fresh = json.loads(post("/api/reset", {"task": "taxis", "seed": 0})[2]) + assert fresh["step"] == 0 + + +def test_a_reset_can_keep_the_brain_or_replace_it(): + """Two different things a person watching wants, and one flag between them.""" + post("/api/reset", {"task": "avoid", "seed": 0, "brain": "new"}) + post("/api/train", {"task": "avoid", "episodes": 4}) + post("/api/step", {"steps": 1}) + learned = json.loads(get("/api/state")[2])["brain"]["learned"] + assert learned < 0.0 + post("/api/reset", {"task": "avoid", "seed": 0}) + post("/api/step", {"steps": 1}) + assert json.loads(get("/api/state")[2])["brain"]["learned"] < 0.0 + post("/api/reset", {"task": "avoid", "seed": 0, "brain": "new"}) + post("/api/step", {"steps": 1}) + assert abs(json.loads(get("/api/state")[2])["brain"]["learned"]) < 1e-9 + + +def test_training_through_the_api_fills_in_the_history(): + post("/api/reset", {"task": "avoid", "seed": 0, "brain": "new"}) + data = json.loads(post("/api/train", {"task": "avoid", "episodes": 5})[2]) + assert len(data["history"]) >= 5 + assert all("shocks" in row for row in data["history"]) + + +def test_switching_task_switches_the_arena(): + one = json.loads(post("/api/reset", {"task": "taxis", "seed": 0})[2]) + two = json.loads(post("/api/reset", {"task": "choice", "seed": 0})[2]) + assert one["task"] == "taxis" and two["task"] == "choice" + assert len(two["sources"]) == 2 + + +def test_every_response_is_the_type_it_says_it_is(): + for method, path in (("GET", "/"), ("GET", "/api/state"), ("GET", "/api/tasks")): + status, content_type, body = handle(method, path) + assert status == 200 + if "json" in content_type: + json.loads(body) + + +def test_a_trailing_slash_is_the_same_route(): + assert get("/api/tasks")[0] == get("/api/tasks/")[0] == 200 + + +def test_a_query_string_is_ignored(): + assert get("/api/state?t=1")[0] == 200 +''', + 'tests/test_tasks.py': r'''"""The three assays, and the properties each one has to have to measure anything.""" + +import math + +from src import make_world, step_world + +TASKS = ("taxis", "avoid", "choice") +SHOCK_RADIUS = 5.0 + + +def distance(world, name): + for source in world["sources"]: + if source["name"] == name: + return math.hypot(world["position"][0] - source["position"][0], + world["position"][1] - source["position"][1]) + raise AssertionError(f"no source {name}") + + +def test_every_task_builds(): + for task in TASKS: + world = make_world(task, 0) + assert world["task"] == task and world["sources"] + + +def test_an_unknown_task_is_refused(): + try: + make_world("fly-to-the-moon", 0) + except ValueError: + return + raise AssertionError("an unknown task should raise ValueError") + + +def test_taxis_has_one_source_and_nothing_punished(): + world = make_world("taxis", 0) + assert len(world["sources"]) == 1 + assert not any(s.get("punished") for s in world["sources"]) + + +def test_the_animal_does_not_start_on_top_of_the_source(): + for task in TASKS: + for seed in range(6): + world = make_world(task, seed) + assert distance(world, "A") > 2.0 + + +def test_avoid_punishes_its_source(): + world = make_world("avoid", 0) + assert any(s.get("punished") for s in world["sources"]) + + +def test_avoid_leaves_somewhere_safe_to_go(): + """An assay with no safe region measures the arena, not the animal. + + With the punished source in the middle of a walled arena the animal learns the + odour is bad, turns away, hits the wall, is reflected back through the gradient and + is punished again -- 61 shocks an episode from a model whose valence had correctly + reversed by tick 40. + """ + world = make_world("avoid", 0) + punished = [s for s in world["sources"] if s.get("punished")][0] + corners = [(x, y) for x in (-20.0, 20.0) for y in (-20.0, 20.0)] + far = max(math.hypot(c[0] - punished["position"][0], c[1] - punished["position"][1]) + for c in corners) + assert far > 3.0 * SHOCK_RADIUS + + +def test_being_at_the_punished_source_costs(): + world = make_world("avoid", 0) + punished = [s for s in world["sources"] if s.get("punished")][0] + world["position"] = tuple(punished["position"]) + _, reward, _ = step_world(world, (0.0, 0.0)) + assert reward < 0.0 + + +def test_being_far_from_everything_costs_nothing(): + world = make_world("avoid", 0) + world["position"] = (19.0, 19.0) + _, reward, _ = step_world(world, (0.0, 0.0)) + assert reward == 0.0 + + +def test_choice_offers_two_odours_and_punishes_one(): + world = make_world("choice", 0) + assert len(world["sources"]) == 2 + assert sum(bool(s.get("punished")) for s in world["sources"]) == 1 + + +def test_the_two_odours_in_a_choice_smell_different(): + world = make_world("choice", 0) + a, b = world["sources"] + assert any(abs(x - y) > 1e-6 for x, y in zip(a["profile"], b["profile"])) + + +def test_reaching_a_safe_source_ends_the_episode(): + world = make_world("taxis", 0) + world["position"] = world["sources"][0]["position"] + _, reward, done = step_world(world, (0.0, 0.0)) + assert done and reward > 0.0 + + +def test_an_episode_does_not_run_for_ever(): + world = make_world("avoid", 0) + world["position"] = (19.0, 19.0) + done = False + for _ in range(2000): + world, _, done = step_world(world, (0.0, 0.0)) + if done: + break + assert done +''', +} + +#: Held out: the same requirements asked differently, never in the repository. +_AUDIT = { + 'audit/test_behaviour.py': r'''"""The audit set: the same animal asked differently, and never in the repository. + +Every one of these restates something `tests/` already requires. They exist because an +executor is shown the source of the test it is failing and could otherwise write to +that one assertion -- so the headline number is measured on tests no agent has read. +""" + +import math +import random + +from src import learn, make_brain, run_episode, step_brain, train + +GLOMERULI = 24 + + +def odour(seed, n=GLOMERULI): + rng = random.Random(seed) + return [rng.random() ** 2 for _ in range(n)] + + +def present(brain, receptors, scale=10.0): + return step_brain(brain, {"odor": [r * scale for r in receptors]}) + + +def test_the_code_stays_sparse_over_many_odours(): + brain = make_brain({"seed": 7}) + fractions = [sum(1 for c in present(brain, odour(s))["kenyon"] if c) + / len(present(brain, odour(s))["kenyon"]) for s in range(40, 70)] + assert max(fractions) <= 0.15 + assert sum(fractions) / len(fractions) >= 0.02 + + +def test_odours_stay_separable_across_many_pairs(): + brain = make_brain({"seed": 7}) + codes = [{i for i, c in enumerate(present(brain, odour(s))["kenyon"]) if c} + for s in range(100, 118)] + overlaps = [len(a & b) / max(1, len(a)) + for i, a in enumerate(codes) for b in codes[i + 1:]] + assert sum(overlaps) / len(overlaps) < 0.2 + + +def test_the_compass_is_accurate_over_a_long_and_uneven_walk(): + brain = make_brain({"seed": 7}) + silent = [0.0] * GLOMERULI + start = step_brain(brain, {"odor": silent})["heading"] + total, dt = 0.0, 0.05 + for i in range(600): + rate = math.sin(i / 37.0) * 2.0 + step_brain(brain, {"odor": silent, "angular_velocity": rate}, dt) + total += rate * dt + end = step_brain(brain, {"odor": silent})["heading"] + error = math.atan2(math.sin(end - start - total), math.cos(end - start - total)) + assert abs(error) < 0.5, error + + +def test_punishment_is_not_forgotten_while_other_odours_come_and_go(): + brain = make_brain({"seed": 7}) + for _ in range(8): + learn(brain, present(brain, odour(200)), -1.0) + for _ in range(50): + present(brain, odour(201)) + assert present(brain, odour(200))["valence"] < 0.0 + + +def test_the_two_compartments_do_not_share_a_synapse(): + brain = make_brain({"seed": 7}) + for _ in range(8): + learn(brain, present(brain, odour(202)), -1.0) + punished = present(brain, odour(202))["mbon"] + for _ in range(8): + learn(brain, present(brain, odour(202)), 1.0) + both = present(brain, odour(202))["mbon"] + assert both[0] == punished[0] + assert both[1] < punished[1] + + +def test_learning_generalises_no_further_than_the_code_overlaps(): + brain = make_brain({"seed": 7}) + before = [present(brain, odour(s))["valence"] for s in range(300, 310)] + for _ in range(10): + learn(brain, present(brain, odour(299)), -1.0) + after = [present(brain, odour(s))["valence"] for s in range(300, 310)] + assert sum(1 for a, b in zip(after, before) if a < 0.5 * b) <= 2 + + +def test_a_trained_animal_takes_fewer_shocks_in_the_choice_assay(): + trained = train("choice", episodes=10, seed=0)["brain"] + naive = make_brain({"seed": 0}) + unseen = range(2000, 2005) + before = sum(run_episode(naive, "choice", s, learning=False)["shocks"] for s in unseen) + after = sum(run_episode(trained, "choice", s, learning=False)["shocks"] for s in unseen) + assert after < before + + +def test_an_animal_that_learnt_nothing_still_finds_food(): + """Nothing in the plasticity may be load-bearing for innate approach.""" + brain = make_brain({"seed": 7}) + reached = [run_episode(brain, "taxis", s, learning=False)["reached"] for s in range(8)] + assert sum(reached) >= 6 + + +def test_shocks_fall_enough_to_call_it_learning(): + rows = train("avoid", episodes=14, seed=3)["episodes"] + shocks = [r["shocks"] for r in rows] + assert shocks[0] > 0 + assert sum(shocks[-4:]) < sum(shocks[:4]) +''', +} + + +def initial_files() -> Dict[str, str]: + """Everything the run starts from. No `CONTEXT.md`: `--mode b` designs the tree.""" + files = {"spec/CONTEXT.md": _SPEC, "fly.py": _DRIVER, + f"src/{SKILLS_DIR}/python-modules.md": PYTHON_MODULE_SKILL} + files.update(_TESTS) + return files + + +FLY = TestSuite(name="fly", given=initial_files(), frozen=FROZEN, audit=_AUDIT) + +build_tasks = FLY.build_tasks +make_runner = FLY.make_runner +suite_review = FLY.review +#: The whole suite in one interpreter -- which is the only way the shadowing +#: collision in `test_integration.py` shows up at all. +suite_failures = FLY.suite_failures +#: One test, pass or fail. No tolerance of its own -- the test owns that. +reward = reward_test +#: Exactly the audit set in the held-out tail, every driven test in the search. +HELD_OUT_FRAC = FLY.held_out_frac() + + +def llm_manager(complete): + return _llm_manager(complete) + + +def llm_executor(complete, *, editable=("**",), frozen=FROZEN): + return _llm_executor(complete, editable=editable, frozen=frozen, + failure=TEST_FAILURE) diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index deb9c65c..e4db8aef 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -89,6 +89,7 @@ from ._delegation import RecursiveDelegation from . import _domain as minilang +from . import _fly as fly from . import _jqx as jqx from . import _md as md from . import _stackvm as stackvm @@ -111,8 +112,12 @@ #: package nobody can invoke -- and md takes the oracle out of the scoring path #: altogether, because a frozen test suite is what a human actually writes and #: what upstream validates against. All four are stand-ins for upstream's -#: 123.4-hour compiler run and say so. -DOMAINS = {"minilang": minilang, "stackvm": stackvm, "jqx": jqx, "md": md} +#: 123.4-hour compiler run and say so. `fly` is the fifth and the first that grows a +#: *product* rather than a library -- a simulated animal, an HTTP backend and the page a +#: person watches it learn on -- and the first shipped with no reference implementation +#: at all, which is a cost `_fly` states rather than hides. +DOMAINS = {"minilang": minilang, "stackvm": stackvm, "jqx": jqx, "md": md, + "fly": fly} #: "No wall-clock budget", spelled as a number the runtime can add to `time.time()`. #: `float("inf")` cannot be: the shutdown deadline is `t0 + max_seconds` and joining a @@ -132,6 +137,8 @@ "staged suite"), "md": ("Lennard-Jones molecular dynamics with a frozen driver (6 nodes, " "14 files), test suite -- invariants, not values"), + "fly": ("a Drosophila brain, three assays and a web page to watch it learn on, " + "test suite -- a product, not a library"), } @@ -433,6 +440,13 @@ def main(argv=None) -> None: # is designed before anything is written against it, and Phase 2 is handed # "the architecture, directory structure, CONTEXT.md routing tables ... already in # place (created by an Architect agent)". + if getattr(spec, "REQUIRES_MODEL", False) and complete is None: + print(f"--domain {args.domain} needs --model: it ships no reference " + "implementation, so there is no offline rule-based actor to fall back " + "on -- and one written for it would be the design the run is meant to " + "invent") + return + if args.mode == "a": if complete is None: print("--mode a needs --model: extracting a Context Tree is reading code, " From fe696d21028fe0f38061229b26e45be3654dc330 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Sep 2026 09:49:11 +0000 Subject: [PATCH 02/42] Genesis: blind mode -- the agents write the tests, and cannot read the suite Upstream the agents write the tests. They have file and shell tools, they write `test/*.exs`, and "tests are the definition of done" (agents/manager.ex); the external suites Genesis validates against -- c-testsuite, LLVM, Csmith -- are the experimenter's measurement, applied afterwards. This port had it backwards. A human wrote every assertion, and then `TEST_FAILURE` pasted its **source** into the prompt, which is the strongest hint there is: an agent handed the assertion is not implementing a specification, it is writing to an assertion. `TestSuite.hidden` is the other way round. The acceptance suite never enters the repository -- it is merged into the scratch copy one evaluation sees, the way the audit set already was -- and the prompt becomes the requirement it was written from plus the assertion's name. The name carries real information and that is deliberate: `test_the_code_is_sparse_at_every_concentration` is a requirement written as a sentence. The threshold, the tolerance and the inputs are exactly what it does not carry. So every test *in* the repository is an agent's own, and two new things can be asked of a child. `own_test_failures` runs them in one interpreter -- `mix test`, theirs -- scoped to one node's subtree. `own_review` refuses a child whose own tests fail, and a child that wrote an implementation file and no test for it: upstream every node is accountable for its own subtree, so a node with untested code is a node whose parent has nothing to run. Also `ConcurrencyGauge`, because the obvious arithmetic is wrong. `usage.seconds / wallclock` looks like it says how many calls were in flight and does not -- `seconds` spans phases that run before `evolve()` does, while a stage profile's `wallclock` covers only the stage, and the md run's ratio came out at 8.2 with four workers. It counts them directly instead, and the run reports the peak. In `examples/`, not in the engine: what the engine's concurrency *is* is its own business; what an example observed is the example's. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/_common.py | 46 ++++++- examples/genesis/_suite.py | 125 ++++++++++++++++++- examples/genesis/genesis_recursive_worlds.py | 14 ++- tests/test_genesis_example.py | 124 ++++++++++++++++++ 4 files changed, 302 insertions(+), 7 deletions(-) diff --git a/examples/_common.py b/examples/_common.py index 000fb768..08f04932 100644 --- a/examples/_common.py +++ b/examples/_common.py @@ -19,7 +19,8 @@ import argparse import sys -from typing import Optional +import threading +from typing import Callable, Optional from agentdescent.agents import Usage, claude, openai_compatible @@ -395,6 +396,42 @@ def worker_count(args: argparse.Namespace, requested: int) -> int: return 1 +class ConcurrencyGauge: + """How many model calls were actually in flight at once, and at most. + + `Usage` records calls, tokens and seconds, and none of those answer "how hard did + this run lean on the endpoint". The arithmetic that looks like it should -- + `usage.seconds / wallclock` -- does not: `seconds` covers the whole process + including phases that run before `evolve()` does, while the `wallclock` a stage + profile reports covers only the stage, so the ratio comes out above the worker + count and means nothing. This counts the thing directly. + + `agentdescent/` is not the place for it: the engine's concurrency is its own + business and an example asking what it *observed* is the example's business. + """ + + __slots__ = ("peak", "_live", "_lock") + + def __init__(self) -> None: + self.peak = 0 + self._live = 0 + self._lock = threading.Lock() + + def wrap(self, complete: Callable[[str], str]) -> Callable[[str], str]: + def gauged(prompt: str) -> str: + with self._lock: + self._live += 1 + if self._live > self.peak: + self.peak = self._live + try: + return complete(prompt) + finally: + with self._lock: + self._live -= 1 + + return gauged + + def completion_for(args: argparse.Namespace, *, usage: Optional[Usage] = None, **kwargs): """Build the ``Completion`` that ``--provider`` and ``--model`` select. @@ -418,8 +455,10 @@ def completion_for(args: argparse.Namespace, *, usage: Optional[Usage] = None, leave reasoning *on* while the run reports it off -- which is the failure that matters, since the wall-clock would then be attributed to the scheduler. """ + gauge = getattr(args, "_concurrency", None) if is_openai_compatible(args): - return openai_compatible(model=args.model, usage=usage, **kwargs) + built = openai_compatible(model=args.model, usage=usage, **kwargs) + return gauge.wrap(built) if gauge is not None else built if getattr(args, "no_thinking", False): kwargs.setdefault("thinking", {"type": "disabled"}) # --timeout reaches this path too: claude()'s 120s default assumes Claude @@ -428,4 +467,5 @@ def completion_for(args: argparse.Namespace, *, usage: Optional[Usage] = None, # 120s x 3 retries before any evolution happened. if getattr(args, "timeout", None): kwargs.setdefault("timeout", float(args.timeout)) - return claude(model=args.model, usage=usage, **kwargs) + built = claude(model=args.model, usage=usage, **kwargs) + return gauge.wrap(built) if gauge is not None else built diff --git a/examples/genesis/_suite.py b/examples/genesis/_suite.py index a7840ffe..9e165840 100644 --- a/examples/genesis/_suite.py +++ b/examples/genesis/_suite.py @@ -357,6 +357,31 @@ def review(parent, returned): way round -- and the tests that already pass have to keep passing.""" +#: What an executor is shown when the suite is **blind**: the requirement, the name of +#: the assertion that is failing, and what it reported -- and not one line of its source. +#: +#: Upstream the agents write the tests. They have file and shell tools, they write +#: `test/*.exs`, and "tests are the definition of done" (`agents/manager.ex`); the +#: external suites Genesis validates against -- c-testsuite, LLVM, Csmith -- are the +#: experimenter's measurement, applied afterwards. This port had it backwards: a human +#: wrote every test and then pasted its **source** into the prompt, which is the +#: strongest hint there is. An agent handed the assertion is not implementing a +#: specification, it is writing to an assertion. +#: +#: So in blind mode the acceptance suite never enters the repository, the prompt is the +#: requirement it came from, and the tests *in* the repository are the agents' own. +BLIND_FAILURE = """An acceptance assertion is failing. You cannot read it -- the \ +acceptance suite is not in this repository and never will be. What you get is the \ +requirement it was written from, its name, and what it reported: + +{prompt} + +and it reports: {output} + +Write your own test that reproduces this, put it beside the code it covers, and make \ +both pass. A test you write is the only test you can read.""" + + def llm_manager(complete) -> Callable[[Brief], Sequence[Delegation]]: """Ask a model where to situate children. A bad reply means "handle it here".""" @@ -765,6 +790,18 @@ class TestSuite: baselines: Mapping[str, Mapping[str, str]] = field(default_factory=dict) #: Seconds one test may take. timeout: float = 60.0 + #: **Blind mode.** Acceptance tests that drive the search but are not in the + #: repository: the agents are told the requirement and the assertion's verdict, + #: never its source, and write their own tests. See :data:`BLIND_FAILURE`. + hidden: Mapping[str, str] = field(default_factory=dict) + #: ``test file stem -> the requirement that file was written from``. What a blind + #: prompt is built out of, in place of the source. + requirements: Mapping[str, str] = field(default_factory=dict) + + @property + def blind(self) -> bool: + """Is the acceptance suite outside the repository entirely?""" + return bool(self.hidden) def initial_files(self) -> Dict[str, str]: return dict(self.given) @@ -828,6 +865,7 @@ def suite_failures(self, state: Mapping[str, str], *, for path, content in self.given.items(): if match_any(path, self.frozen): state[path] = content + state.update(self.hidden) state.update(self.audit) plan = [(t.meta["file"], t.meta["func"]) for t in self.build_tasks() if not t.meta.get("suite") and (audit or not t.meta.get("audit"))] @@ -849,6 +887,8 @@ def discover(self) -> List[Tuple[str, str, str]]: Parsed rather than imported, so discovery works against a repository with no implementation in it -- which is every repository this port starts from. """ + if self.blind: + return _discover_tests(self.hidden) return _discover_tests({path: body for path, body in self.given.items() if match_any(path, self.tests)}) @@ -898,9 +938,91 @@ def build_tasks(self, limit: Optional[int] = None) -> List[Task]: def _task(self, entry: Tuple[str, str, str], *, audit: bool = False) -> Task: path, func, source = entry stem = path.rsplit("/", 1)[-1].removeprefix("test_").removesuffix(".py") + if self.blind: + source = self._requirement(stem, func) return Task(id=f"{'audit:' if audit else ''}{stem}::{func}", prompt=source, meta={"file": path, "func": func, "kind": stem, "audit": audit}) + def _requirement(self, stem: str, func: str) -> str: + """A blind prompt: the requirement the assertion came from, and its name. + + The name carries real information and that is deliberate -- an assertion called + `test_the_code_is_sparse_at_every_concentration` is a requirement written as a + sentence. What it does not carry is the threshold, the tolerance, or the inputs, + which is exactly the part an agent would otherwise write to instead of to the + specification. + """ + sentence = func.removeprefix("test_").replace("_", " ") + heading = self.requirements.get(stem, "") + return (f"{heading}\n\n" if heading else "") + f"ACCEPTANCE: {sentence}." + + def own_test_failures(self, state, under: str = "") -> List[str]: + """The tests **the agents wrote**, run in one interpreter. `mix test`, theirs. + + Everything in `state` is the agents' own work by construction: the acceptance + suite lives in `hidden` and `audit` and is merged into a scratch copy only while + one evaluation runs, never into the artifact. So any `test_*` function here was + written by an agent, and this is the suite upstream's manager actually runs -- + "tests are the definition of done". + + `under` restricts it to one node's subtree, which is what a parent reviewing one + child's work wants. + """ + owned = {path: body for path, body in state.items() + if path.endswith(".py") + and path.rsplit("/", 1)[-1].startswith("test_") + and (not under or path == under or path.startswith(under + "/"))} + if not owned: + return [] + try: + plan = [(path, func) for path, func, _ in _discover_tests(owned)] + except SyntaxError as exc: # a half-written test file + return [f"{exc.filename}: {type(exc).__name__}: {exc.msg}"] + if not plan: + return [] + return run_suite(dict(state), plan, timeout=max(60.0, self.timeout * 4)) + + def own_review(self, *, require_tests: bool = True): + """A :data:`~examples.genesis._delegation.Review`: run the child's own tests. + + Two refusals, and both are things a frozen acceptance suite cannot say. + + A child whose own tests **fail** has not finished, whatever the acceptance suite + thinks -- it may be failing on code the acceptance suite does not reach yet. And + a child that wrote an implementation file and **no test for it** has not + finished either: upstream every node is accountable for its own subtree and + tests are the definition of done, so a node with untested code is a node whose + parent has nothing to run. + """ + + def review(parent, returned): + candidate = dict(parent.state) + for edit in returned: + if edit.content is None: + candidate.pop(edit.path, None) + else: + candidate[edit.path] = edit.content + path = parent.world.path + failures = self.own_test_failures(candidate, path) + if failures: + return ("rework", "your own tests fail: " + "; ".join(failures[:3])) + if require_tests: + wrote = [e.path for e in returned + if e.content is not None and e.path.endswith(".py") + and not e.path.rsplit("/", 1)[-1].startswith("test_") + and "__init__" not in e.path] + tested = any(p.endswith(".py") + and p.rsplit("/", 1)[-1].startswith("test_") + and (not path or p == path or p.startswith(path + "/")) + for p in candidate) + if wrote and not tested: + return ("rework", f"{wrote[0]} has no test anywhere under " + f"{path or './'}; tests are the definition of done") + return None + + return review + + def make_runner(self) -> Callable[[str, Task], str]: """``run(rendered, task)`` -- one test against one candidate repository.""" @@ -913,7 +1035,8 @@ def run(rendered: str, task: Task) -> str: if match_any(path, self.frozen): state[path] = content # Into the scratch copy the subprocess sees, never into the artifact: - # the audit tests exist only for the length of one evaluation. + # the acceptance tests exist only for the length of one evaluation. + state.update(self.hidden) state.update(self.audit) if task.meta.get("suite"): plan = [(t.meta["file"], t.meta["func"]) for t in self.build_tasks() diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index e4db8aef..18a005af 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -84,8 +84,8 @@ from agentdescent.governance import SKILL_BLAST_RADIUS, classify from agentdescent.sampling import DifficultyWeighted from agentdescent.staleness import get_policy -from examples._common import (add_standard_args, budget_kwargs, completion_for, - confirm, report_engine, worker_count) +from examples._common import (ConcurrencyGauge, add_standard_args, budget_kwargs, + completion_for, confirm, report_engine, worker_count) from ._delegation import RecursiveDelegation from . import _domain as minilang @@ -427,6 +427,12 @@ def main(argv=None) -> None: " -- the tail of the case list")) usage = Usage() + #: How many model calls were in flight at once. The engine's barrier-free + #: concurrency *is* `n_workers` and its merge side runs at `max_concurrency=1`, + #: so this should come back equal to `--workers`; it is reported because + #: `usage.seconds / wallclock` looks like it answers the same question and does + #: not -- `seconds` spans phases that run before `evolve()` does. + args._concurrency = ConcurrencyGauge() complete = None if args.model: if not confirm(args): @@ -640,7 +646,9 @@ def _superseded(rendered, task, output, reward_): f"partial={judge.partial}") if log.pending_rework: print(f"open rework : {sorted(log.pending_rework)}") - print(f"model usage : {usage.summary()}") + print(f"model usage : {usage.summary()}" + + (f", peak {args._concurrency.peak} concurrent" + if args._concurrency.peak else "")) report_engine(result) if args.write_repo: diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index db4bfb24..0a5cd7aa 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -40,6 +40,7 @@ from examples.genesis._spatial import SpatialContract, parse_situated_edits from examples.genesis._worktree import (Rollout, WorktreeLedger, git_worktrees_available) +from examples.genesis._suite import TestSuite as SuiteOfTests from examples.genesis._suite import cold_start, preflight from examples.genesis._world import (CONTEXT_FILE, KNOWN_ISSUES, ROUTING_HEADING, SKILLS_DIR, TRUNCATED, @@ -1434,6 +1435,129 @@ def test_every_domain_briefs_its_agents_with_an_objective_not_a_catalogue_line() assert f"src/{word}" not in md.OBJECTIVE +# --------------------------------------------------------------------------- +# Blind mode: the agents write the tests, and cannot read the acceptance suite +# --------------------------------------------------------------------------- + +_ACCEPTANCE = {"acceptance/test_sparse.py": ( + "TOLERANCE = 0.15\n\n\n" + "def test_the_code_is_sparse_at_every_concentration():\n" + " assert 0.04 < 0.05 < TOLERANCE\n")} + + +def _blind_suite(**kw): + return SuiteOfTests(name="blind", given={"spec/CONTEXT.md": "# spec\n"}, + frozen=("spec/**",), hidden=_ACCEPTANCE, + requirements={"sparse": "SPEC 1.3 -- the mushroom body code."}, + **kw) + + +def test_a_blind_suite_never_puts_its_tests_in_the_repository(): + """Upstream the agents write the tests; the external suite is the experimenter's. + + This port had it backwards -- a human wrote every assertion and then pasted its + **source** into the prompt, which is the strongest hint there is. An agent handed + the assertion is not implementing a specification, it is writing to an assertion. + """ + suite = _blind_suite() + assert suite.blind + assert not any(path.startswith("acceptance/") for path in suite.initial_files()) + assert not any("test_" in path for path in suite.initial_files()) + + +def test_a_blind_prompt_carries_the_requirement_and_not_the_assertion(): + suite = _blind_suite() + task = [t for t in suite.build_tasks() if not t.meta.get("suite")][0] + assert "SPEC 1.3" in task.prompt + # The name is a requirement written as a sentence, and that much is deliberate. + assert "the code is sparse at every concentration" in task.prompt + # The threshold, the tolerance and the inputs are not. + assert "TOLERANCE" not in task.prompt and "0.15" not in task.prompt + assert "assert" not in task.prompt + + +def test_a_sighted_suite_still_shows_its_source(): + """The default is unchanged: `md` and the others still hand over the test.""" + task = [t for t in md.build_tasks() if not t.meta.get("suite")][0] + assert "def test_" in task.prompt and "assert" in task.prompt + + +def test_the_parent_runs_the_tests_the_child_wrote(): + """`mix test`, theirs. A child whose own tests fail has not finished. + + Everything in the artifact is the agents' own work by construction -- the + acceptance suite is merged into a scratch copy only while one evaluation runs -- + so every `test_*` found here was written by an agent. + """ + suite = _blind_suite() + good = {"src/a/kc.py": "X = 1\n", "src/a/test_kc.py": "def test_x():\n assert 1\n"} + bad = {"src/a/kc.py": "X = 1\n", + "src/a/test_kc.py": "def test_x():\n assert 0, 'nope'\n"} + assert suite.own_test_failures(good) == [] + assert "nope" in suite.own_test_failures(bad)[0] + # Scoped to one node's subtree, which is what a parent reviewing one child wants. + assert suite.own_test_failures(bad, "src/b") == [] + assert suite.own_test_failures(bad, "src/a") != [] + + +def test_a_node_that_wrote_code_and_no_test_is_sent_back(): + """Upstream every node is accountable for its own subtree, and tests are done. + + A node with untested code is a node whose parent has nothing to run. + """ + suite = _blind_suite() + review = suite.own_review() + + class _Parent: + def __init__(self, state, path): + self.state, self.world = state, LocalWorld(version=0, path=path) + + untested = [Edit("src/a", "src/a/kc.py", "X = 1\n")] + verdict = review(_Parent({CONTEXT_FILE: "# root\n"}, "src/a"), untested) + assert verdict is not None and verdict[0] == "rework" + assert "no test" in verdict[1] + + tested = untested + [Edit("src/a", "src/a/test_kc.py", "def test_x():\n assert 1\n")] + assert review(_Parent({CONTEXT_FILE: "# root\n"}, "src/a"), tested) is None + + failing = [Edit("src/a", "src/a/test_kc.py", "def test_x():\n assert 0\n")] + verdict = review(_Parent({CONTEXT_FILE: "# root\n"}, "src/a"), failing) + assert verdict is not None and verdict[0] == "rework" + assert "your own tests fail" in verdict[1] + + +def test_the_run_reports_how_many_model_calls_were_in_flight_at_once(): + """`usage.seconds / wallclock` looks like it answers this and does not. + + `seconds` spans the whole process, including the phases that run before + `evolve()` does; the `wallclock` a stage profile reports covers only the stage. The + md run's ratio came out at 8.2 with four workers, which is not a measurement of + anything. This counts the thing directly. + """ + import threading + + from examples._common import ConcurrencyGauge + + gauge = ConcurrencyGauge() + assert gauge.peak == 0 + inside = threading.Barrier(3, timeout=10) + + def slow(prompt): + inside.wait() + return prompt + + wrapped = gauge.wrap(slow) + threads = [threading.Thread(target=wrapped, args=("x",)) for _ in range(3)] + for t in threads: + t.start() + for t in threads: + t.join() + assert gauge.peak == 3 + # And it comes back down, so a later quiet phase does not inflate the peak. + gauge.wrap(lambda p: p)("x") + assert gauge.peak == 3 + + def test_a_node_may_not_shadow_a_sibling_module(): """`rdf/` next to `rdf.py` are one name to Python, and the module wins. From ffc1d5743b2684df98899c6982480b8f9bd76ca3 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Sep 2026 10:03:18 +0000 Subject: [PATCH 03/42] Genesis: the fly domain gives a requirement and a driver, and grows the rest The domain now hands over two things and both are requirements rather than design. `REQUIREMENTS.md` is what the user asked for, in their words -- a connectome-grounded Drosophila brain, three assays, learning that visibly changes behaviour, a frontend and a backend, and a test beside every directory's code -- plus a reading list. No equations, no coefficients, no module layout. `fly.py` is that requirement made executable, and it imports exactly four names from `src`, so the public surface is four functions and everything else is the agents' internal business. Everything else grows: the CONTEXT.md tree, the circuit, the layout, the frontend, and every test in the repository. Both assertion sets stay outside it. `hidden` is black-box acceptance and drives the search -- the agent is shown the requirement heading, the assertion's name as a sentence, and what it reported, never a line of source. `audit` is sealed: written before the run, never in the repository, the same requirements asked with different seeds and different tasks, and run once at the end. The gap between "acceptance passes" and "the sealed suite passes" is the measurement of whether it built the thing or wrote to the acceptance. `suite_failures` now asks what upstream's manager asks -- do *your own* tests pass -- because the human suite is not something this domain's agents can see. `own_review` joins the review chain, so a child that wrote an implementation file and no test for it is sent back. One requirement neither set can check, stated rather than hidden: whether the brain is really built from the connectome. Black-box cannot ask. Only the parent reading the diff, and a person reading the result, can. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/genesis/_fly.py | 1579 +++++------------- examples/genesis/genesis_recursive_worlds.py | 26 +- tests/test_genesis_example.py | 10 +- 3 files changed, 447 insertions(+), 1168 deletions(-) diff --git a/examples/genesis/_fly.py b/examples/genesis/_fly.py index 6acb6254..cbe54810 100644 --- a/examples/genesis/_fly.py +++ b/examples/genesis/_fly.py @@ -1,1332 +1,591 @@ -"""The fifth formation domain: a fruit fly, its brain, and a web page to watch it on. - -The four domains before this one grow a *library*. This one grows a **product**: a -simulated animal, the tasks it is set, the dopamine that changes what it does next, an -HTTP backend and the page a person actually looks at -- and it is the first where the -specification alone had to carry the design, because nothing in this port has ever been -asked for a frontend before. - -It is also the first domain shipped **without a reference implementation**. The others -keep one beside them for two jobs that are not scoring: driving the offline rule-based -actor, and letting a test prove the suite is passable at all. Here there is none, and -the cost is stated rather than hidden. There is no `--domain fly` run without a model, -and no `kill_report`: nothing in the repository checks what this suite *rejects*, which -is the check that caught a molecular-dynamics run reporting 1.000 for a force field -that was identically zero. What stands in its place is that every number in the -specification was measured on a throwaway implementation while the suite was being -written -- 91 assertions passing, and three of the specification's least obvious -paragraphs are there because that implementation failed them first: - -* a ring attractor sharpened with any *pointwise* nonlinearity quantises heading to the - nearest wedge, so the lateral term has to be linear; -* an avoidance assay with the punished source in the middle of a walled arena measures - the arena, not the animal -- the model's valence had correctly reversed by tick 40 and - it still took 61 shocks an episode, because the walls kept reflecting it back through - the gradient; -* and `src/learn/` beside a public `learn()` is two things called `src.learn`, where the - submodule import rebinds the name and every caller after it gets `TypeError: 'module' - object is not callable`. - -The last of those is the same collision this port had already been bitten by twice from -the other direction, which is why the specification names it and `test_integration.py` -asserts it. +"""第五个域：一份需求，一个冻结的驱动，其余全部生长 —— 包括测试。 + +前四个域生长一个**库**，并且人写好整套测试再把测试的**源码**贴进提示词。上游不是这样做 +的：那里的 agent 自己写测试，自己跑 `mix test`，"tests are the definition of done" +(`agents/manager.ex`)；Genesis 验证用的 c-testsuite、LLVM、Csmith 是实验者**事后**拿来量 +的外部基准，从不进 agent 的内循环。把断言的源码交给 agent，它就不再是在实现需求，而是在对 +着断言写代码。 + +所以这个域只给两样东西，两样都是**需求**而不是设计： + +* `REQUIREMENTS.md` —— 用户要什么，用户的原话。要一只连接组级真实的果蝇大脑、三种任务、 + 学习要能改变行为、要前后端、每个目录要为自己的文件写测试。外加一份文献清单。里面没有方 + 程、没有系数、没有模块划分。 +* `fly.py` —— 那份需求的可执行形式，冻结。它只 import `src` 的**四个**名字 + (`make_brain`、`train`、`run_episode`、`handle`)，所以公开接口也只有这四个；分几层、谁 + 调用谁、写哪些测试，全是它们的内部事务。 + +其余一切生长：`CONTEXT.md` 树由 `--mode b` 的 architect 设计，电路、模块、前端、以及**仓 +库里的每一个测试**，都是 agent 写的。 + +两套断言都在仓库外面。`hidden` 是黑盒验收，驱动搜索 —— agent 看到的是需求标题、断言的名 +字和它报了什么，看不到一行源码。`audit` 是**封存**的事后测量：同一批要求换了问法、换了种 +子、换了任务，开跑前写死，跑完只用来报一个数。"验收全过"和"封存套件全过"之间的差距，就是 +它把东西做对了、还是只对着验收写的度量。 + +有一条要求这两套都测不了，写在这里不藏着：**大脑是不是真的照连接组建的**。黑盒问不出来。 +能问的只有父节点评审读 diff（`ParentCodeReview`）和人读结果。 """ from __future__ import annotations -from typing import Dict +from typing import Dict, List, Mapping -from ._suite import PYTHON_MODULE_SKILL, TEST_FAILURE, TestSuite, reward_test +from ._suite import BLIND_FAILURE, PYTHON_MODULE_SKILL, TestSuite, reward_test from ._suite import llm_executor as _llm_executor from ._suite import llm_manager as _llm_manager from ._world import SKILLS_DIR __all__ = ["CASE_NOUN", "CONTRACTS", "FLY", "FROZEN", "GROUP_NOUN", "HELD_OUT_FRAC", "OBJECTIVE", "REQUIRES_MODEL", "SCORING", "build_tasks", "initial_files", - "llm_executor", "llm_manager", "make_runner", "reward", "suite_failures", - "suite_review"] + "llm_executor", "llm_manager", "make_runner", "own_review", "reward", + "suite_failures", "suite_review"] -#: The specification, the suite and the driver. Human-supplied, refused to every -#: proposal, and restored pristine before scoring. -FROZEN = ("spec/**", "tests/**", "fly.py") +#: 需求和驱动。人写的全部，拒绝给任何提案，评分前恢复原样。 +FROZEN = ("REQUIREMENTS.md", "fly.py") -#: What is pushed into every brief, in this order. -CONTRACTS = ("spec/**", "fly.py") +#: 推进每一个 brief 的东西，按这个顺序。 +CONTRACTS = ("REQUIREMENTS.md", "fly.py") -SCORING = "frozen test suite, no reference implementation anywhere in the port" -CASE_NOUN = "test functions" -GROUP_NOUN = "test files" +SCORING = "黑盒验收（agent 读不到），外加一套开跑前封存的事后测量" +CASE_NOUN = "验收断言" +GROUP_NOUN = "验收文件" ENTRY = "src/__init__.py" -#: There is no offline actor for this domain, and pretending otherwise would mean -#: shipping the decomposition the run is meant to invent. See the module docstring. +#: 没有参考实现，也就没有离线规则化 actor；写一个出来就等于把该生长的设计先写了。 REQUIRES_MODEL = True OBJECTIVE = ( - "Build the Drosophila model the frozen driver `fly.py` and the frozen test suite " - "both import: a Python package rooted at `src/`, whose `src/__init__.py` exposes " - "exactly `make_brain`, `step_brain`, `sense`, `act`, `learn`, `make_world`, " - "`step_world`, `run_episode`, `train` and `handle`. It simulates a fly's olfactory " - "brain -- antennal lobe, mushroom body, lateral horn, central complex -- walking in " - "an arena, being punished or rewarded, and changing what it does next; and it " - "serves the whole thing as a web page a person can watch it learn on. " - "`spec/CONTEXT.md` pins those signatures, the circuit and the assays; everything " - "below `src/` -- how many modules, what they are called, who owns what -- is yours " - "to decide. Pure Python, no third-party packages." + "实现 `REQUIREMENTS.md` 要的那个软件：一个根在 `src/` 的 Python 包，把一只果蝇的大脑" + "照真实连接组建出来，让它在竞技场里执行任务、领取奖惩、并因此改变行为，再把整个东西" + "作为一个网页端出来。冻结的 `fly.py` 只 import `src` 的四个名字 —— `make_brain`、" + "`train`、`run_episode`、`handle` —— 那四个就是公开接口。`src/` 以下怎么分层、每层叫" + "什么、谁调用谁，全部由你们决定。可以用 numpy 和其它第三方包。" + "**每个目录都要为自己的文件写测试**：测试是“做完了”的定义，一个写了实现却没有测试的" + "节点，它的上级没有任何东西可以运行来验收它。" ) -#: The specification. The only design a person wrote. -_SPEC = r'''# Specification — a Drosophila brain you can watch learn - -Build the library the frozen driver `fly.py` and the frozen suite in `tests/` import: a -pure-Python package rooted at `src/` that simulates a fruit fly — its olfactory brain, -its body in an arena, the tasks it is set, and the dopamine that changes what it does -next — and serves the whole thing as a web page. - -Reduced units throughout: one body length, one second, one arbitrary unit of odour. -Pure Python, no third-party packages, no numpy. Determinism is a requirement, not a -convenience: the same seed must give the same trajectory on any machine. - -## What the animal is - -Four structures, and the division of labour between them is the point of the model. - -**Antennal lobe.** 24 glomeruli. Olfactory receptor neurons respond across orders of -magnitude of concentration and the projection neurons they drive do not: lateral -inhibition divides each glomerulus by the activity of the whole lobe, so the PN -*pattern* says which odour this is and carries almost nothing about how much of it -there is. Use `r^1.5 / (sigma^1.5 + sum_j r_j^1.5)` with `sigma = 12`. - -**Mushroom body.** 400 Kenyon cells, each sampling 6 glomeruli at random, fixed for the -life of the animal. Feedback inhibition (the APL neuron) leaves the most strongly driven -**5%** active — a rank, not a fixed threshold, so the code stays sparse for a faint -odour and a strong one alike. Two output neurons: MBON 0 is *approach*, MBON 1 is -*avoid*, each reading the active cells through plastic weights that start at 1.0. - -**Learning is depression, and this is the part that is usually modelled backwards.** A -Kenyon cell active at the same moment as a dopaminergic neuron has its synapse onto that -DAN's MBON *weakened*, floored at zero — `w -= 0.5 * kc * da`. Punishment therefore does -not build an avoid response; it removes the approach response for that odour and lets -the untouched avoid pathway win. Nothing is learned about an odour that was absent, -because its cells were silent and the rule is a coincidence rule. - -**Lateral horn.** The projection neurons fork. One branch goes to the mushroom body, -where experience can change what an odour means; the other goes to the lateral horn, -which is stereotyped and does not learn. A naive fly walks towards food odour without -ever having met it. Innate valence is `1.0 * sum(projection)` — positive, so every -odour here starts attractive. - -Keeping these two apart is what lets learning show up as a **reversal**: the lateral -horn says approach, the mushroom body after punishment subtracts more than that, the sum -changes sign, and the animal turns around. Fuse them into one number and there is -nothing left to reverse. - -**Central complex.** A ring of 16 wedges holding one bump of activity whose position is -the heading. There is no compass: the bump is *shifted* by the animal's own turning, so -heading is dead reckoning and it runs in the dark. Two properties, and both are tested. -Turning by an angle and back must return the bump where it started. And the bump must -neither smear flat nor collapse onto a single wedge — hold it with a **linear** filter, -`(1+2a)` on the wedge against `a` on each neighbour at `a = 0.1`. Any *pointwise* -nonlinearity — squaring, thresholding — snaps the bump toward the nearest wedge and -quantises every heading; measured over four full turns, no lateral term drifts 0.45 rad, -a pointwise power loses the heading entirely, and the linear filter drifts 0.04. - -## What the animal has for a body - -A square arena, half-width 20, with **reflecting** walls: a fly in a dish has edges, and -an animal that learns to leave the arena has learned nothing. Odour sources each carry a -receptor profile and a concentration falling off as `exp(-distance / 6)`. - -**Two antennae**, 0.6 apart, and the animal steers on the difference between them: -`turn = wanted * 12 * (left - right) / (left + right)`, where `wanted` is `+1` when -valence is positive and `-1` when it is negative. That sign is the entire behavioural -consequence of learning. - -Bilateral steering has a blind spot exactly where it matters most — pointed straight at -a source both antennae read the same thing — so the temporal gradient covers the head-on -case: when the odour is changing the way the animal does *not* want, it turns, and which -way is one bit of memory it carries between ticks. The relative change is about 2.5% a -tick, so that term needs a gain around 200 to be worth anything. - -Walking speed is `4 * (1 + 0.6 * tanh(valence))`, turning is capped at 3 rad/s, the tick -is 0.05 s and an episode is at most 400 ticks. - -## Three assays - -| task | arena | paid | -|---|---|---| -| `taxis` | one safe source at the centre | `+1` on arrival, which ends the episode | -| `avoid` | one punished source, off centre | `-1` every tick within 5 of it | -| `choice` | a punished source and a safe one, opposite ends | both of the above | - -`avoid` and `choice` put the punished source **off centre on purpose**. With it in the -middle of a walled arena there is nowhere to go: the animal learns the odour is bad, -turns away, hits the wall, is reflected back through the gradient and is punished again -— 61 shocks an episode from a model whose valence had correctly reversed by tick 40. An -avoidance assay has to contain somewhere safe or it measures the arena. - -Reward is paid **per tick**, not per episode, because the plasticity rule is a -coincidence rule: it can only act on what the animal was smelling when the dopamine -arrived. - -## `src/__init__.py` exposes exactly these ten names - -```python -make_brain(config=None) -> dict - # config keys: seed, glomeruli, kenyon_cells, wedges, heading. All optional. - -step_brain(brain, sensory, dt=0.05) -> activity # mutates brain -sense(world) -> sensory -act(activity) -> (forward, turn) -learn(brain, activity, reward) -> brain # mutates brain - -make_world(task="taxis", seed=0, glomeruli=24) -> world -step_world(world, action, dt=0.05) -> (world, reward, done) - -run_episode(brain, task="taxis", seed=0, dt=0.05, learning=True) -> result -train(task="taxis", episodes=20, seed=0) -> {"task", "episodes": [...], "brain"} - -handle(method, path, body=None) -> (status, content_type, body) -``` - -Each MUST import its layer **lazily**, inside the function body, so a missing layer does -not stop the layers before it from passing their tests. - -**No module below `src/` may be named after one of these ten.** A package laid out with -`src/learn/` beside a public `learn()` has two things called `src.learn`; the moment -anything inside imports the submodule, the name on the package is rebound to the module -and every caller gets `TypeError: 'module' object is not callable`. Tests that call the -function first still pass. The one that calls `train()` first does not. - -### `sensory` -`odor` (one float per glomerulus), `odor_left`, `odor_right`, `angular_velocity`. - -### `activity` -`projection` (24 floats), `kenyon` (one 0.0/1.0 per cell), `mbon` (2 floats, approach -then avoid), `ring` (one float per wedge, summing to 1), `heading` (the compass, radians -in `(-pi, pi]`), `innate`, `learned` (= `mbon[0] - mbon[1]`), `valence` (= `innate + -learned`), `delta_drive`, `tumble_sign`, `odor_left`, `odor_right`. - -### `world` -`task`, `seed`, `glomeruli`, `t`, `step`, `position` (x, y), `heading`, -`angular_velocity`, `sources` (each `name`, `position`, `profile`, `strength`, -`punished`), `shocked`. - -### `result` from an episode -`reward`, `shocks`, `steps`, `nearest`, `distance`, `reached`. - -## The web page - -`handle` is a **pure function** — it opens no socket. `fly.py` wraps it in -`http.server` and does nothing else, so the part worth testing is testable without one. - -| route | does | -|---|---| -| `GET /` | the page: one HTML file, no build step, no CDN, no network | -| `GET /api/tasks` | `{"tasks": [...]}` | -| `GET /api/state` | the arena, the brain, and the episode history | -| `POST /api/reset` | `{task, seed, brain}` — `brain: "new"` forgets what was learned | -| `POST /api/step` | `{steps, learning}` — advance the live animal | -| `POST /api/train` | `{task, episodes}` — run episodes, append to the history | - -Anything else is `404` with a JSON body; a body that is not JSON is `400`. - -The page must draw, from `/api/state` and nothing else: the **arena** with the animal, -the sources and the punished region; the **Kenyon-cell code** as a raster with the -active cells marked; the **ellipsoid body** as a ring with the bump and the compass -heading; the **valence** panel breaking the number into innate, approach, avoid and net; -and the **learning curve**, shocks per episode. Controls for task, play/pause, step, -train, and a fresh brain. It must work at phone width and follow the viewer's colour -scheme. - -## How this is scored - -A frozen suite in `tests/`, and no reference implementation anywhere in the loop. -One task per test function; the prompt is that test's own source. - -| file | asserts | -|---|---| -| `test_antennal_lobe.py` | bounds, compression in the range the arena uses, order preserved under rescaling | -| `test_mushroom_body.py` | sparsity at every concentration, the same odour lights the same cells, separation, depression only, compartment specificity | -| `test_central_complex.py` | one bump, neither flat nor collapsed, reversible, accurate over four turns, runs in the dark | -| `test_body.py` | two antennae, the sign of steering, walls, containment, determinism | -| `test_tasks.py` | each assay's shape, that `avoid` has a safe region, what is paid and when | -| `test_learning.py` | valence reverses, other odours keep theirs, shocks fall, and it transfers to arenas it never saw | -| `test_server.py` | every route, every JSON shape, the page, 404 and 400 | -| `test_integration.py` | the loop closes by hand, and no module shadows an entry point | - -A further set of assertions is **held out**: the same requirements asked differently, in -files that are not in the repository and that no agent can read, injected only at -scoring time. That is the honest number, because an executor is shown the source of the -test it is failing and could otherwise write to that one assertion. -''' - -#: The driver: a program you can run, rather than a package nobody can invoke. -_DRIVER = r'''#!/usr/bin/env python3 -"""fly -- a Drosophila brain you can watch learn. - -Human-supplied, frozen, and never written by an agent: the model under ``src/`` is what -the run grows, and this is the shell around it, so the result is a program you can run -rather than a package nobody can invoke. +#: 每个验收文件是从需求的哪一条来的 —— blind 提示词由它加上断言名拼成。 +REQUIREMENTS = { + "training": "需求 二、四：三种任务都要能训练；`python fly.py train` 要能跑，" + "每一局报出它挨了多少惩罚、拿了多少回报、走了多少步。同一个种子给同一条轨迹。", + "learning": "需求 二：学习必须改变行为，而且要能被看见。同一个气味，训练前后动物的" + "反应应当不同；关掉可塑性时动物不得被改变。", + "page": "需求 三：一个网页，打开就能看 —— 动物在竞技场里走、大脑各层此刻的活动、" + "学习曲线。", + "api": "需求 三：后端要能取当前状态、单步推进、重置、跑训练；非法路由和非法请求体" + "要被拒绝。", + "sealed": "需求 二、三：同样的要求，换一种问法。", +} - python fly.py train --task avoid --episodes 20 - python fly.py probe --task avoid --episodes 10 --trials 5 - python fly.py serve --port 8000 +#: 黑盒验收，驱动搜索。**不在仓库里** —— 只在一次评分的临时副本里出现。 +_ACCEPTANCE = { + 'acceptance/test_api.py': r'''"""用户验收：只通过驱动用到的四个名字，黑盒地问"我要的东西能用吗"。 -It touches the library only through the entry points ``spec/CONTEXT.md`` pins, so it -runs against any implementation that passes ``tests/``, whatever the internal layout -turns out to be. Pure Python and no third-party packages: this has to run anywhere. +这些断言从不进入仓库。agent 看到的是它们的名字和报错，看不到一行源码 —— +仓库里的每一个测试都是 agent 自己写的。 """ -import argparse import json -import sys - - -def cmd_train(args): - from src import train - result = train(args.task, args.episodes, args.seed) - rows = result["episodes"] - print(f"{'ep':>4} {'shocks':>7} {'reward':>8} {'steps':>6} nearest") - for row in rows: - print(f"{row['episode']:>4} {row['shocks']:>7} {row['reward']:>8.1f} " - f"{row['steps']:>6} {row['nearest']}") - head = sum(r["shocks"] for r in rows[:3]) / max(1, len(rows[:3])) - tail = sum(r["shocks"] for r in rows[-3:]) / max(1, len(rows[-3:])) - print(f"\nfirst 3 episodes: {head:.1f} shocks last 3: {tail:.1f}") - return 0 - - -def cmd_probe(args): - """Train one animal, then test it and a naive one with plasticity switched off. - - The honest measurement of learning: same arenas, seeds neither animal was trained - on, and nothing changing during the test. - """ - from src import make_brain, run_episode, train - trained = train(args.task, args.episodes, args.seed)["brain"] - naive = make_brain({"seed": args.seed}) - out = {} - for label, brain in (("naive", naive), ("trained", trained)): - shocks = [run_episode(brain, args.task, 1000 + i, learning=False)["shocks"] - for i in range(args.trials)] - out[label] = sum(shocks) - print(f"{label:>8}: {sum(shocks):>5} shocks over {args.trials} unseen arenas {shocks}") - print(f"\nlearned reduction: {out['naive'] - out['trained']} shocks") - return 0 - - -def cmd_serve(args): - from http.server import BaseHTTPRequestHandler, HTTPServer - - from src import handle - - class Handler(BaseHTTPRequestHandler): - def _respond(self, method): - length = int(self.headers.get("content-length") or 0) - body = self.rfile.read(length).decode("utf-8") if length else None - status, content_type, payload = handle(method, self.path, body) - data = payload.encode("utf-8") - self.send_response(status) - self.send_header("content-type", content_type) - self.send_header("content-length", str(len(data))) - self.end_headers() - self.wfile.write(data) - - def do_GET(self): - self._respond("GET") - - def do_POST(self): - self._respond("POST") - - def log_message(self, *a): - pass - - server = HTTPServer((args.host, args.port), Handler) - print(f"http://{args.host}:{args.port}/") - sys.stdout.flush() - server.serve_forever() - return 0 - - -def main(argv=None): - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - sub = parser.add_subparsers(dest="cmd", required=True) - for name, fn in (("train", cmd_train), ("probe", cmd_probe), ("serve", cmd_serve)): - p = sub.add_parser(name) - p.set_defaults(fn=fn) - if name in ("train", "probe"): - p.add_argument("--task", default="avoid") - p.add_argument("--episodes", type=int, default=20) - p.add_argument("--seed", type=int, default=0) - if name == "probe": - p.add_argument("--trials", type=int, default=5) - if name == "serve": - p.add_argument("--host", default="127.0.0.1") - p.add_argument("--port", type=int, default=8000) - args = parser.parse_args(argv) - return args.fn(args) - - -if __name__ == "__main__": - raise SystemExit(main()) -''' -#: The suite. One task per test function; the prompt is the test's own source. -_TESTS = { - 'tests/test_antennal_lobe.py': r'''"""The antennal lobe: what survives a change of concentration, and what does not.""" +from src import handle, make_brain, run_episode, train -from src import make_brain, step_brain - -GLOMERULI = 24 - - -def odour(seed, n=GLOMERULI): - """A reproducible receptor pattern. Squared uniforms: a few glomeruli dominate.""" - import random - rng = random.Random(seed) - return [rng.random() ** 2 for _ in range(n)] - - -def present(brain, receptors, scale=1.0): - return step_brain(brain, {"odor": [r * scale for r in receptors]}) - - -def test_projection_is_one_rate_per_glomerulus(): - brain = make_brain({"seed": 0}) - activity = present(brain, odour(1)) - assert len(activity["projection"]) == GLOMERULI - - -def test_projection_rates_are_bounded_below_one(): - """Divisive normalisation: a glomerulus is a share of the lobe, never more than it.""" - brain = make_brain({"seed": 0}) - for scale in (0.01, 1.0, 100.0, 10000.0): - activity = present(brain, odour(2), scale) - assert all(0.0 <= p < 1.0 for p in activity["projection"]) - assert sum(activity["projection"]) < 1.0 - - -def test_a_silent_antenna_gives_a_silent_lobe(): - brain = make_brain({"seed": 0}) - activity = step_brain(brain, {"odor": [0.0] * GLOMERULI}) - assert all(p == 0.0 for p in activity["projection"]) - - -def test_the_lobe_compresses_concentration_where_the_arena_puts_it(): - """Ten times the odour is nowhere near ten times the response -- in range. - - The range matters and the test states it. Far below saturation the lobe is if - anything expansive; what the arena actually delivers at a source is an order of - magnitude above that, and there ten times the odour buys a few per cent. - """ - brain = make_brain({"seed": 0}) - weak = sum(present(brain, odour(3), 10.0)["projection"]) - strong = sum(present(brain, odour(3), 100.0)["projection"]) - assert weak < strong < 1.5 * weak - - -def test_more_odour_is_never_less_response(): - brain = make_brain({"seed": 0}) - totals = [sum(present(brain, odour(7), s)["projection"]) - for s in (0.1, 1.0, 10.0, 100.0, 1000.0)] - assert totals == sorted(totals) - - -def test_the_strongest_glomerulus_stays_the_strongest(): - """Normalisation rescales the pattern; it must not reorder it.""" - brain = make_brain({"seed": 0}) - receptors = odour(4) - loudest = max(range(GLOMERULI), key=lambda g: receptors[g]) - for scale in (0.5, 5.0, 50.0): - projection = present(brain, receptors, scale)["projection"] - assert max(range(GLOMERULI), key=lambda g: projection[g]) == loudest - - -def test_two_odours_give_two_different_patterns(): - brain = make_brain({"seed": 0}) - a = present(brain, odour(5))["projection"] - b = present(brain, odour(6))["projection"] - assert any(abs(x - y) > 1e-6 for x, y in zip(a, b)) -''', - 'tests/test_body.py': r'''"""The body: two antennae, a pair of legs, and walls that do not let go.""" - -import math - -from src import act, make_world, sense, step_world - -SIZE = 20.0 - - -def test_sensing_gives_one_reading_per_glomerulus_and_one_per_antenna(): - world = make_world("taxis", 0) - sensory = sense(world) - assert len(sensory["odor"]) == world["glomeruli"] - assert sensory["odor_left"] >= 0.0 and sensory["odor_right"] >= 0.0 - - -def test_the_antennae_read_more_odour_nearer_the_source(): - near = make_world("taxis", 0) - near["position"] = (2.0, 0.0) - far = make_world("taxis", 0) - far["position"] = (18.0, 0.0) - assert sum(sense(near)["odor"]) > 10.0 * sum(sense(far)["odor"]) - - -def test_the_antenna_facing_the_source_reads_more(): - """Bilateral comparison is the only thing that tells the animal which way to turn.""" - world = make_world("taxis", 0) - world["position"] = (10.0, 0.0) - # Facing +y from x=+10 with the source at the origin puts it to the animal's left. - world["heading"] = math.pi / 2.0 - assert sense(world)["odor_left"] > sense(world)["odor_right"] - # Turn it around and the same source is on the other side. - world["heading"] = -math.pi / 2.0 - assert sense(world)["odor_right"] > sense(world)["odor_left"] - - -def test_an_attractive_odour_turns_the_animal_toward_the_stronger_antenna(): - left = act({"mbon": [1.0, 0.0], "valence": 1.0, "odor_left": 2.0, - "odor_right": 1.0, "delta_drive": 0.0})[1] - right = act({"mbon": [1.0, 0.0], "valence": 1.0, "odor_left": 1.0, - "odor_right": 2.0, "delta_drive": 0.0})[1] - assert left > 0.0 > right - - -def test_an_aversive_odour_turns_the_animal_the_other_way(): - """The same gradient, the opposite sign -- and that is all learning changes.""" - attracted = act({"mbon": [1.0, 0.0], "valence": 1.0, "odor_left": 2.0, - "odor_right": 1.0, "delta_drive": 0.0})[1] - repelled = act({"mbon": [0.0, 1.0], "valence": -1.0, "odor_left": 2.0, - "odor_right": 1.0, "delta_drive": 0.0})[1] - assert attracted > 0.0 > repelled - - -def test_the_animal_walks_faster_toward_something_it_likes(): - liked = act({"mbon": [1.0, 0.0], "valence": 1.0, "odor_left": 1.0, - "odor_right": 1.0, "delta_drive": 0.0})[0] - disliked = act({"mbon": [0.0, 1.0], "valence": -1.0, "odor_left": 1.0, - "odor_right": 1.0, "delta_drive": 0.0})[0] - assert liked > disliked > 0.0 +TASKS = ("taxis", "avoid", "choice") -def test_the_animal_stays_in_the_arena(): - world = make_world("taxis", 0) - for _ in range(600): - world, _, _ = step_world(world, (8.0, 0.3)) - assert abs(world["position"][0]) <= SIZE + 1e-6 - assert abs(world["position"][1]) <= SIZE + 1e-6 +def rows(task, episodes=4, seed=0): + return train(task, episodes, seed)["episodes"] -def test_walking_into_a_wall_turns_the_animal_around(): - world = make_world("taxis", 0) - world["position"] = (SIZE - 0.1, 0.0) - world["heading"] = 0.0 - world, _, _ = step_world(world, (8.0, 0.0)) - assert math.cos(world["heading"]) < 0.0 +def test_the_api_reports_where_the_animal_is_and_what_its_brain_is_doing(): + status, content_type, body = handle("GET", "/api/state") + assert status == 200 and "json" in content_type + state = json.loads(body) + assert "position" in state and len(state["position"]) == 2 + assert isinstance(state.get("brain"), dict) and len(state["brain"]) >= 4 -def test_the_world_is_a_function_of_its_seed(): - one = make_world("taxis", 3) - two = make_world("taxis", 3) - assert one["position"] == two["position"] - assert make_world("taxis", 4)["position"] != one["position"] +def test_the_api_can_step_reset_and_train(): + handle("POST", "/api/reset", json.dumps({"task": "avoid", "seed": 0})) + before = json.loads(handle("GET", "/api/state")[2])["step"] + handle("POST", "/api/step", json.dumps({"steps": 10})) + after = json.loads(handle("GET", "/api/state")[2])["step"] + assert after > before + status, _, _ = handle("POST", "/api/train", json.dumps({"task": "avoid", + "episodes": 2})) + assert status == 200 + assert json.loads(handle("POST", "/api/reset", + json.dumps({"task": "avoid", "seed": 0}))[2])["step"] == 0 -def test_a_step_reports_reward_and_whether_the_episode_is_over(): - world = make_world("taxis", 0) - world, reward, done = step_world(world, (0.0, 0.0)) - assert isinstance(reward, float) and isinstance(done, bool) +def test_an_unknown_route_is_refused_and_a_bad_body_is_too(): + assert handle("GET", "/api/nothing-here")[0] == 404 + assert handle("POST", "/api/step", "{not json")[0] == 400 ''', - 'tests/test_central_complex.py': r'''"""The ellipsoid body: one bump, and what it costs to move it accurately.""" - -import math + 'acceptance/test_learning.py': r'''"""用户验收：只通过驱动用到的四个名字，黑盒地问"我要的东西能用吗"。 -from src import make_brain, step_brain - -SILENT = {"odor": [0.0] * 24} - - -def turn(brain, rate, seconds, dt=0.05): - """Spin the animal at `rate` rad/s for `seconds` and return the last activity.""" - activity = None - for _ in range(int(round(seconds / dt))): - activity = step_brain(brain, dict(SILENT, angular_velocity=rate), dt) - return activity - - -def wrapped(angle): - return math.atan2(math.sin(angle), math.cos(angle)) - - -def test_the_ring_is_a_distribution(): - brain = make_brain({"seed": 0}) - ring = step_brain(brain, SILENT)["ring"] - assert len(ring) >= 8 - assert all(a >= 0.0 for a in ring) - assert abs(sum(ring) - 1.0) < 1e-6 - - -def test_there_is_exactly_one_bump(): - brain = make_brain({"seed": 0}) - ring = turn(brain, 0.8, 1.3)["ring"] - n = len(ring) - peaks = sum(1 for i in range(n) if ring[i] > ring[i - 1] and ring[i] >= ring[(i + 1) % n]) - assert peaks == 1, ring - - -def test_the_bump_neither_smears_flat_nor_collapses_to_one_wedge(): - """Both failures look fine on a reversibility test and neither is a compass. - - Flat is what happens with no lateral term at all; one wedge is what a pointwise - nonlinearity gives, and it quantises every heading to a multiple of 22.5 degrees. - """ - brain = make_brain({"seed": 0}) - ring = turn(brain, 1.0, 2.0)["ring"] - uniform = 1.0 / len(ring) - assert max(ring) > 2.0 * uniform - assert max(ring) < 0.6 - - -def test_turning_back_returns_the_bump_where_it_started(): - """The integration is reversible, which is the property a dead-reckoner needs.""" - brain = make_brain({"seed": 0}) - start = step_brain(brain, SILENT)["heading"] - turn(brain, 1.0, 1.0) - back = turn(brain, -1.0, 1.0)["heading"] - assert abs(wrapped(back - start)) < 0.05 - - -def test_the_heading_follows_the_turn(): - brain = make_brain({"seed": 0}) - start = step_brain(brain, SILENT)["heading"] - after = turn(brain, 1.0, 1.0)["heading"] - assert wrapped(after - start) > 0.5 - back = turn(brain, -2.0, 1.0)["heading"] - assert wrapped(back - after) < -0.5 - - -def test_the_compass_survives_several_full_turns(): - """Four turns of dead reckoning, and the error stays a fraction of a wedge.""" - brain = make_brain({"seed": 0}) - start = step_brain(brain, SILENT)["heading"] - total = 4.0 * 2.0 * math.pi - after = turn(brain, 1.0, total)["heading"] - assert abs(wrapped(after - start - total)) < 0.4 - - -def test_standing_still_does_not_move_the_compass(): - brain = make_brain({"seed": 0}) - start = step_brain(brain, SILENT)["heading"] - after = turn(brain, 0.0, 5.0)["heading"] - assert abs(wrapped(after - start)) < 1e-6 - - -def test_the_compass_runs_in_the_dark(): - """Nothing external drives it: with no odour at all the bump still tracks turning.""" - brain = make_brain({"seed": 0}) - after = turn(brain, 1.5, 1.0)["heading"] - assert abs(wrapped(after - 1.5)) < 0.3 -''', - 'tests/test_integration.py': r'''"""The whole animal, and the one way a package can break itself.""" +这些断言从不进入仓库。agent 看到的是它们的名字和报错，看不到一行源码 —— +仓库里的每一个测试都是 agent 自己写的。 +""" import json -import random - -from src import (act, handle, learn, make_brain, make_world, run_episode, sense, - step_brain, step_world, train) - - -def test_every_public_name_is_callable(): - """A module is not a function. This catches the one import mistake that hides. - - A package laid out so that `src/learn/` sits beside a public `learn()` has two - things called `src.learn`, and the moment anything inside imports the submodule the - name on the package is rebound to the module. Every test that calls the function - first still passes; the one that calls `train()` first gets `TypeError: 'module' - object is not callable`. No module below `src/` may be named after an entry point. - """ - for name in (act, handle, learn, make_brain, make_world, run_episode, sense, - step_brain, step_world, train): - assert callable(name), name - - -def test_training_first_does_not_break_the_other_entry_points(): - """The order that catches it: touch the deep layers, then use the shallow names.""" - train("taxis", episodes=1, seed=0) - handle("GET", "/api/state") - brain = make_brain({"seed": 0}) - rng = random.Random(0) - activity = step_brain(brain, {"odor": [rng.random() for _ in range(24)]}) - learn(brain, activity, -1.0) - world = make_world("taxis", 0) - step_world(world, act(activity)) - run_episode(brain, "taxis", 0, learning=False) - - -def test_the_loop_closes_by_hand(): - """Sense, think, act, move, be paid, learn -- in that order, with nothing hidden.""" - brain = make_brain({"seed": 0}) - world = make_world("avoid", 0) - total = 0.0 - for _ in range(120): - activity = step_brain(brain, sense(world)) - world, reward, done = step_world(world, act(activity)) - total += reward - if reward: - learn(brain, activity, reward) - if done: - break - assert world["step"] > 0 - assert total <= 0.0 - - -def test_the_animal_the_server_shows_is_the_animal_that_learns(): - handle("POST", "/api/reset", json.dumps({"task": "avoid", "seed": 0, "brain": "new"})) - handle("POST", "/api/step", json.dumps({"steps": 1})) - before = json.loads(handle("GET", "/api/state")[2])["brain"]["learned"] - handle("POST", "/api/train", json.dumps({"task": "avoid", "episodes": 4})) - handle("POST", "/api/step", json.dumps({"steps": 1})) - after = json.loads(handle("GET", "/api/state")[2])["brain"]["learned"] - assert before == 0.0 > after - - -def test_a_brain_is_plain_data(): - """No classes required, and nothing that cannot be written to a file.""" - brain = make_brain({"seed": 0}) - assert isinstance(brain, dict) - - -def test_the_model_scales_down(): - """The sizes are configuration, not constants baked into the wiring.""" - small = make_brain({"seed": 0, "kenyon_cells": 60, "wedges": 8}) - activity = step_brain(small, {"odor": [1.0] * 24}) - assert len(activity["kenyon"]) == 60 - assert len(activity["ring"]) == 8 -''', - 'tests/test_learning.py': r'''"""What the animal does differently after it has been punished.""" - -import random - -from src import learn, make_brain, run_episode, step_brain, train - -GLOMERULI = 24 - - -def odour(seed, n=GLOMERULI): - rng = random.Random(seed) - return [rng.random() ** 2 for _ in range(n)] +from src import handle, make_brain, run_episode, train -def present(brain, receptors, scale=10.0): - return step_brain(brain, {"odor": [r * scale for r in receptors]}) - - -def test_a_naive_animal_finds_every_odour_attractive(): - """Innate valence comes from the lateral horn, and it is there before experience.""" - brain = make_brain({"seed": 0}) - for seed in range(5): - assert present(brain, odour(seed))["valence"] > 0.0 - - -def test_valence_is_innate_plus_learned_and_nothing_else(): - brain = make_brain({"seed": 0}) - activity = present(brain, odour(30)) - assert abs(activity["valence"] - (activity["innate"] + activity["learned"])) < 1e-9 - - -def test_a_naive_animal_has_learned_nothing(): - brain = make_brain({"seed": 0}) - assert abs(present(brain, odour(31))["learned"]) < 1e-9 +TASKS = ("taxis", "avoid", "choice") -def test_punishment_reverses_an_odour_the_animal_used_to_approach(): - """The headline: the sign of valence flips, and the innate part never moved.""" - brain = make_brain({"seed": 0}) - before = present(brain, odour(32)) - for _ in range(8): - learn(brain, present(brain, odour(32)), -1.0) - after = present(brain, odour(32)) - assert before["valence"] > 0.0 > after["valence"] - assert abs(after["innate"] - before["innate"]) < 1e-9 - assert after["learned"] < -0.5 +def rows(task, episodes=4, seed=0): + return train(task, episodes, seed)["episodes"] -def test_an_odour_that_was_never_punished_keeps_its_valence(): - brain = make_brain({"seed": 0}) - before = present(brain, odour(34))["valence"] - for _ in range(8): - learn(brain, present(brain, odour(33)), -1.0) - assert present(brain, odour(34))["valence"] > 0.8 * before +def test_a_fresh_animal_is_punished_in_the_avoidance_assay(): + """没学过的动物会一头撞进惩罚区 —— 否则这个实验什么也测不到。""" + naive = make_brain({"seed": 0}) + assert sum(run_episode(naive, "avoid", 1000 + i, learning=False)["shocks"] + for i in range(3)) > 0 -def test_training_reduces_the_shocks_an_animal_takes(): - result = train("avoid", episodes=12, seed=0) - shocks = [e["shocks"] for e in result["episodes"]] - first = sum(shocks[:3]) / 3.0 - last = sum(shocks[-3:]) / 3.0 - assert first > 0.0 - assert last < 0.5 * first +def test_training_reduces_the_punishment_the_animal_takes(): + shocks = [r["shocks"] for r in rows("avoid", 12)] + assert sum(shocks[:3]) > 0 + assert sum(shocks[-3:]) < sum(shocks[:3]) def test_what_was_learned_transfers_to_arenas_it_never_saw(): - """Plasticity off, seeds it was not trained on: the only honest test of learning.""" - trained = train("avoid", episodes=10, seed=0)["brain"] + """关掉可塑性、用没训练过的种子 —— 唯一诚实的学习度量。""" + trained = train("avoid", 10, 0)["brain"] naive = make_brain({"seed": 0}) - unseen = range(1000, 1005) + unseen = range(2000, 2005) before = sum(run_episode(naive, "avoid", s, learning=False)["shocks"] for s in unseen) after = sum(run_episode(trained, "avoid", s, learning=False)["shocks"] for s in unseen) - assert before > 0 - assert after < 0.25 * before - - -def test_training_reports_one_row_per_episode(): - result = train("taxis", episodes=4, seed=0) - assert len(result["episodes"]) == 4 - for i, row in enumerate(result["episodes"]): - assert row["episode"] == i - for key in ("shocks", "reward", "steps", "nearest", "distance"): - assert key in row - + assert before > 0 and after < before -def test_training_is_reproducible(): - one = train("avoid", episodes=5, seed=0)["episodes"] - two = train("avoid", episodes=5, seed=0)["episodes"] - assert [e["shocks"] for e in one] == [e["shocks"] for e in two] - -def test_an_episode_with_learning_off_changes_nothing(): - brain = make_brain({"seed": 0}) - run_episode(brain, "avoid", 0, learning=False) - assert abs(present(brain, odour(35))["learned"]) < 1e-9 - - -def test_a_naive_animal_walks_to_an_unpunished_source(): - """Innate approach plus bilateral steering is enough; nothing is learned here.""" +def test_an_episode_with_learning_off_leaves_the_animal_unchanged(): brain = make_brain({"seed": 0}) - reached = sum(run_episode(brain, "taxis", s, learning=False)["reached"] - for s in range(5)) - assert reached >= 4 + for s in range(3): + run_episode(brain, "avoid", s, learning=False) + fresh = make_brain({"seed": 0}) + before = [run_episode(fresh, "avoid", 3000 + i, learning=False)["shocks"] + for i in range(3)] + after = [run_episode(brain, "avoid", 3000 + i, learning=False)["shocks"] + for i in range(3)] + assert before == after ''', - 'tests/test_mushroom_body.py': r'''"""Kenyon cells: a sparse code, the same every time, and what dopamine does to it.""" - -from src import learn, make_brain, step_brain - -GLOMERULI = 24 + 'acceptance/test_page.py': r'''"""用户验收：只通过驱动用到的四个名字，黑盒地问"我要的东西能用吗"。 +这些断言从不进入仓库。agent 看到的是它们的名字和报错，看不到一行源码 —— +仓库里的每一个测试都是 agent 自己写的。 +""" -def odour(seed, n=GLOMERULI): - import random - rng = random.Random(seed) - return [rng.random() ** 2 for _ in range(n)] - - -def present(brain, receptors, scale=10.0): - return step_brain(brain, {"odor": [r * scale for r in receptors]}) - - -def active(code): - return {i for i, c in enumerate(code) if c} - - -def test_the_code_is_sparse(): - """A few per cent of the population, which is what makes odours separable.""" - brain = make_brain({"seed": 0}) - for seed in range(8): - code = present(brain, odour(seed))["kenyon"] - fraction = len(active(code)) / len(code) - assert 0.01 <= fraction <= 0.15, fraction - - -def test_the_code_is_sparse_at_every_concentration(): - """Feedback inhibition sets a rank, not a fixed threshold.""" - brain = make_brain({"seed": 0}) - for scale in (0.1, 1.0, 10.0, 1000.0): - code = present(brain, odour(9), scale)["kenyon"] - assert 0.01 <= len(active(code)) / len(code) <= 0.15 - - -def test_the_same_odour_always_lights_the_same_cells(): - brain = make_brain({"seed": 0}) - first = active(present(brain, odour(10))["kenyon"]) - for _ in range(5): - assert active(present(brain, odour(10))["kenyon"]) == first - - -def test_concentration_does_not_change_which_cells_fire(): - """Identity is the pattern; the projection neurons have already removed the rest.""" - brain = make_brain({"seed": 0}) - weak = active(present(brain, odour(11), 1.0)["kenyon"]) - strong = active(present(brain, odour(11), 20.0)["kenyon"]) - overlap = len(weak & strong) / max(1, len(weak)) - assert overlap > 0.8, overlap - - -def test_two_odours_barely_share_cells(): - brain = make_brain({"seed": 0}) - a = active(present(brain, odour(12))["kenyon"]) - b = active(present(brain, odour(13))["kenyon"]) - assert len(a & b) / max(1, len(a)) < 0.4 - - -def test_two_brains_wire_their_kenyon_cells_differently(): - """The claw map is drawn per animal. Two flies do not share a code.""" - one = active(present(make_brain({"seed": 1}), odour(14))["kenyon"]) - two = active(present(make_brain({"seed": 2}), odour(14))["kenyon"]) - assert one != two - - -def test_punishment_only_ever_removes_drive(): - """Depression, not potentiation: no MBON rate rises because of learning.""" - brain = make_brain({"seed": 0}) - before = present(brain, odour(15))["mbon"] - learn(brain, present(brain, odour(15)), -1.0) - after = present(brain, odour(15))["mbon"] - assert all(a <= b + 1e-9 for a, b in zip(after, before)) - - -def test_punishment_depresses_approach_and_leaves_avoid_alone(): - brain = make_brain({"seed": 0}) - before = present(brain, odour(16))["mbon"] - for _ in range(6): - learn(brain, present(brain, odour(16)), -1.0) - after = present(brain, odour(16))["mbon"] - assert after[0] < before[0] - 0.1 - assert abs(after[1] - before[1]) < 1e-9 - +import json -def test_reward_depresses_avoid_and_leaves_approach_alone(): - brain = make_brain({"seed": 0}) - before = present(brain, odour(17))["mbon"] - for _ in range(6): - learn(brain, present(brain, odour(17)), 1.0) - after = present(brain, odour(17))["mbon"] - assert after[1] < before[1] - 0.1 - assert abs(after[0] - before[0]) < 1e-9 +from src import handle, make_brain, run_episode, train +TASKS = ("taxis", "avoid", "choice") -def test_learning_is_specific_to_the_odour_that_was_present(): - """A coincidence rule can only reach cells that fired, so another odour barely moves. - Barely, not never: sparse codes drawn at random do share the occasional cell, and - that shared cell is generalisation rather than a defect. Measured on the reference, - eight pairings take the trained odour's approach drive to zero and leave the other - at 0.95 -- the one cell in twenty the two codes have in common. - """ - brain = make_brain({"seed": 0}) - trained_before = present(brain, odour(18))["mbon"][0] - other_before = present(brain, odour(19))["mbon"][0] - for _ in range(8): - learn(brain, present(brain, odour(18)), -1.0) - trained_after = present(brain, odour(18))["mbon"][0] - other_after = present(brain, odour(19))["mbon"][0] - assert trained_after < 0.1 * trained_before - assert other_after > 0.8 * other_before +def rows(task, episodes=4, seed=0): + return train(task, episodes, seed)["episodes"] -def test_nothing_is_learned_without_a_signal(): - brain = make_brain({"seed": 0}) - before = present(brain, odour(20))["mbon"] - for _ in range(5): - learn(brain, present(brain, odour(20)), 0.0) - assert present(brain, odour(20))["mbon"] == before +def test_the_page_is_served_at_the_root(): + status, content_type, body = handle("GET", "/") + assert status == 200 and "html" in content_type.lower() + assert body.lstrip().lower().startswith("= 0.0 for m in present(brain, odour(21))["mbon"]) +def test_the_page_shows_the_arena_the_brain_and_the_learning_curve(): + _, _, page = handle("GET", "/") + low = page.lower() + assert "canvas" in low + for word in ("arena", "brain", "learn"): + assert word in low, word ''', - 'tests/test_server.py': r'''"""The backend, as a function. No socket is opened anywhere in this file. + 'acceptance/test_training.py': r'''"""用户验收：只通过驱动用到的四个名字，黑盒地问"我要的东西能用吗"。 -`handle(method, path, body)` is the whole API surface. The frozen `fly.py` wraps it in -`http.server`, and a shell around a pure function is the part not worth testing. +这些断言从不进入仓库。agent 看到的是它们的名字和报错，看不到一行源码 —— +仓库里的每一个测试都是 agent 自己写的。 """ import json -from src import handle - - -def get(path): - status, content_type, body = handle("GET", path) - return status, content_type, body +from src import handle, make_brain, run_episode, train +TASKS = ("taxis", "avoid", "choice") -def post(path, payload=None): - status, content_type, body = handle("POST", path, - json.dumps(payload) if payload is not None else None) - return status, content_type, body +def rows(task, episodes=4, seed=0): + return train(task, episodes, seed)["episodes"] -def state(): - status, _, body = post("/api/reset", {"task": "taxis", "seed": 0, "brain": "new"}) - assert status == 200 - return json.loads(body) +def test_every_task_can_be_trained(): + for task in TASKS: + assert len(rows(task, 2)) == 2, task -def test_the_root_serves_a_page(): - status, content_type, body = get("/") - assert status == 200 - assert "text/html" in content_type - assert body.lstrip().lower().startswith("= 0 + assert isinstance(row["reward"], (int, float)) + assert row["steps"] > 0 -def test_the_page_draws_the_arena_and_the_brain(): - """What the page must show, named. A backend nobody can see is half a product.""" - _, _, page = get("/") - for element in ("canvas", "arena", "kenyon", "ring", "mbon"): - assert element.lower() in page.lower(), element +def test_training_returns_the_animal_it_trained(): + result = train("avoid", 3, 0) + assert result["brain"] is not None + assert run_episode(result["brain"], "avoid", 7, learning=False)["shocks"] >= 0 -def test_the_page_only_calls_routes_that_exist(): - _, _, page = get("/") - for route in ("/api/state", "/api/step", "/api/reset", "/api/train"): - assert route in page +def test_the_same_seed_gives_the_same_run(): + assert [r["shocks"] for r in rows("avoid", 5)] == [r["shocks"] for r in rows("avoid", 5)] +''', +} -def test_unknown_routes_are_refused(): - status, content_type, body = get("/api/nonsense") - assert status == 404 - assert "application/json" in content_type - assert "error" in json.loads(body) +#: 封存的事后测量。开跑前写死，全程不进仓库，跑完只用来报一个数。 +_SEALED = { + 'sealed/test_sealed.py': r'''"""封存的事后测量：写在开跑之前，全程不进仓库，任何 agent 读不到，跑完只用来报一个数。 +这是上游的协议 —— Genesis 拿 c-testsuite、LLVM、Csmith 验证，那些是实验者事后量的外部 +基准，不是 agent 的内循环。它们和 `acceptance/` 问的是同一批要求，但换了问法、换了种子、 +换了任务，所以"验收全过"和"封存套件全过"之间的差距，就是它把东西做对了还是只对着验收写的 +度量。 -def test_a_body_that_is_not_json_is_refused(): - status, _, body = handle("POST", "/api/step", "{not json") - assert status == 400 - assert "error" in json.loads(body) +同样只用驱动那四个名字。有一条要求是这套测不了的：**大脑是不是真的照连接组建的**。黑盒 +问不出来，只能靠父节点评审读 diff、和人读结果。这条限制写在这里，不藏着。 +""" +import json -def test_the_task_list_is_served(): - status, _, body = get("/api/tasks") - assert status == 200 - tasks = json.loads(body)["tasks"] - assert set(tasks) == {"taxis", "avoid", "choice"} +from src import handle, make_brain, run_episode, train -def test_state_reports_the_arena(): - data = state() - for key in ("task", "step", "position", "heading", "sources", "shocked"): - assert key in data, key - assert len(data["position"]) == 2 +def test_learning_shows_up_in_the_choice_assay_too(): + trained = train("choice", 10, 1)["brain"] + naive = make_brain({"seed": 1}) + unseen = range(4000, 4006) + before = sum(run_episode(naive, "choice", s, learning=False)["shocks"] for s in unseen) + after = sum(run_episode(trained, "choice", s, learning=False)["shocks"] for s in unseen) + assert before > 0 and after < before -def test_state_reports_the_brain(): - post("/api/step", {"steps": 1}) - data = json.loads(get("/api/state")[2]) - brain = data["brain"] - for key in ("kenyon", "kenyon_cells", "projection", "ring", "mbon", - "innate", "learned", "valence"): - assert key in brain, key - assert 0 < len(brain["kenyon"]) < brain["kenyon_cells"] +def test_the_gain_from_training_survives_a_different_seed(): + trained = train("avoid", 10, 5)["brain"] + naive = make_brain({"seed": 5}) + unseen = range(5000, 5006) + before = sum(run_episode(naive, "avoid", s, learning=False)["shocks"] for s in unseen) + after = sum(run_episode(trained, "avoid", s, learning=False)["shocks"] for s in unseen) + assert after < 0.6 * before -def test_stepping_advances_the_animal(): - before = state() - after = json.loads(post("/api/step", {"steps": 20})[2]) - assert after["step"] > before["step"] - assert after["position"] != before["position"] +def test_more_training_is_not_worse_than_less(): + short = train("avoid", 4, 2)["brain"] + long = train("avoid", 16, 2)["brain"] + unseen = range(6000, 6006) + a = sum(run_episode(short, "avoid", s, learning=False)["shocks"] for s in unseen) + b = sum(run_episode(long, "avoid", s, learning=False)["shocks"] for s in unseen) + assert b <= a -def test_resetting_puts_the_animal_back(): - state() - moved = json.loads(post("/api/step", {"steps": 30})[2]) - assert moved["step"] > 0 - fresh = json.loads(post("/api/reset", {"task": "taxis", "seed": 0})[2]) - assert fresh["step"] == 0 +def test_an_animal_trained_on_one_task_still_works_on_another(): + """不应该为了通过一个实验把另一个弄坏。""" + trained = train("avoid", 10, 3)["brain"] + assert run_episode(trained, "taxis", 7000, learning=False)["steps"] > 0 -def test_a_reset_can_keep_the_brain_or_replace_it(): - """Two different things a person watching wants, and one flag between them.""" - post("/api/reset", {"task": "avoid", "seed": 0, "brain": "new"}) - post("/api/train", {"task": "avoid", "episodes": 4}) - post("/api/step", {"steps": 1}) - learned = json.loads(get("/api/state")[2])["brain"]["learned"] - assert learned < 0.0 - post("/api/reset", {"task": "avoid", "seed": 0}) - post("/api/step", {"steps": 1}) - assert json.loads(get("/api/state")[2])["brain"]["learned"] < 0.0 - post("/api/reset", {"task": "avoid", "seed": 0, "brain": "new"}) - post("/api/step", {"steps": 1}) - assert abs(json.loads(get("/api/state")[2])["brain"]["learned"]) < 1e-9 +def test_two_animals_with_different_seeds_are_not_the_same_animal(): + one = [run_episode(make_brain({"seed": 11}), "avoid", 8000 + i, learning=False)["shocks"] + for i in range(4)] + two = [run_episode(make_brain({"seed": 22}), "avoid", 8000 + i, learning=False)["shocks"] + for i in range(4)] + assert one != two -def test_training_through_the_api_fills_in_the_history(): - post("/api/reset", {"task": "avoid", "seed": 0, "brain": "new"}) - data = json.loads(post("/api/train", {"task": "avoid", "episodes": 5})[2]) - assert len(data["history"]) >= 5 - assert all("shocks" in row for row in data["history"]) +def test_the_taxis_assay_ends_sooner_than_it_times_out(): + """趋向实验要真的能到达 —— 每局都跑满上限说明动物根本没找到源。""" + steps = [r["steps"] for r in train("taxis", 6, 0)["episodes"]] + assert min(steps) < max(steps) or min(steps) < 400 -def test_switching_task_switches_the_arena(): - one = json.loads(post("/api/reset", {"task": "taxis", "seed": 0})[2]) - two = json.loads(post("/api/reset", {"task": "choice", "seed": 0})[2]) - assert one["task"] == "taxis" and two["task"] == "choice" - assert len(two["sources"]) == 2 +def test_the_state_the_page_draws_changes_as_the_animal_moves(): + handle("POST", "/api/reset", json.dumps({"task": "taxis", "seed": 0})) + first = json.loads(handle("GET", "/api/state")[2]) + handle("POST", "/api/step", json.dumps({"steps": 25})) + later = json.loads(handle("GET", "/api/state")[2]) + assert first["position"] != later["position"] -def test_every_response_is_the_type_it_says_it_is(): - for method, path in (("GET", "/"), ("GET", "/api/state"), ("GET", "/api/tasks")): - status, content_type, body = handle(method, path) - assert status == 200 - if "json" in content_type: - json.loads(body) +def test_the_page_exposes_more_than_one_layer_of_the_brain(): + """需求里写了"要能看见大脑各层此刻的活动"，所以状态里不能只有一个标量。""" + handle("POST", "/api/reset", json.dumps({"task": "avoid", "seed": 0})) + handle("POST", "/api/step", json.dumps({"steps": 5})) + brain = json.loads(handle("GET", "/api/state")[2])["brain"] + populations = [k for k, v in brain.items() if isinstance(v, (list, dict)) and v] + assert len(populations) >= 3, sorted(brain) -def test_a_trailing_slash_is_the_same_route(): - assert get("/api/tasks")[0] == get("/api/tasks/")[0] == 200 +def test_a_new_brain_through_the_api_has_forgotten_everything(): + handle("POST", "/api/reset", json.dumps({"task": "avoid", "seed": 0, "brain": "new"})) + handle("POST", "/api/train", json.dumps({"task": "avoid", "episodes": 6})) + handle("POST", "/api/reset", json.dumps({"task": "avoid", "seed": 0, "brain": "new"})) + handle("POST", "/api/step", json.dumps({"steps": 1})) + fresh = json.loads(handle("GET", "/api/state")[2]) + assert fresh["step"] >= 1 -def test_a_query_string_is_ignored(): - assert get("/api/state?t=1")[0] == 200 +def test_the_driver_is_reproducible_across_two_calls_in_one_process(): + one = [r["reward"] for r in train("choice", 5, 9)["episodes"]] + two = [r["reward"] for r in train("choice", 5, 9)["episodes"]] + assert one == two ''', - 'tests/test_tasks.py': r'''"""The three assays, and the properties each one has to have to measure anything.""" - -import math - -from src import make_world, step_world - -TASKS = ("taxis", "avoid", "choice") -SHOCK_RADIUS = 5.0 +} +#: 用户的要求，原话。 +_REQUIREMENTS = r'''# 需求：一个果蝇大脑的模拟软件 -def distance(world, name): - for source in world["sources"]: - if source["name"] == name: - return math.hypot(world["position"][0] - source["position"][0], - world["position"][1] - source["position"][1]) - raise AssertionError(f"no source {name}") +我要一个软件，把一只果蝇（*Drosophila melanogaster*）建模出来：它的大脑、它的身体、 +它要做的事，以及它怎么从做事的结果里学到东西。我要能在浏览器里看着它学。 +这份文件是**需求**，不是设计。怎么分层、怎么划模块、写哪些测试、页面怎么排，都由你们决定。 -def test_every_task_builds(): - for task in TASKS: - world = make_world(task, 0) - assert world["task"] == task and world["sources"] +## 一、大脑要复杂，而且要真实 +这是最重要的一条。我不要一个"能跑通任务的神经网络"，我要一个**照着真实果蝇大脑建的**模型： -def test_an_unknown_task_is_refused(): - try: - make_world("fly-to-the-moon", 0) - except ValueError: - return - raise AssertionError("an unknown task should raise ValueError") +- 神经元的分群、数量、连接极性（兴奋还是抑制）、脑区划分，要能对得上已发表的连接组和电路 + 生理工作，不能是拍脑袋编的。 +- 代码里要出现**真实的细胞类型名**，并注明它在真实果蝇里有多少个；模型里如果缩小了规模， + 要写清楚真实数字是多少、为什么缩小。 +- 涉及嗅觉学习和导航的主要脑区都要有，不能只做一条通路。 +- 参考下面那份文献清单，以最新的为准。哪一条设计是从哪篇来的，写在对应的 `CONTEXT.md` 里。 +**做不到的地方要说出来**，写进 `## Known Issues`，不要用一个看起来像那么回事的近似糊过去 +然后当成真的。 -def test_taxis_has_one_source_and_nothing_punished(): - world = make_world("taxis", 0) - assert len(world["sources"]) == 1 - assert not any(s.get("punished") for s in world["sources"]) +## 二、它要能执行任务、拿到反馈、并因此改变行为 +至少三种任务，覆盖三种不同的情形： -def test_the_animal_does_not_start_on_top_of_the_source(): - for task in TASKS: - for seed in range(6): - world = make_world(task, seed) - assert distance(world, "A") > 2.0 +1. 趋向一个气味源； +2. 学会**回避**一个会带来惩罚的气味源； +3. 在两个气味源之间做选择，其中一个有惩罚。 +关键要求：**学习必须改变行为，而且要能被看见**。同一个气味，训练前后动物的反应应当不同； +没有被训练过的气味不应该受影响。 -def test_avoid_punishes_its_source(): - world = make_world("avoid", 0) - assert any(s.get("punished") for s in world["sources"]) +奖惩按时间步给，不是按整局给。 +## 三、前端和后端 -def test_avoid_leaves_somewhere_safe_to_go(): - """An assay with no safe region measures the arena, not the animal. +**后端**：一套 HTTP 接口，能取当前状态、单步推进、重置、跑训练。 - With the punished source in the middle of a walled arena the animal learns the - odour is bad, turns away, hits the wall, is reflected back through the gradient and - is punished again -- 61 shocks an episode from a model whose valence had correctly - reversed by tick 40. - """ - world = make_world("avoid", 0) - punished = [s for s in world["sources"] if s.get("punished")][0] - corners = [(x, y) for x in (-20.0, 20.0) for y in (-20.0, 20.0)] - far = max(math.hypot(c[0] - punished["position"][0], c[1] - punished["position"][1]) - for c in corners) - assert far > 3.0 * SHOCK_RADIUS +**前端**：一个网页，打开就能看。要能看见的东西：动物在竞技场里走；大脑各层此刻的活动； +学习曲线。控件至少要能选任务、播放/暂停、单步、训练、换一只没学过的新大脑。 +手机宽度下要能用，跟随系统的明暗配色。不要依赖 CDN，不要在页面里访问外网。 -def test_being_at_the_punished_source_costs(): - world = make_world("avoid", 0) - punished = [s for s in world["sources"] if s.get("punished")][0] - world["position"] = tuple(punished["position"]) - _, reward, _ = step_world(world, (0.0, 0.0)) - assert reward < 0.0 +## 四、怎么算做完 +`fly.py` 是冻结的，你们不能改它。它就是这份需求的可执行形式 —— 我要的是一个**能跑的程序**， +不是一个没人能调用的包： -def test_being_far_from_everything_costs_nothing(): - world = make_world("avoid", 0) - world["position"] = (19.0, 19.0) - _, reward, _ = step_world(world, (0.0, 0.0)) - assert reward == 0.0 +``` +python fly.py train --task avoid --episodes 20 # 训练，打出每集的电击数 +python fly.py probe --task avoid --trials 5 # 关掉可塑性，朴素 vs 训练过 +python fly.py serve --port 8000 # 起服务，浏览器打开 +``` +它只通过 `src` 这一个包调用你们写的东西。`fly.py` 里 import 了哪些名字，那些就是公开接口， +其余全是你们的内部事务。 + +## 五、工程要求 + +- Python 3。**可以用 numpy 和其它第三方包**；用了什么，写在 `requirements.txt` 里。 +- **确定性**：同一个种子，任何机器上给出同一条轨迹。 +- **每个目录都要为自己的文件写测试。** 测试是"做完了"的定义 —— 一个写了实现却没有测试的 + 节点，它的上级没有任何东西可以运行来验收它。测试放在它覆盖的代码旁边。 +- 不要留半个文件。不要留没人 import 的死模块。 + +## 参考文献 + +连接组与全脑模型 +- Dorkenwald 等，*Neuronal wiring diagram of an adult brain*，Nature 2024（FlyWire） +- Schlegel 等，*Whole-brain annotation and multi-connectome cell typing of Drosophila*，Nature 2024 +- Shiu 等，*A leaky integrate-and-fire computational model based on the connectome of the entire adult Drosophila brain*，Nature 2024 +- Lappalainen 等，*Connectome-constrained networks predict neural activity across the fly visual system*，Nature 2024 + +嗅觉、蘑菇体与学习 +- Olsen、Bhandawat & Wilson，*Divisive normalization in olfactory population codes*，Neuron 2010 +- Caron 等，*Random convergence of olfactory inputs in the Drosophila mushroom body*，Nature 2013 +- Zheng 等，*Structured sampling of olfactory input by the fly mushroom body*，Current Biology 2022 +- Aso 等，*The neuronal architecture of the mushroom body provides a logic for associative learning*，eLife 2014 +- Hige 等，*Heterosynaptic plasticity underlies aversive olfactory learning in Drosophila*，Neuron 2015 +- Handler 等，*Distinct dopamine receptor pathways underlie the temporal sensitivity of associative learning*，Cell 2019 +- Felsenberg 等，*Integration of parallel opposing memories underlies memory extinction*，Cell 2018 +- Dolan 等，*Neurogenetic dissection of the Drosophila lateral horn*，eLife 2019 + +中央复合体与导航 +- Seelig & Jayaraman，*Neural dynamics for landmark orientation and angular path integration*，Nature 2015 +- Green 等，*A neural circuit architecture for angular integration in Drosophila*，Nature 2017 +- Turner-Evans 等，*The neuroanatomical ultrastructure and function of a biological ring attractor*，Neuron 2020 +- Hulse 等，*A connectome of the Drosophila central complex*，eLife 2021 +- Lyu、Abbott & Maimon，*Building an allocentric travelling direction signal via vector computation*，Nature 2022 +- Lu 等，*Transforming representations of movement from body- to world-centric space*，Nature 2022 +- Westeinde 等，*Transforming a head direction signal into a goal-oriented steering command*，Nature 2024 +''' -def test_choice_offers_two_odours_and_punishes_one(): - world = make_world("choice", 0) - assert len(world["sources"]) == 2 - assert sum(bool(s.get("punished")) for s in world["sources"]) == 1 +#: 那份需求的可执行形式。冻结：它是需求，不是实现。 +_DRIVER = r'''#!/usr/bin/env python3 +"""fly -- 一只果蝇，一个大脑，和一个能看着它学习的网页。 +这个文件是**冻结的**：没有 agent 可以写它。它是 `REQUIREMENTS.md` 的可执行形式 —— +我要的是一个能跑的程序，而不是一个没人能调用的包。 -def test_the_two_odours_in_a_choice_smell_different(): - world = make_world("choice", 0) - a, b = world["sources"] - assert any(abs(x - y) > 1e-6 for x, y in zip(a["profile"], b["profile"])) + python fly.py train --task avoid --episodes 20 + python fly.py probe --task avoid --episodes 10 --trials 5 + python fly.py serve --port 8000 +它只通过 `src` 这一个包调用你们写的东西，并且**只用到下面四个名字**。其余一切 —— +分几层、每层叫什么、谁调用谁、写哪些测试 —— 都是你们的内部事务。 -def test_reaching_a_safe_source_ends_the_episode(): - world = make_world("taxis", 0) - world["position"] = world["sources"][0]["position"] - _, reward, done = step_world(world, (0.0, 0.0)) - assert done and reward > 0.0 + make_brain(config=None) -> brain + 一只没有任何经验的动物。`config` 是个字典，至少认得 `seed`。 + train(task, episodes, seed) -> {"episodes": [...], "brain": brain} + 跑 `episodes` 局，每局一行，按顺序放在 "episodes" 里。每行至少要有 + `episode`（从 0 数）、`shocks`（这一局挨了多少次惩罚）、`reward`、`steps`。 + "brain" 是训练完的那只动物。 -def test_an_episode_does_not_run_for_ever(): - world = make_world("avoid", 0) - world["position"] = (19.0, 19.0) - done = False - for _ in range(2000): - world, _, done = step_world(world, (0.0, 0.0)) - if done: - break - assert done -''', -} + run_episode(brain, task, seed, learning=True) -> {"shocks": ..., ...} + 用给定的动物跑一局。`learning=False` 时不得改变它 —— 这是唯一能诚实 + 比较"学过"和"没学过"的办法。 -#: Held out: the same requirements asked differently, never in the repository. -_AUDIT = { - 'audit/test_behaviour.py': r'''"""The audit set: the same animal asked differently, and never in the repository. + handle(method, path, body=None) -> (status, content_type, body) + HTTP 的全部逻辑，一个纯函数，不开 socket。下面那层 http.server 是我写的。 + `GET /` 必须返回那个网页。 -Every one of these restates something `tests/` already requires. They exist because an -executor is shown the source of the test it is failing and could otherwise write to -that one assertion -- so the headline number is measured on tests no agent has read. +任务名至少要有 `taxis`、`avoid`、`choice` 三个。 """ -import math -import random - -from src import learn, make_brain, run_episode, step_brain, train - -GLOMERULI = 24 - - -def odour(seed, n=GLOMERULI): - rng = random.Random(seed) - return [rng.random() ** 2 for _ in range(n)] - - -def present(brain, receptors, scale=10.0): - return step_brain(brain, {"odor": [r * scale for r in receptors]}) - +import argparse +import sys -def test_the_code_stays_sparse_over_many_odours(): - brain = make_brain({"seed": 7}) - fractions = [sum(1 for c in present(brain, odour(s))["kenyon"] if c) - / len(present(brain, odour(s))["kenyon"]) for s in range(40, 70)] - assert max(fractions) <= 0.15 - assert sum(fractions) / len(fractions) >= 0.02 +def cmd_train(args): + from src import train + rows = train(args.task, args.episodes, args.seed)["episodes"] + print(f"{'ep':>4} {'shocks':>7} {'reward':>9} {'steps':>6}") + for row in rows: + print(f"{row['episode']:>4} {row['shocks']:>7} {row['reward']:>9.2f} " + f"{row['steps']:>6}") + head = sum(r["shocks"] for r in rows[:3]) / max(1, len(rows[:3])) + tail = sum(r["shocks"] for r in rows[-3:]) / max(1, len(rows[-3:])) + print(f"\n前三局平均 {head:.1f} 次惩罚，后三局 {tail:.1f}") + return 0 -def test_odours_stay_separable_across_many_pairs(): - brain = make_brain({"seed": 7}) - codes = [{i for i, c in enumerate(present(brain, odour(s))["kenyon"]) if c} - for s in range(100, 118)] - overlaps = [len(a & b) / max(1, len(a)) - for i, a in enumerate(codes) for b in codes[i + 1:]] - assert sum(overlaps) / len(overlaps) < 0.2 +def cmd_probe(args): + """训练一只，然后把它和一只没学过的放在同样的、没见过的竞技场里，关掉可塑性。 -def test_the_compass_is_accurate_over_a_long_and_uneven_walk(): - brain = make_brain({"seed": 7}) - silent = [0.0] * GLOMERULI - start = step_brain(brain, {"odor": silent})["heading"] - total, dt = 0.0, 0.05 - for i in range(600): - rate = math.sin(i / 37.0) * 2.0 - step_brain(brain, {"odor": silent, "angular_velocity": rate}, dt) - total += rate * dt - end = step_brain(brain, {"odor": silent})["heading"] - error = math.atan2(math.sin(end - start - total), math.cos(end - start - total)) - assert abs(error) < 0.5, error + 这是唯一诚实的学习度量：同样的环境、没训练过的种子、测试期间什么都不改变。 + """ + from src import make_brain, run_episode, train + trained = train(args.task, args.episodes, args.seed)["brain"] + naive = make_brain({"seed": args.seed}) + totals = {} + for label, brain in (("naive", naive), ("trained", trained)): + shocks = [run_episode(brain, args.task, 1000 + i, learning=False)["shocks"] + for i in range(args.trials)] + totals[label] = sum(shocks) + print(f"{label:>8}: {sum(shocks):>5} 次惩罚 / {args.trials} 局 {shocks}") + print(f"\n学习带来的减少：{totals['naive'] - totals['trained']}") + return 0 -def test_punishment_is_not_forgotten_while_other_odours_come_and_go(): - brain = make_brain({"seed": 7}) - for _ in range(8): - learn(brain, present(brain, odour(200)), -1.0) - for _ in range(50): - present(brain, odour(201)) - assert present(brain, odour(200))["valence"] < 0.0 +def cmd_serve(args): + from http.server import BaseHTTPRequestHandler, HTTPServer + from src import handle -def test_the_two_compartments_do_not_share_a_synapse(): - brain = make_brain({"seed": 7}) - for _ in range(8): - learn(brain, present(brain, odour(202)), -1.0) - punished = present(brain, odour(202))["mbon"] - for _ in range(8): - learn(brain, present(brain, odour(202)), 1.0) - both = present(brain, odour(202))["mbon"] - assert both[0] == punished[0] - assert both[1] < punished[1] + class Handler(BaseHTTPRequestHandler): + def _respond(self, method): + length = int(self.headers.get("content-length") or 0) + body = self.rfile.read(length).decode("utf-8") if length else None + status, content_type, payload = handle(method, self.path, body) + data = payload.encode("utf-8") if isinstance(payload, str) else payload + self.send_response(status) + self.send_header("content-type", content_type) + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + def do_GET(self): + self._respond("GET") -def test_learning_generalises_no_further_than_the_code_overlaps(): - brain = make_brain({"seed": 7}) - before = [present(brain, odour(s))["valence"] for s in range(300, 310)] - for _ in range(10): - learn(brain, present(brain, odour(299)), -1.0) - after = [present(brain, odour(s))["valence"] for s in range(300, 310)] - assert sum(1 for a, b in zip(after, before) if a < 0.5 * b) <= 2 + def do_POST(self): + self._respond("POST") + def log_message(self, *a): + pass -def test_a_trained_animal_takes_fewer_shocks_in_the_choice_assay(): - trained = train("choice", episodes=10, seed=0)["brain"] - naive = make_brain({"seed": 0}) - unseen = range(2000, 2005) - before = sum(run_episode(naive, "choice", s, learning=False)["shocks"] for s in unseen) - after = sum(run_episode(trained, "choice", s, learning=False)["shocks"] for s in unseen) - assert after < before + server = HTTPServer((args.host, args.port), Handler) + print(f"http://{args.host}:{args.port}/") + sys.stdout.flush() + server.serve_forever() + return 0 -def test_an_animal_that_learnt_nothing_still_finds_food(): - """Nothing in the plasticity may be load-bearing for innate approach.""" - brain = make_brain({"seed": 7}) - reached = [run_episode(brain, "taxis", s, learning=False)["reached"] for s in range(8)] - assert sum(reached) >= 6 +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = parser.add_subparsers(dest="cmd", required=True) + for name, fn in (("train", cmd_train), ("probe", cmd_probe), ("serve", cmd_serve)): + p = sub.add_parser(name) + p.set_defaults(fn=fn) + if name in ("train", "probe"): + p.add_argument("--task", default="avoid") + p.add_argument("--episodes", type=int, default=20) + p.add_argument("--seed", type=int, default=0) + if name == "probe": + p.add_argument("--trials", type=int, default=5) + if name == "serve": + p.add_argument("--host", default="127.0.0.1") + p.add_argument("--port", type=int, default=8000) + args = parser.parse_args(argv) + return args.fn(args) -def test_shocks_fall_enough_to_call_it_learning(): - rows = train("avoid", episodes=14, seed=3)["episodes"] - shocks = [r["shocks"] for r in rows] - assert shocks[0] > 0 - assert sum(shocks[-4:]) < sum(shocks[:4]) -''', -} +if __name__ == "__main__": + raise SystemExit(main()) +''' def initial_files() -> Dict[str, str]: - """Everything the run starts from. No `CONTEXT.md`: `--mode b` designs the tree.""" - files = {"spec/CONTEXT.md": _SPEC, "fly.py": _DRIVER, - f"src/{SKILLS_DIR}/python-modules.md": PYTHON_MODULE_SKILL} - files.update(_TESTS) - return files + """整个仓库的起点：一份需求、一个驱动、一份写模块的须知。没有测试，没有记录。""" + return {"REQUIREMENTS.md": _REQUIREMENTS, "fly.py": _DRIVER, + f"src/{SKILLS_DIR}/python-modules.md": PYTHON_MODULE_SKILL} -FLY = TestSuite(name="fly", given=initial_files(), frozen=FROZEN, audit=_AUDIT) +FLY = TestSuite(name="fly", given=initial_files(), frozen=FROZEN, + hidden=_ACCEPTANCE, audit=_SEALED, requirements=REQUIREMENTS) build_tasks = FLY.build_tasks make_runner = FLY.make_runner suite_review = FLY.review -#: The whole suite in one interpreter -- which is the only way the shadowing -#: collision in `test_integration.py` shows up at all. -suite_failures = FLY.suite_failures -#: One test, pass or fail. No tolerance of its own -- the test owns that. +#: 父节点评审的另一半：跑**子节点自己写的**测试，并打回写了实现却没写测试的节点。 +own_review = FLY.own_review +def suite_failures(state: Mapping[str, str], *, audit: bool = False) -> List[str]: + """`complete_task` 的前提，以及跑完时报的那一行。 + + 问的不是“人的套件过没过” —— 人的套件它看不见 —— 而是**它自己那套过没过**，这正是 + 上游 manager 跑 `mix test` 的那一步。`audit` 在这里没有意义：仓库里不存在保留测试， + 保留的那套在仓库外面，由评分路径注入。保留这个形参只是为了和别的域同签名。 + """ + del audit + return FLY.own_test_failures(state) reward = reward_test -#: Exactly the audit set in the held-out tail, every driven test in the search. HELD_OUT_FRAC = FLY.held_out_frac() @@ -1336,4 +595,4 @@ def llm_manager(complete): def llm_executor(complete, *, editable=("**",), frozen=FROZEN): return _llm_executor(complete, editable=editable, frozen=frozen, - failure=TEST_FAILURE) + failure=BLIND_FAILURE) diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index 18a005af..e262306f 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -519,7 +519,11 @@ def main(argv=None) -> None: spec.llm_executor(complete) if complete else spec.offline_executor), log=log, max_depth=args.depth, max_edits=4, contracts=spec.CONTRACTS, readonly=spec.FROZEN, + # Three links, and the middle one is the only one that can ask "did you test + # what you just wrote". A domain whose tests are the agents' own supplies it. review=chain_reviews(None if args.no_parent_tests else spec.suite_review(tasks), + (getattr(spec, "own_review", lambda: None)() + if not args.no_parent_tests else None), code_review), accountability=not args.no_accountability) judge = ParentJudge(log=log, enabled=not args.engine_gate) @@ -604,12 +608,22 @@ def _superseded(rendered, task, output, reward_): # rebound its own name on first call. whole = getattr(spec, "suite_failures", None) if whole is not None and delegation.last_state is not None: - failures = whole(result.state if isinstance(result.state, dict) - else dict(result.state), audit=True) - print(f"suite, 1 process: {len(tasks) - len(failures)}/{len(tasks)}" - + ("" if not failures else - " <- these pass one-per-process and fail together:\n " - + "\n ".join(failures[:6]))) + state = result.state if isinstance(result.state, dict) else dict(result.state) + failures = whole(state, audit=True) + if getattr(getattr(spec, "FLY", None), "blind", False) or getattr( + spec, "REQUIRES_MODEL", False) and not hasattr(spec, "reference_tree"): + # A blind domain's repository suite is the agents' own, so there is no + # denominator from `tasks` to report it against -- count what is there. + written = sum(1 for p in state if p.endswith(".py") + and p.rsplit("/", 1)[-1].startswith("test_")) + print(f"their own suite : {written} test file(s), " + + ("all passing" if not failures else f"{len(failures)} failing:\n " + + "\n ".join(failures[:6]))) + else: + print(f"suite, 1 process: {len(tasks) - len(failures)}/{len(tasks)}" + + ("" if not failures else + " <- these pass one-per-process and fail together:\n " + + "\n ".join(failures[:6]))) label = "audit reward" if audited else "held-out reward" print(f"{label:<16}: {result.final_reward:.3f}" + (" (tests no agent ever saw)" if audited else "")) diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index 0a5cd7aa..a11b29fc 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -1426,9 +1426,15 @@ def test_every_domain_briefs_its_agents_with_an_objective_not_a_catalogue_line() objective = objective_for(domain) assert objective is not DOMAIN_BLURB[domain] assert objective != DOMAIN_BLURB[domain] - # Where the deliverable goes, and what has to be reachable from there. - assert "`src/`" in objective and "`src/__init__.py`" in objective + # Where the deliverable goes. That much is the contract in every domain -- + # the run that this test exists for lost `src/` out of a routing table. + assert "`src/`" in objective assert len(objective) > len(DOMAIN_BLURB[domain]) + # And what has to be reachable from there, stated however that domain states + # it: four domains name `src/__init__.py` and its entry points, and `fly` + # names the four functions its frozen driver imports, because there the + # public surface is whatever the driver reaches and nothing else. + assert "`src/__init__.py`" in objective or "`fly.py`" in objective # ... and the one thing it does not say is how to decompose below that root. for word in ("geometry", "potentials", "forces", "integrate", "observe"): From 0f1365f8c7c260fdb16a156493dac30d82c31bf5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Sep 2026 10:11:25 +0000 Subject: [PATCH 04/42] Genesis: a provider that runs the locally authenticated Claude Code binary The SDK path needs an `ANTHROPIC_API_KEY`. The CLI is authenticated another way entirely -- an OAuth session, a subscription, a corporate login -- and on a machine where that is the credential that exists there was no way to run a port at all. This session hit exactly that: the endpoint returned `401 ModelArts.81015, Invalid client ip ... rejected by api key ip whitelist setting`, and `preflight` caught it in one call rather than in 400 episodes of silent nothing. `--provider claude-cli` is a one-shot `claude -p` with every tool denied, which is a plain prompt-to-text function and so is exactly what a Completion is. Two caveats are in the docstring rather than hidden, because both change the output and not only the plumbing: the CLI wraps the prompt in its own system prompt, tool definitions and any CLAUDE.md it finds, so a call carries tens of thousands of cached input tokens the API path would not; and the cost lands on the CLI's credentials, where no budget flag here can see it. Token accounting counts cache reads and writes as the prompt tokens they are -- `input_tokens` alone reported 10 for a call that really carried 28,000. And `--no-thinking` says it has no effect here instead of being dropped in silence. Two fixes the same run needed. The Claude Code executor now runs the model the run asked for, so the architect and every executor episode are not on different models by default. And its failure template comes from the domain rather than a constant: handing a blind domain's session the assertion's source is the one thing that domain exists to prevent. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/_common.py | 84 +++++++++++++++++++- examples/genesis/_fly.py | 7 +- examples/genesis/genesis_recursive_worlds.py | 10 ++- 3 files changed, 95 insertions(+), 6 deletions(-) diff --git a/examples/_common.py b/examples/_common.py index 08f04932..ec31199f 100644 --- a/examples/_common.py +++ b/examples/_common.py @@ -25,7 +25,7 @@ from agentdescent.agents import Usage, claude, openai_compatible -PROVIDER_CHOICES = ("claude", "openai", "glm") +PROVIDER_CHOICES = ("claude", "claude-cli", "openai", "glm") # Providers served by the OpenAI-compatible adapter; 'glm' is a legacy alias. OPENAI_COMPATIBLE = ("openai", "glm") DEFAULT_MODEL = "claude-haiku-4-5" @@ -69,9 +69,10 @@ def add_standard_args( "--provider", default="claude", choices=PROVIDER_CHOICES, - help=("claude, or any OpenAI-compatible endpoint (DeepSeek, GLM, " - "vLLM, ...) via OPENAI_BASE_URL + OPENAI_API_KEY; 'glm' is a " - "legacy alias"), + help=("claude via the API, `claude-cli` via the locally " + "authenticated Claude Code binary, or any OpenAI-compatible " + "endpoint (DeepSeek, GLM, vLLM, ...) via OPENAI_BASE_URL + " + "OPENAI_API_KEY; 'glm' is a legacy alias"), ) parser.add_argument("--model", default=model_default, help=model_help) parser.add_argument("--seed", type=int, default=0) @@ -396,6 +397,72 @@ def worker_count(args: argparse.Namespace, requested: int) -> int: return 1 +def claude_cli(model: str = "haiku", *, usage: Optional[Usage] = None, + timeout: float = 300.0, binary: str = "claude"): + """A ``Completion`` that shells out to the locally authenticated Claude Code CLI. + + The SDK path needs an ``ANTHROPIC_API_KEY``. The CLI is authenticated another way + entirely -- an OAuth session, a subscription, a corporate login -- and on a machine + where that is the credential that exists, there was no way to run a port at all. + A one-shot `claude -p` with every tool denied is a plain prompt-to-text function, + which is exactly what a ``Completion`` is. + + Two honest caveats, because they change the output and not only the plumbing. The + CLI wraps the prompt in **its own** system prompt, its tool definitions and any + `CLAUDE.md` it finds, so a call carries tens of thousands of cached input tokens + that the API path would not, and the reply is coloured by an agent harness rather + than being a bare completion. And the cost lands on the CLI's credentials, not on + an API key any budget flag here can see. + """ + import json as _json + import subprocess + import time + + def complete(prompt: str) -> str: + argv = [binary, "-p", "--output-format", "json", "--model", model, + "--max-turns", "1", "--allowedTools", ""] + started = time.time() + try: + done = subprocess.run(argv, input=prompt, capture_output=True, text=True, + timeout=timeout) + except subprocess.TimeoutExpired: + if usage is not None: + usage.record(failed=True, seconds=time.time() - started) + raise RuntimeError(f"{binary} timed out after {timeout:.0f}s") + elapsed = time.time() - started + if done.returncode != 0: + if usage is not None: + usage.record(failed=True, seconds=elapsed) + raise RuntimeError(f"{binary} exited {done.returncode}: " + f"{(done.stderr or '').strip()[:300]}") + try: + envelope = _json.loads(done.stdout) + except ValueError: + if usage is not None: + usage.record(failed=True, seconds=elapsed) + raise RuntimeError(f"{binary} did not return JSON: " + f"{done.stdout.strip()[:300]}") + if envelope.get("is_error"): + if usage is not None: + usage.record(failed=True, seconds=elapsed) + raise RuntimeError(f"{binary} reported an error: " + f"{str(envelope.get('result'))[:300]}") + if usage is not None: + counts = envelope.get("usage") or {} + # Cache reads and cache writes are prompt tokens that were really sent; + # `input_tokens` alone counts only the uncached remainder and would report + # 10 for a call that carried 28,000. + usage.record(prompt_tokens=sum(int(counts.get(k) or 0) for k in + ("input_tokens", + "cache_creation_input_tokens", + "cache_read_input_tokens")), + completion_tokens=int(counts.get("output_tokens") or 0), + seconds=elapsed) + return envelope.get("result") or "" + + return complete + + class ConcurrencyGauge: """How many model calls were actually in flight at once, and at most. @@ -456,6 +523,15 @@ def completion_for(args: argparse.Namespace, *, usage: Optional[Usage] = None, that matters, since the wall-clock would then be attributed to the scheduler. """ gauge = getattr(args, "_concurrency", None) + if args.provider == "claude-cli": + if getattr(args, "no_thinking", False): + # Silently dropping it is the failure this module exists to prevent: the + # run would report thinking off while the CLI kept it on. + print("note: --no-thinking has no effect with --provider claude-cli; the " + "CLI decides its own reasoning budget", file=sys.stderr) + built = claude_cli(model=args.model, usage=usage, + timeout=float(getattr(args, "timeout", None) or 300.0)) + return gauge.wrap(built) if gauge is not None else built if is_openai_compatible(args): built = openai_compatible(model=args.model, usage=usage, **kwargs) return gauge.wrap(built) if gauge is not None else built diff --git a/examples/genesis/_fly.py b/examples/genesis/_fly.py index cbe54810..26b2bfa3 100644 --- a/examples/genesis/_fly.py +++ b/examples/genesis/_fly.py @@ -38,7 +38,8 @@ __all__ = ["CASE_NOUN", "CONTRACTS", "FLY", "FROZEN", "GROUP_NOUN", "HELD_OUT_FRAC", "OBJECTIVE", "REQUIRES_MODEL", "SCORING", "build_tasks", "initial_files", - "llm_executor", "llm_manager", "make_runner", "own_review", "reward", + "FAILURE", "llm_executor", "llm_manager", "make_runner", "own_review", + "reward", "suite_failures", "suite_review"] #: 需求和驱动。人写的全部，拒绝给任何提案，评分前恢复原样。 @@ -593,6 +594,10 @@ def llm_manager(complete): return _llm_manager(complete) +#: 失败时给执行者看什么。blind 域必须用这一套：需求、断言名、它报了什么，没有源码。 +FAILURE = BLIND_FAILURE + + def llm_executor(complete, *, editable=("**",), frozen=FROZEN): return _llm_executor(complete, editable=editable, frozen=frozen, failure=BLIND_FAILURE) diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index e262306f..c9173d66 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -510,7 +510,15 @@ def main(argv=None) -> None: return # The frozen globs are denied inside the session as well as enforced outside # it, and the failure block is the domain's own wording. - sessions = ClaudeCodeExecutor(frozen=spec.FROZEN, failure=TEST_FAILURE, + # The failure template is the domain's, not a constant: a blind domain must + # not hand the session the assertion's source, which is the whole point of it. + # And the session runs the model the run asked for -- left unset it takes the + # CLI's default, which on a `--provider claude-cli` run means the architect is + # on one model and every executor episode on another. + sessions = ClaudeCodeExecutor(frozen=spec.FROZEN, + failure=getattr(spec, "FAILURE", TEST_FAILURE), + model=(args.model if args.provider == "claude-cli" + else None), max_turns=args.max_turns) delegation = RecursiveDelegation( rollout_factory=(None if ledger is None else lambda: Rollout(ledger)), From d4cae41f3175cbfb5af0391f47fb2504286aed5d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Sep 2026 10:28:07 +0000 Subject: [PATCH 05/42] Genesis: an architect comes back, and an episode gets upstream's turn budget Three of the four gaps the deviations table recorded, closed. **Upstream's architect Phase 3.** It does not stop at design: it reviews the implementation and re-spawns refinement architects where a node misaligns (`agents/architect.ex`). This port stopped after the design, so a record written before any code existed stayed the map for ever. `--refine` checks a node's record against what is actually in the directory -- a routing table promising a child nobody created, an `## API Surface` that never mentions a file sitting right there -- and where it has drifted, re-spawns an architect on that one node to rewrite it against the code as it is. **And that is also the 62 updates.** Upstream's archive shows 26 `CONTEXT.md` creations and 62 later accepted updates affecting 19 files: a record is maintained, not written once. This port wrote them on two occasions, and a `--mode a` run sat at 0.938 for 30 003 rollouts reading a map of a layout the work had already left behind, with nothing in the mechanism able to say so. The hook fires on the leaf path as well as the accountability pass, and leaves need it more -- a leaf is where the code lands, so its API Surface is the first thing to go stale. **One episode, two budgets.** Upstream gives a root agent up to 2,048 model-tool turns and a child 128. The Claude Code executor had one number, 24, for both, which either starves the root or hands every leaf a session it has no use for. It now carries upstream's own two numbers, and a single `--max-turns` still means a single number for a cheap run. **And the node budget.** The fly domain's architect named 17 children it never reached at the default of 12, so the tree came out two deep and truncated. `--nodes` sets it. The fourth gap, the human merge on the dashboard, stays recorded and unbuilt. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- docs/algo-genesis.md | 17 +++- examples/genesis/_architect.py | 85 +++++++++++++++++++- examples/genesis/_claude_code.py | 20 +++-- examples/genesis/_delegation.py | 34 +++++++- examples/genesis/genesis_recursive_worlds.py | 34 +++++++- tests/test_genesis_example.py | 62 +++++++++++++- 6 files changed, 239 insertions(+), 13 deletions(-) diff --git a/docs/algo-genesis.md b/docs/algo-genesis.md index a250655f..01be6250 100644 --- a/docs/algo-genesis.md +++ b/docs/algo-genesis.md @@ -501,7 +501,20 @@ manager *"Own `src/__init__.py`"*. It delegated every time instead. Upstream's Architect works in three phases — *architecture & design → implementation delegation → **review & accountability*** — and is "ACCOUNTABLE for all code in its node path" (`agents/architect.ex:23`). This port stopped after the -second. A manager now gets one turn at its own node **after** its children return, +second, twice over: a manager never wrote its own node, and an architect never came +back to a node it had designed. + +The second of those is now `--refine` (on by default, `--no-refine` turns it off). +After a node's episode, its record is checked against what is actually in the +directory — a routing table promising a child nobody created, an `## API Surface` +that never mentions a file sitting right there — and where it has drifted, an +architect is re-spawned **on that one node** to rewrite the record against the code +as it is. It fires on leaves too, and leaves need it most: a leaf is where the code +lands, so its API Surface is the first thing to go stale. This is where upstream's +**62 later accepted `CONTEXT.md` updates** come from, against this port's previous +two occasions — and a `--mode a` run sat at 0.938 for 30 003 rollouts reading a map +of a layout the work had already left behind, with nothing in the mechanism able to +say so. A manager now gets one turn at its own node **after** its children return, seeing the tree as they left it, restricted to files directly at the node because a manager free to rewrite its children's work would make the decomposition decorative (`--no-accountability` turns it back into a pure router). `situate()` @@ -805,7 +818,7 @@ run — the one `md` stands in for — from §4.1 and the appendix tables. | wall clock | **123.402 h** (666.385 h of agent time) | 0.6 h | no — the domain is a stand-in, by design | | archived agent episodes | **1,019** | 237 (from ~42 proposals over 4 000 rollouts) | in kind only | | observed delegation depth | **5** (configured max 8, retries 15) | 3 (configured max 4) | in kind — both bottom out below their ceiling | -| what one episode *is* | a supervised session of **up to 2 048 root turns / 128 child turns**, with file, shell and test tools | **one model call** (plus one for a manager's accountability turn, one for the parent's review) | **no**, and this is the largest single gap | +| what one episode *is* | a supervised session of **up to 2 048 root turns / 128 child turns**, with file, shell and test tools | **one model call** by default; `--executor claude-code` makes it a Claude Code session at upstream's own two budgets — 2 048 at the root, 128 below | **with `--executor claude-code`**, and a plain completion otherwise | | result size | 750 tracked files, **248 989** physical lines | 25 files, ~700 lines | no | | model-token cost | **US$44.3760** | not billed by this endpoint; 2.9 M prompt + 0.15 M completion tokens | no | | concurrency | max **22** overlapping episodes | 4 workers | in kind | diff --git a/examples/genesis/_architect.py b/examples/genesis/_architect.py index 6293040b..b8c64281 100644 --- a/examples/genesis/_architect.py +++ b/examples/genesis/_architect.py @@ -34,7 +34,8 @@ STANDARD_SECTIONS, looks_like_file, normalise, parse_routing, shadowed_by_module) -__all__ = ["ARCHITECT_PROMPT", "ArchitectPhase", "harness_record"] +__all__ = ["ARCHITECT_PROMPT", "REFINE_PROMPT", "ArchitectPhase", + "harness_record", "misaligned"] ARCHITECT_PROMPT = """You are an architect agent in a recursive software world, \ situated at the repository path `{path}`. You design; you do not implement. @@ -102,6 +103,61 @@ An empty `children` list means this node is a leaf and its files are written here.""" +REFINE_PROMPT = """You are an architect agent, returning to a node you designed. \ +It has code in it now, and the record no longer describes what is there. + +{context} + +THE OBJECTIVE +{objective} + +WHAT DRIFTED +{reason} + +Rewrite this node's `CONTEXT.md` so it describes the directory **as it actually is**. \ +Keep the four sections and their order -- `## Intent`, `## API Surface`, \ +`## Constraints`, `## Routing Table`. Name the files that are here and what each one \ +really exposes; route only to child directories that exist. Carry over anything under \ +`## Known Issues` that is still true. + +You are revising a record, not redesigning the node: do not invent children, do not \ +rename files, and do not describe work you would rather someone had done. If the code \ +went somewhere the old record did not anticipate, the record is what is wrong. + +Reply with ONE JSON object and nothing else: +{{"record": ""}}""" + + +def misaligned(record: str, state: Mapping[str, str], path: str) -> str: + """Why this node's record no longer describes its directory, or `""`. + + Upstream's archive shows 26 `CONTEXT.md` creations and **62 later accepted + updates affecting 19 files**: a record is maintained, not written once. This port + wrote them on exactly two occasions, and the cost showed -- a `--mode a` run sat at + 0.938 because the tree it inherited described a layout the work had already left + behind, and nothing in the mechanism could say so. + + Two drifts, both cheap to see and both actionable: a routing table that promises a + child directory nobody ever created, and a file sitting at the node that the record + never mentions. Either one sends the next agent to read a map of somewhere else. + """ + prefix = f"{path}/" if path else "" + here = sorted(key for key in state + if key.startswith(prefix) and "/" not in key[len(prefix):] + and key.endswith(".py") and not key.endswith("__init__.py")) + reasons = [] + for child in parse_routing(record): + child = normalise(child) + if not any(key == child or key.startswith(child + "/") for key in state): + reasons.append(f"the routing table sends work to `{child}/`, " + f"which does not exist") + undocumented = [key for key in here if key.rsplit("/", 1)[-1] not in record] + if undocumented: + reasons.append("the API Surface does not mention " + + ", ".join(f"`{k}`" for k in undocumented[:4])) + return "; ".join(reasons[:3]) + + #: The repository root in a formation run, and the one record phase 1 does not write. #: #: Upstream's architect is started on the codebase it is creating; the repository @@ -168,6 +224,8 @@ def __init__(self, complete, *, contracts: Sequence[str] = (), self.unparsed = 0 #: Nodes still queued when the budget ran out. self.truncated = 0 + #: Records rewritten after the code moved on -- upstream's 62 updates. + self.revised = 0 def design(self, given: Mapping[str, str], objective: str) -> Dict[str, str]: """``given`` plus one ``CONTEXT.md`` per node the architect decided on.""" @@ -228,6 +286,31 @@ def design(self, given: Mapping[str, str], objective: str) -> Dict[str, str]: self.truncated = len(queue) return state + def refine(self, state: Mapping[str, str], path: str, objective: str, + reason: str) -> Optional[str]: + """Re-spawn an architect on one node that has drifted, and return its record. + + Upstream's architect does not stop at design. It works "architecture & design -> + implementation delegation -> **review & accountability**", runs the build, + reviews the implementation, and re-spawns refinement architects where a node + misaligns (`agents/architect.ex`). This port stopped after the design, so a + record written before any code existed stayed the map for ever. + """ + world = LocalWorld(version=0, path=path, readonly=self._contracts) + try: + reply = self._complete(REFINE_PROMPT.format( + objective=objective, reason=reason, + context=world.situate(state, contracts=self._contracts))) or "" + except Exception: # noqa: BLE001 - one dead call costs one revision + self.unparsed += 1 + return None + record, _ = _parse(reply, f"{path}/" if path else "") + if record is None: + self.unparsed += 1 + return None + self.revised += 1 + return record + def summary(self) -> str: return (f"designed {len(self.nodes)} nodes, deepest {self.depth}" + (f", {self.refused} outside their subtree" if self.refused else "") diff --git a/examples/genesis/_claude_code.py b/examples/genesis/_claude_code.py index 9b203fd6..71b17ca4 100644 --- a/examples/genesis/_claude_code.py +++ b/examples/genesis/_claude_code.py @@ -97,14 +97,23 @@ class ClaudeCodeExecutor: return type, and the edits it produces go through the same spatial contract. """ + #: Upstream's own budget, and the two numbers are not the same: a root agent gets + #: up to **2,048** model-tool turns and a child **128**. A root is running the whole + #: objective and a child one node of it, so one ceiling for both either starves the + #: root or hands every leaf a session it has no use for. + ROOT_TURNS = 2048 + CHILD_TURNS = 128 + def __init__(self, *, frozen: Sequence[str] = (), binary: str = "claude", - model: str = "", max_turns: int = 24, timeout: float = 900.0, + model: str = "", max_turns: int = 0, root_turns: int = 0, + timeout: float = 900.0, allow_bash: bool = True, failure: str = "", root: Optional[str] = None): self._frozen = tuple(frozen) self._binary = binary self._model = model - self._max_turns = max_turns + self._max_turns = max_turns or self.CHILD_TURNS + self._root_turns = root_turns or max_turns or self.ROOT_TURNS self._timeout = timeout self._allow_bash = allow_bash self._failure = failure @@ -151,7 +160,7 @@ def _write_settings(self, workspace: str) -> None: encoding="utf-8") as handle: json.dump(settings, handle) - def _command(self, prompt: str) -> List[str]: + def _command(self, prompt: str, turns: int) -> List[str]: tools = ["Read", "Write", "Edit", "Glob", "Grep", "TodoWrite"] if self._allow_bash: # The point of a session rather than a completion is that it can run the @@ -160,7 +169,7 @@ def _command(self, prompt: str) -> List[str]: command = [self._binary, "-p", prompt, "--output-format", "json", "--permission-mode", "acceptEdits", - "--max-turns", str(self._max_turns), + "--max-turns", str(turns), "--allowedTools", ",".join(tools), "--disallowedTools", "WebFetch,WebSearch,Task"] if self._model: @@ -175,7 +184,8 @@ def _run(self, workspace: str, owner: str, brief: Brief) -> bool: output=(brief.output or "")[:400], reward=brief.reward) if self._failure else "") try: - out = subprocess.run(self._command(prompt), cwd=workspace, + turns = self._root_turns if brief.depth == 0 else self._max_turns + out = subprocess.run(self._command(prompt, turns), cwd=workspace, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=self._timeout) except Exception: # noqa: BLE001 - a dead session costs its episode diff --git a/examples/genesis/_delegation.py b/examples/genesis/_delegation.py index aefcdbb3..19832491 100644 --- a/examples/genesis/_delegation.py +++ b/examples/genesis/_delegation.py @@ -176,6 +176,12 @@ class RecursiveDelegation: #: manager gets one turn at its own node. Off makes a manager a pure router, #: which is what this port was and why a node's own file never appeared. accountability: bool = True + #: ``(path, state) -> a rewritten CONTEXT.md, or None``. Upstream's architect does + #: not stop at design: it reviews the implementation and re-spawns refinement + #: architects where a node misaligns (`agents/architect.ex`). Called once per + #: accountability pass; the callable decides whether the node has drifted at all, + #: so this module does not have to know what drift looks like. + refine: Optional[Callable[[str, Mapping[str, str]], Optional[str]]] = None #: The parent's judgement beyond scope. Upstream a parent decides "using the #: available tests, constraints and integration evidence" (paper 3.3), which #: is a *test* run on the child's work before it is offered to the version @@ -212,6 +218,10 @@ class RecursiveDelegation: #: Review findings carried into a manager's own turn rather than left with the #: child that could not act on them -- see `_accountability_pass`. accountability_findings: int = 0 + #: Records rewritten because the code moved on. Upstream's archive shows 26 + #: `CONTEXT.md` creations and **62 later accepted updates**: a record is + #: maintained, not written once. + records_revised: int = 0 #: Edits whose path was written relative to the agent's own node rather than #: to the repository. Counted because the alternative to counting is a file #: appearing somewhere nobody asked for it. @@ -306,6 +316,7 @@ def _episode(self, world: LocalWorld, objective: str, state: Mapping[str, str], continue produced.append(Edit(owner=world.path, path=path, content=raw.content, kind=raw.kind)) + produced += self._refresh_record(world, state) for edit in produced: if edit.path.endswith(CONTEXT_FILE) and state.get(edit.path) != edit.content: self.record_updates += 1 @@ -475,7 +486,7 @@ def _accountability_pass(self, world: LocalWorld, objective: str, context=world.situate(amended, contracts=self.contracts), state=amended, task=ctx.task, output=ctx.output, reward=ctx.reward, depth=depth) - out: List[Edit] = [] + out: List[Edit] = list(self._refresh_record(world, amended)) for raw in self.executor(brief) or (): path, relative = resolve_edit_path(amended, world.path, raw.path) self.resolved_relative += int(relative) @@ -492,6 +503,24 @@ def _accountability_pass(self, world: LocalWorld, objective: str, self.accountability_edits += len(out) return out + def _refresh_record(self, world: LocalWorld, + state: Mapping[str, str]) -> List[Edit]: + """Re-spawn an architect on this node if its record no longer fits it. + + Both paths need it and the leaf path needs it more: a manager's record drifts + when a child goes somewhere unplanned, but a **leaf** is where the code actually + lands, so its `## API Surface` is the one that goes stale first. Hooking only + the accountability pass would have missed every leaf in the tree. + """ + if self.refine is None: + return [] + key = f"{world.path}/{CONTEXT_FILE}" if world.path else CONTEXT_FILE + revised = self.refine(world.path, state) + if not revised or revised == state.get(key): + return [] + self.records_revised += 1 + return [Edit(owner=world.path, path=key, content=revised, kind="context")] + def _fold(self, held: Dict[str, Edit], owner_of: Dict[str, str], returned: Sequence[Edit], child: LocalWorld, state: Mapping[str, str]) -> List[str]: @@ -693,7 +722,8 @@ def stats(self) -> str: f"node_relative_paths={self.resolved_relative} " f"accountability={self.accountability_edits}/" f"{self.accountability_declined} " - f"findings_carried={self.accountability_findings}") + f"findings_carried={self.accountability_findings} " + f"records_revised={self.records_revised}") def _with_findings(objective: str, path: str, diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index c9173d66..d9a0a6d9 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -93,7 +93,8 @@ from . import _jqx as jqx from . import _md as md from . import _stackvm as stackvm -from ._architect import ArchitectPhase, harness_record, missing_sections +from ._architect import (ArchitectPhase, harness_record, misaligned, + missing_sections) from ._claude_code import ClaudeCodeExecutor, claude_code_available from ._extract import ExtractPhase from ._judge import ParentJudge @@ -273,6 +274,13 @@ def build_parser() -> argparse.ArgumentParser: "domain's empty one. With --mode a this is the Context " "Tree extraction upstream runs on an existing codebase; " "the frozen contract is always the domain's own") + parser.add_argument("--no-refine", action="store_true", + help="do not re-spawn an architect on a node whose record has " + "drifted from its files (upstream's architect Phase 3)") + parser.add_argument("--nodes", type=int, default=12, + help="how many CONTEXT.md records phase 1 may write. The fly " + "domain's architect named 17 children it never reached at " + "the default, so the tree came out two deep and truncated") parser.add_argument("--architect", action="store_true", help="upstream's Phase 1: an agent designs the CONTEXT.md tree " "-- intent, API surface, constraints, routing tables -- " @@ -481,7 +489,8 @@ def main(argv=None) -> None: initial[CONTEXT_FILE] = harness_record(root, spec.FROZEN, objective_for(args.domain)) architect = ArchitectPhase(complete, contracts=spec.CONTRACTS, - max_depth=args.depth, root_path=root) + max_depth=args.depth, max_nodes=args.nodes, + root_path=root) initial = architect.design(initial, objective_for(args.domain)) print(f"\nPhase 1 : architect {architect.summary()}") for path in architect.nodes: @@ -520,7 +529,28 @@ def main(argv=None) -> None: model=(args.model if args.provider == "claude-cli" else None), max_turns=args.max_turns) + def _refine(path, state): + """Upstream's architect Phase 3, as a hook on the accountability pass. + + It does not stop at design: it reviews the implementation and re-spawns + refinement architects where a node misaligns (`agents/architect.ex`). This port + stopped after the design, so a record written before any code existed stayed the + map for ever -- and a `--mode a` run sat at 0.938 reading a map of a layout the + work had already left behind. Upstream's archive shows 26 record creations and + **62 later accepted updates**; this is where the updates come from. + """ + if architect is None: + return None + key = f"{path}/{CONTEXT_FILE}" if path else CONTEXT_FILE + record = state.get(key) + if not record: + return None + reason = misaligned(record, state, path) + return architect.refine(state, path, objective_for(args.domain), + reason) if reason else None + delegation = RecursiveDelegation( + refine=None if (architect is None or args.no_refine) else _refine, rollout_factory=(None if ledger is None else lambda: Rollout(ledger)), manager=spec.llm_manager(complete) if complete else spec.offline_manager, executor=(sessions if sessions is not None else diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index a11b29fc..19f1fd1f 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -1564,6 +1564,66 @@ def slow(prompt): assert gauge.peak == 3 +def test_a_root_session_gets_a_bigger_turn_budget_than_a_child(): + """Upstream's own two numbers: up to 2,048 model-tool turns at the root, 128 below. + + A root agent is running the whole objective and a child one node of it, so a single + ceiling for both either starves the root or hands every leaf a session it has no + use for. The port had one number, 24, for both. + """ + from examples.genesis._claude_code import ClaudeCodeExecutor + + executor = ClaudeCodeExecutor() + assert executor.ROOT_TURNS == 2048 and executor.CHILD_TURNS == 128 + assert executor._root_turns == 2048 and executor._max_turns == 128 + root = executor._command("x", executor._root_turns) + child = executor._command("x", executor._max_turns) + assert root[root.index("--max-turns") + 1] == "2048" + assert child[child.index("--max-turns") + 1] == "128" + # One number given explicitly still means one number, for a cheap run. + pinned = ClaudeCodeExecutor(max_turns=12) + assert pinned._root_turns == 12 and pinned._max_turns == 12 + + +def test_a_record_is_maintained_rather_than_written_once(): + """Upstream: 26 `CONTEXT.md` creations and **62 later accepted updates**. + + This port wrote them on exactly two occasions, and a `--mode a` run sat at 0.938 + reading a map of a layout the work had already left behind -- nothing in the + mechanism could say the record was stale. Upstream's architect does not stop at + design: it reviews the implementation and re-spawns refinement architects where a + node misaligns (`agents/architect.ex`). + """ + from examples.genesis._architect import misaligned + + record = ("## Intent\nx\n\n## API Surface\n`kc.py` does things.\n\n" + "## Routing Table\n- `./src/a/gone/` -> nothing is here\n") + drift = misaligned(record, {"src/a/kc.py": "x", "src/a/extra.py": "y"}, "src/a") + assert "does not exist" in drift # a route to a directory nobody made + assert "extra.py" in drift # a file the record never mentions + assert misaligned(record, {"src/a/kc.py": "x", "src/a/gone/y.py": "z"}, + "src/a") == "" + + # And the hook produces a record edit rather than a verdict, because a verdict + # cannot fix a map. + revisions = [] + + def refine(path, state): + revisions.append(path) + return "## Intent\nrewritten\n" + + policy = RecursiveDelegation(manager=lambda b: [], executor=lambda b: [], + log=WorldLog(), refine=refine) + state = {CONTEXT_FILE: "# root\n"} + proposals = policy.propose(_proposal_ctx(state, Task(id="t", prompt="x"))) + assert revisions == [""] + assert policy.records_revised == 1 + # A leaf that delegated nothing still gets its record refreshed -- a leaf is where + # the code lands, so its API Surface is the one that goes stale first. + assert any(CONTEXT_FILE in payload and "rewritten" in payload + for payload in proposals) + + def test_a_node_may_not_shadow_a_sibling_module(): """`rdf/` next to `rdf.py` are one name to Python, and the module wins. @@ -1808,7 +1868,7 @@ def test_the_session_is_fenced_before_the_contract_ever_sees_it(tmp_path): """Three fences, and this is the first two: the frozen globs are denied by name in the session's own settings, and the network tools are off.""" executor = ClaudeCodeExecutor(frozen=md.FROZEN, binary="claude") - command = executor._command("do the thing") + command = executor._command("do the thing", executor.CHILD_TURNS) assert "--disallowedTools" in command assert "WebFetch,WebSearch,Task" in command assert "--permission-mode" in command and "acceptEdits" in command From 36af996924605e43606dee5bc9ca4af73fff5483 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Sep 2026 10:30:16 +0000 Subject: [PATCH 06/42] Genesis: --max-turns defaults to upstream's split rather than overriding it The executor learned upstream's two budgets -- 2048 at the root, 128 below -- and then the parser's own default of 24 fed straight past them, so a run configured for the split reported "up to 24 turns" and meant it. The default is now 0, which means "use the split", and a number given explicitly still pins both to it for a cheap run. The header prints both numbers rather than one. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/genesis/genesis_recursive_worlds.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index d9a0a6d9..277504d8 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -196,9 +196,11 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--workers", type=int, default=4) parser.add_argument("--depth", type=int, default=3, help="maximum recursive delegation depth (v,p) -> (v,q)") - parser.add_argument("--max-turns", type=int, default=24, - help="turns inside one --executor claude-code episode. " - "Upstream's runs allow 128 for a non-root episode") + parser.add_argument("--max-turns", type=int, default=0, + help="turns inside one --executor claude-code episode. The " + "default is upstream's own split -- 2048 at the root, 128 " + "below it -- and a number here pins both to it, which is " + "what a cheap run wants") parser.add_argument("--cold-start", action="store_true", help="start from the goal, the contract and the suite only: " "no node CONTEXT.md records, no routing tables, no " @@ -353,7 +355,10 @@ def main(argv=None) -> None: " [git missing: every contested file falls back]")) print(f"Gate : {gate}") print("Episode : " + ("one headless Claude Code session in a throwaway worktree, " - f"up to {args.max_turns} turns, frozen paths denied, network " + f"up to {args.max_turns or ClaudeCodeExecutor.ROOT_TURNS} " + f"turns at the root and " + f"{args.max_turns or ClaudeCodeExecutor.CHILD_TURNS} below, " + f"frozen paths denied, network " "tools off -- billed to the local CLI's credentials" if args.executor == "claude-code" else "one model call returning whole files")) From 325894176db921babb3b3e103979da9a18b08b64 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Sep 2026 10:59:23 +0000 Subject: [PATCH 07/42] Genesis: phase 1 can continue a phase 1 that ran out of budget The fly domain's architect spent 27 minutes designing 32 nodes and was about to hit `--nodes 40` with a queue still behind it. Raising the budget meant paying for all 32 again, because `design()` asked for every node it reached whether or not a record was already there. With `resume`, a node that already carries a record is kept: its routing table is taken as the design, its children are queued, and no call is spent. The thing that revises a record once code exists is the refinement architect, not this. Off by default, and deliberately. At `root_path=""` the record already in `given` is the *harness* record -- generated, not designed -- so reusing it would skip the root and leave the tree with no design at all. The runner turns it on only for `--continue-from`, where a record that is there really is an earlier phase 1's work. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/genesis/_architect.py | 28 +++++++++++++++++++- examples/genesis/genesis_recursive_worlds.py | 5 +++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/examples/genesis/_architect.py b/examples/genesis/_architect.py index b8c64281..9cb96793 100644 --- a/examples/genesis/_architect.py +++ b/examples/genesis/_architect.py @@ -210,12 +210,20 @@ class ArchitectPhase: """ def __init__(self, complete, *, contracts: Sequence[str] = (), - max_depth: int = 3, max_nodes: int = 12, root_path: str = ""): + max_depth: int = 3, max_nodes: int = 12, root_path: str = "", + resume: bool = False): self._complete = complete self._contracts = tuple(contracts) self._max_depth = max_depth self._max_nodes = max_nodes self._root = normalise(root_path) + #: Keep a record an earlier phase 1 already wrote instead of re-asking. + #: Off by default and deliberately: at `root_path=""` the record already in + #: `given` is the *harness* record, which is generated rather than designed, + #: and skipping the root because of it would leave the tree with no design at + #: all. The runner turns it on only for `--continue-from`, where a record that + #: is there really is an earlier phase 1's work. + self._resume = resume #: Nodes it designed, and the deepest it went. self.nodes: List[str] = [] self.depth = 0 @@ -226,6 +234,8 @@ def __init__(self, complete, *, contracts: Sequence[str] = (), self.truncated = 0 #: Records rewritten after the code moved on -- upstream's 62 updates. self.revised = 0 + #: Nodes an earlier phase 1 already designed and this one kept. + self.reused = 0 def design(self, given: Mapping[str, str], objective: str) -> Dict[str, str]: """``given`` plus one ``CONTEXT.md`` per node the architect decided on.""" @@ -233,6 +243,20 @@ def design(self, given: Mapping[str, str], objective: str) -> Dict[str, str]: queue: List[Tuple[str, str, int]] = [(self._root, objective, 0)] while queue and len(self.nodes) < self._max_nodes: path, node_objective, depth = queue.pop(0) + key = f"{path}/{CONTEXT_FILE}" if path else CONTEXT_FILE + if self._resume and state.get(key): + # Already designed, by an earlier phase 1 that ran out of budget. + # Re-asking would spend a call to get the same answer and would throw + # away a record the run may already have grown code against; the thing + # that revises a record once code exists is the refinement architect, + # not this. So take the routing table as the design and walk on. + self.reused += 1 + self.nodes.append(path) + self.depth = max(self.depth, depth) + if depth + 1 <= self._max_depth: + queue += [(normalise(child), node_objective, depth + 1) + for child in parse_routing(state[key])] + continue record, children = self._ask(state, path, node_objective, depth) if record is None: continue @@ -313,6 +337,8 @@ def refine(self, state: Mapping[str, str], path: str, objective: str, def summary(self) -> str: return (f"designed {len(self.nodes)} nodes, deepest {self.depth}" + + (f" ({self.reused} kept from an earlier phase 1)" + if self.reused else "") + (f", {self.refused} outside their subtree" if self.refused else "") + (f", {self.mistaken_nodes} file paths refused as nodes" if self.mistaken_nodes else "") diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index 277504d8..aa984f11 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -495,7 +495,10 @@ def main(argv=None) -> None: objective_for(args.domain)) architect = ArchitectPhase(complete, contracts=spec.CONTRACTS, max_depth=args.depth, max_nodes=args.nodes, - root_path=root) + root_path=root, + # A phase 1 that ran out of budget is worth + # continuing rather than paying for twice. + resume=bool(args.continue_from)) initial = architect.design(initial, objective_for(args.domain)) print(f"\nPhase 1 : architect {architect.summary()}") for path in architect.nodes: From 3eff96a962b996d1af87da273942f005e9f63cfe Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Sep 2026 11:06:18 +0000 Subject: [PATCH 08/42] Genesis: a resumed phase 1 hands each child its own objective The routing line's right-hand side is not decoration -- it is the objective the parent's architect handed that child. A phase 1 continuing from records rather than from replies has nothing else to give one, and the first version passed down the *parent's* objective instead. What that does is visible in the run that found it. An architect asked to design `src/brain/circuits/cell_types/mushroom_body`, while carrying the root objective "implement the software REQUIREMENTS.md asks for", came back with children `brain, learning, environment, simulation` -- it had redesigned the whole library from the top, at depth five, three nodes running. `parse_routes` returns `(path, what it handles)` where `parse_routing` returns the paths alone, and resume queues each child with its own line. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/genesis/_architect.py | 13 +++++++++--- examples/genesis/_world.py | 35 +++++++++++++++++++++++++++++++++ tests/test_genesis_example.py | 36 ++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 3 deletions(-) diff --git a/examples/genesis/_architect.py b/examples/genesis/_architect.py index 9cb96793..906cd98e 100644 --- a/examples/genesis/_architect.py +++ b/examples/genesis/_architect.py @@ -32,7 +32,7 @@ from ._world import (CONTEXT_FILE, LocalWorld, ROUTING_HEADING, STANDARD_SECTIONS, looks_like_file, normalise, - parse_routing, shadowed_by_module) + parse_routes, parse_routing, shadowed_by_module) __all__ = ["ARCHITECT_PROMPT", "REFINE_PROMPT", "ArchitectPhase", "harness_record", "misaligned"] @@ -254,8 +254,15 @@ def design(self, given: Mapping[str, str], objective: str) -> Dict[str, str]: self.nodes.append(path) self.depth = max(self.depth, depth) if depth + 1 <= self._max_depth: - queue += [(normalise(child), node_objective, depth + 1) - for child in parse_routing(state[key])] + # The routing line's right-hand side is the objective this node's + # architect handed that child. Passing `node_objective` down + # instead sends a leaf the whole project: measured, an architect + # asked to design `.../cell_types/mushroom_body` while carrying the + # root objective came back with `brain, learning, environment, + # simulation`, having redesigned the library from the top at depth + # five. + queue += [(normalise(child), handles or node_objective, depth + 1) + for child, handles in parse_routes(state[key])] continue record, children = self._ask(state, path, node_objective, depth) if record is None: diff --git a/examples/genesis/_world.py b/examples/genesis/_world.py index 75fe5bc1..e4184a05 100644 --- a/examples/genesis/_world.py +++ b/examples/genesis/_world.py @@ -119,6 +119,41 @@ def parse_routing(body: str) -> List[str]: return out +def parse_routes(body: str) -> List[Tuple[str, str]]: + """``(path, what it handles)`` for every routing-table entry, in order. + + The right-hand side of a routing line is not decoration: it is the objective the + parent is handing that child, written by the architect that opened it. A phase 1 + resuming from records rather than from replies has nothing else to give a child, + and giving it the *parent's* objective instead sends a leaf the whole project -- + measured: an architect asked to design `.../cell_types/mushroom_body` while + carrying the root objective came back with `brain, learning, environment, + simulation`, having redesigned the library from the top at depth five. + """ + if not body: + return [] + out: List[Tuple[str, str]] = [] + seen = set() + inside = False + for line in body.splitlines(): + stripped = line.strip() + if stripped.startswith("##"): + inside = stripped.lower().startswith(ROUTING_HEADING.lower()) + continue + if not inside: + continue + match = _ROUTE_LINE.match(line) + if match: + path = normalise(match.group("path")) + if path and path not in seen: + seen.add(path) + # The regex captures only the path; what the entry *handles* is + # whatever follows it on the line. + handles = line[match.end():].strip().lstrip("-—>").strip() + out.append((path, handles)) + return out + + def shadowed_by_module(state, path: str) -> bool: """Would a node at ``path`` be shadowed by a sibling module of the same name? diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index 19f1fd1f..ac120ea8 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -1624,6 +1624,42 @@ def refine(path, state): for payload in proposals) +def test_a_resumed_phase_one_hands_each_child_its_own_objective(): + """The routing line's right-hand side is the objective, not decoration. + + A phase 1 continuing from records rather than from replies has nothing else to + give a child. Handing down the *parent's* objective instead sends a leaf the whole + project: an architect asked to design `.../cell_types/mushroom_body` while carrying + the root objective came back with `brain, learning, environment, simulation`, + having redesigned the library from the top at depth five. + """ + from examples.genesis._world import parse_routes + + record = ("## Routing Table\n" + "- `./src/brain/` -> the connectome-grounded neural model\n" + "- `./src/arena/` -> arena physics and odour fields\n") + assert parse_routes(record) == [("src/brain", "the connectome-grounded neural model"), + ("src/arena", "arena physics and odour fields")] + # And the paths still come back exactly as `parse_routing` reports them. + assert [p for p, _ in parse_routes(record)] == parse_routing(record) + + asked = [] + + def complete(prompt): + asked.append(prompt.split("THE OBJECTIVE\n", 1)[1].splitlines()[0]) + return json.dumps({"record": "# leaf\n", "children": []}) + + given = {CONTEXT_FILE: "# root\n", + "src/" + CONTEXT_FILE: "# src\n" + record} + phase = ArchitectPhase(complete, max_depth=3, max_nodes=10, root_path="src", + resume=True) + phase.design(given, "build the whole product") + assert phase.reused == 1 # `src` kept, no call spent + assert asked == ["the connectome-grounded neural model", + "arena physics and odour fields"] + assert "build the whole product" not in asked + + def test_a_node_may_not_shadow_a_sibling_module(): """`rdf/` next to `rdf.py` are one name to Python, and the module wins. From 3d12d97dd280d7e7e6cefb2a9d8c1179e55f9a39 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Sep 2026 11:44:57 +0000 Subject: [PATCH 09/42] Genesis: a routing table has to agree with the children, in both directions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream calls the routing table "your primary delegation tool ... the map that makes recursive delegation work". A record whose table disagrees with the children the architect just opened is a broken map, and phase 1 now repairs it rather than passing it on. Dropping a refused entry was already there -- a table advertising a node nobody may write is a trap for the next manager, which would delegate there and be refused in turn. Adding a forgotten one turned out to matter more. An architect answered with five children and a `## Routing Table` section holding its API Surface instead: "`__init__.py` — Exports get_regions(...)", then two sentences about FlyWire. Nothing looked wrong that run, because the children were queued from the *reply*. Then the budget ran out, a later phase 1 resumed from records, rebuilt its queue from the **table**, found no routes in it, and dropped an entire subtree of the brain -- five regions, silently, because the record and the reply had disagreed and only the reply was ever right. And a second guard from the same run. "If the objective feels too large, that is exactly the signal to decompose MORE aggressively" has a counterweight upstream states in the same breath: single responsibility, and shared capability belongs at the lowest common ancestor. A node named after one of its own ancestors is that rule broken in the one way a tree can show -- what is really in there belongs to the ancestor, or the ancestor's name was wrong, and either way two places claim it and no later agent can tell which. The run produced `.../receptor/types/catalog/types` and `.../catalog/demographics` beside an existing `.../receptor/population/demographics`. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/genesis/_architect.py | 79 +++++++++++++++++++++++++++++----- tests/test_genesis_example.py | 68 +++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 11 deletions(-) diff --git a/examples/genesis/_architect.py b/examples/genesis/_architect.py index 906cd98e..be8628e5 100644 --- a/examples/genesis/_architect.py +++ b/examples/genesis/_architect.py @@ -236,6 +236,8 @@ def __init__(self, complete, *, contracts: Sequence[str] = (), self.revised = 0 #: Nodes an earlier phase 1 already designed and this one kept. self.reused = 0 + #: Children refused for being named after one of their ancestors. + self.repeated = 0 def design(self, given: Mapping[str, str], objective: str) -> Dict[str, str]: """``given`` plus one ``CONTEXT.md`` per node the architect decided on.""" @@ -295,6 +297,13 @@ def design(self, given: Mapping[str, str], objective: str) -> Dict[str, str]: # file there is refused to every proposal. self.refused += 1 continue + if repeats_an_ancestor(child["path"]): + # "decompose MORE aggressively" has a counterweight and upstream + # states both: shared capability belongs at the lowest common + # ancestor, so a node named after one of its own ancestors is that + # rule broken in the one way a tree can show. + self.repeated += 1 + continue if normalise(child["path"]) == path: # A node routes to its *children*. One architect wrote # `potentials -> potentials`, which is a manager delegating to @@ -308,11 +317,10 @@ def design(self, given: Mapping[str, str], objective: str) -> Dict[str, str]: continue accepted.append(child) queue.append((normalise(child["path"]), child["objective"], depth + 1)) - if len(accepted) != len(children): - # A table that advertises a node the contract refuses is a trap for the - # next manager to read it: it would delegate there and be refused in - # turn. The record keeps only what was accepted. - state[key] = _only_routing(record, [c["path"] for c in accepted]) + # Always, not only when something was refused: the table has to agree with + # the children in both directions. See `_fix_routing` for the subtree that + # went missing because it only ever agreed in one. + state[key] = _fix_routing(record, accepted) if queue: self.truncated = len(queue) return state @@ -349,6 +357,8 @@ def summary(self) -> str: + (f", {self.refused} outside their subtree" if self.refused else "") + (f", {self.mistaken_nodes} file paths refused as nodes" if self.mistaken_nodes else "") + + (f", {self.repeated} repeated an ancestor's name" + if self.repeated else "") + (f", {self.truncated} left undesigned at the node budget" if self.truncated else "") + (f", {self.unparsed} replies unusable" if self.unparsed else "")) @@ -415,26 +425,73 @@ def _parse(reply: str, prefix: str) -> Tuple[Optional[str], List[Dict[str, str]] return record, children -def _only_routing(record: str, keep: Sequence[str]) -> str: - """``record`` with every routing-table line that names a path outside ``keep`` gone.""" - kept = {normalise(p) for p in keep} - out, in_table = [], False +def _fix_routing(record: str, keep: Sequence[Mapping[str, str]]) -> str: + """Make the routing table say exactly which children this node opened. + + Upstream: "the Routing Table is your primary delegation tool ... they are the map + that makes recursive delegation work". A record whose table disagrees with the + children the architect just opened is a broken map, and the phase repairs it rather + than passing it on -- in both directions. + + **Dropping** an entry the contract refused was always necessary: a table that + advertises a node nobody may write is a trap for the next manager, which would + delegate there and be refused in turn. + + **Adding** a child the table forgot turned out to matter more. One architect + answered with five children and a `## Routing Table` section holding its API + Surface instead -- `` `__init__.py` — Exports get_regions(...) ``, then two + sentences about FlyWire. The children were queued and designed that run, so nothing + looked wrong. Then the budget ran out, and a later phase 1 resuming from records + rebuilt its queue from the **table**, found no routes in it, and dropped an entire + subtree of the brain -- five regions, silently, because the record and the reply had + disagreed and only the reply was ever right. + """ + kept = [(normalise(c["path"]), (c.get("objective") or "").strip()) for c in keep] + names = {path for path, _ in kept} + out, in_table, seen, done = [], False, set(), False + + def table_lines(): + return [f"- `./{path}/` -> {objective or 'this subtree'}" + for path, objective in kept if path not in seen] + for line in record.splitlines(): if line.strip().lower().startswith(ROUTING_HEADING.lower()): - in_table = True + in_table, done = True, True out.append(line) continue if in_table: if line.startswith("## "): + out.extend(table_lines()) in_table = False elif line.strip().startswith("-"): named = parse_routing(ROUTING_HEADING + "\n" + line) - if named and normalise(named[0]) not in kept: + if not named or normalise(named[0]) not in names: continue + seen.add(normalise(named[0])) out.append(line) + if in_table: + out.extend(table_lines()) + elif not done and kept: + # No routing section at all, and children to route to. + out += ["", ROUTING_HEADING] + table_lines() return "\n".join(out).rstrip("\n") + "\n" +def repeats_an_ancestor(path: str) -> str: + """The ancestor a node's own name repeats, or `""`. + + `.../receptor/types/catalog/types` is a node named after its own great-grandparent, + and so is `.../mushroom_body/.../mushroom_body`. Upstream states the counterweight + to "decompose MORE aggressively" in the same breath -- single responsibility, and + **shared capability belongs at the lowest common ancestor** -- and a directory named + after something already above it is that rule broken in the one way a tree can show: + whatever is really in there belongs to the ancestor, or the ancestor's name was + wrong. Either way two places now claim it and no later agent can tell which. + """ + parts = [p for p in normalise(path).split("/") if p] + return parts[-1] if len(parts) > 1 and parts[-1] in parts[:-1] else "" + + def missing_sections(record: str) -> List[str]: """Which of upstream's four standard sections a record does not have.""" lowered = record.lower() diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index ac120ea8..70dbd249 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -1624,6 +1624,74 @@ def refine(path, state): for payload in proposals) +def test_the_routing_table_is_made_to_agree_with_the_children_it_opened(): + """Upstream: the routing table "is your primary delegation tool ... the map that + makes recursive delegation work". A record whose table disagrees with the children + the architect just opened is a broken map, and it has to be repaired in *both* + directions. + + Dropping a refused entry was always necessary -- a table advertising a node nobody + may write is a trap for the next manager. Adding a forgotten one turned out to + matter more. An architect answered with five children and a `## Routing Table` + holding its API Surface instead: `` `__init__.py` — Exports get_regions(...) ``, + then two sentences about FlyWire. Nothing looked wrong that run, because the + children were queued from the *reply*. Then the budget ran out, a later phase 1 + resumed from records, rebuilt its queue from the **table**, found no routes, and + dropped five brain regions in silence. + """ + from examples.genesis._architect import _fix_routing + + api_surface_in_the_wrong_section = ( + "## Intent\nx\n\n## Routing Table\n" + "- `__init__.py` — Exports `get_regions(...)`\n" + "- All regional organization grounded in FlyWire\n") + fixed = _fix_routing(api_surface_in_the_wrong_section, + [{"path": "src/r/antennal_lobe", "objective": "the AL"}, + {"path": "src/r/mushroom_body", "objective": "the MB"}]) + assert parse_routing(fixed) == ["src/r/antennal_lobe", "src/r/mushroom_body"] + assert "get_regions" not in fixed + + # A refused child still goes, and one the table already lists is not duplicated. + listed = ("## Routing Table\n- `./src/r/keep/` -> stays\n" + "- `./src/r/gone/` -> refused\n") + fixed = _fix_routing(listed, [{"path": "src/r/keep", "objective": "stays"}]) + assert parse_routing(fixed) == ["src/r/keep"] + assert fixed.count("src/r/keep") == 1 + + # And a record with no routing section at all gets one. + fixed = _fix_routing("## Intent\nx\n", [{"path": "a/b", "objective": "o"}]) + assert parse_routing(fixed) == ["a/b"] + + +def test_a_node_may_not_be_named_after_one_of_its_own_ancestors(): + """"Decompose MORE aggressively" has a counterweight and upstream states both. + + Single responsibility, and **shared capability belongs at the lowest common + ancestor**. A directory named after something already above it is that rule broken + in the one way a tree can show: whatever is really in there belongs to the ancestor, + or the ancestor's name was wrong, and either way two places claim it. Measured: an + architect produced `.../receptor/types/catalog/types` and `.../catalog/demographics` + beside an existing `.../receptor/population/demographics`. + """ + from examples.genesis._architect import repeats_an_ancestor + + assert repeats_an_ancestor("a/types/catalog/types") == "types" + assert repeats_an_ancestor("src/brain/mushroom_body/x/mushroom_body") == "mushroom_body" + assert repeats_an_ancestor("src/brain/circuits/cell_types") == "" + assert repeats_an_ancestor("src") == "" + + phase = ArchitectPhase(_scripted_architect({ + "src/a": {"record": "# a\n", "children": [{"path": "src/a/a", "objective": "no"}, + {"path": "src/a/b", "objective": "yes"}]}}), + max_depth=2, root_path="src/a") + tree = phase.design({CONTEXT_FILE: "# root\n"}, "o") + assert phase.repeated == 1 + assert "src/a/b/" + CONTEXT_FILE in tree + assert "src/a/a/" + CONTEXT_FILE not in tree + # and the refused one is not left advertised in the table either + assert parse_routing(tree["src/a/" + CONTEXT_FILE]) == ["src/a/b"] + + def test_a_resumed_phase_one_hands_each_child_its_own_objective(): """The routing line's right-hand side is the objective, not decoration. From 6c0ad763f16ee68537b7de72608041f9aef6c714 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Sep 2026 11:58:44 +0000 Subject: [PATCH 10/42] Genesis: the fly domain freezes one file, and phase 1 designs a level at a time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **The driver was a choice, and this port's own deviations table said so**: "a frozen entry point, so the result is a program rather than a package nobody can invoke. A choice, and a defensible one." Upstream Mode B takes an objective and nothing else; c-testsuite, LLVM and Csmith are the experimenter's benchmark, applied afterwards, never in the repository. Upstream does have a contract -- it is just implicit in the objective. "Build a C compiler" pins `cc -o foo foo.c`, an convention everyone already knows, which is why c-testsuite can run at all. "A fruit fly simulator" has no such convention, so it has to be said -- but it belongs in the *requirement* (how I want to use it) rather than in the repository (here is code you may not touch). `REQUIREMENTS.md` §4 now says which three subcommands must work and that each takes `--format json`, because a person who wants to plot a learning curve asks for machine-readable output. `fly.py` is theirs to write. So both suites are black box now. They start `python fly.py ...`, read its JSON, drive its HTTP server over a socket, and import nothing -- which is the shape of upstream's own validation. One file is frozen: the requirement. **And phase 1 designs a level at a time.** Upstream *spawns* sub-architects, which is a statement about independence: every node at a depth inherits the chain down to its own parent, designed a level ago, so nothing in a level can depend on anything else in it. Running them one after another was this port's choice and it cost the fly domain 32 minutes for 71 nodes. The asks read a snapshot rather than the live tree, so a sibling can never see another sibling's record even if it finishes first, and the tree does not depend on which call returns when. Two bugs the change surfaced, both fixed: `_apply` still carried the serial loop's own `self._ask`, so every node was asked twice; and resume was decided after the dispatch, which spends exactly the call it exists to save. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/genesis/_architect.py | 101 ++- examples/genesis/_fly.py | 674 ++++++++++--------- examples/genesis/genesis_recursive_worlds.py | 7 +- tests/test_genesis_example.py | 45 ++ 4 files changed, 473 insertions(+), 354 deletions(-) diff --git a/examples/genesis/_architect.py b/examples/genesis/_architect.py index be8628e5..027afc75 100644 --- a/examples/genesis/_architect.py +++ b/examples/genesis/_architect.py @@ -211,7 +211,7 @@ class ArchitectPhase: def __init__(self, complete, *, contracts: Sequence[str] = (), max_depth: int = 3, max_nodes: int = 12, root_path: str = "", - resume: bool = False): + resume: bool = False, workers: int = 1): self._complete = complete self._contracts = tuple(contracts) self._max_depth = max_depth @@ -224,6 +224,9 @@ def __init__(self, complete, *, contracts: Sequence[str] = (), #: all. The runner turns it on only for `--continue-from`, where a record that #: is there really is an earlier phase 1's work. self._resume = resume + #: How many siblings to design at once. Serial by default so a + #: run that did not ask for concurrency does not get it. + self._workers = max(1, int(workers)) #: Nodes it designed, and the deepest it went. self.nodes: List[str] = [] self.depth = 0 @@ -240,33 +243,73 @@ def __init__(self, complete, *, contracts: Sequence[str] = (), self.repeated = 0 def design(self, given: Mapping[str, str], objective: str) -> Dict[str, str]: - """``given`` plus one ``CONTEXT.md`` per node the architect decided on.""" + """``given`` plus one ``CONTEXT.md`` per node the architect decided on. + + Breadth first, and **one level at a time**: every node at a depth is asked + concurrently, then the answers are applied in order. Siblings are independent + by construction -- each inherits the chain down to its own parent, which was + designed a level ago -- so nothing in a level can depend on anything else in + it. Upstream an Architect *spawns* sub-architects, which is the same statement + about independence; running them one after another was this port's choice and + it cost the fly domain 32 minutes for 71 nodes. + + The asks read a **snapshot** rather than the live state, so a sibling can never + see another sibling's record even if it finishes first, and the tree does not + depend on which call returns when. + """ state = dict(given) - queue: List[Tuple[str, str, int]] = [(self._root, objective, 0)] - while queue and len(self.nodes) < self._max_nodes: - path, node_objective, depth = queue.pop(0) - key = f"{path}/{CONTEXT_FILE}" if path else CONTEXT_FILE - if self._resume and state.get(key): - # Already designed, by an earlier phase 1 that ran out of budget. - # Re-asking would spend a call to get the same answer and would throw - # away a record the run may already have grown code against; the thing - # that revises a record once code exists is the refinement architect, - # not this. So take the routing table as the design and walk on. - self.reused += 1 - self.nodes.append(path) - self.depth = max(self.depth, depth) - if depth + 1 <= self._max_depth: - # The routing line's right-hand side is the objective this node's - # architect handed that child. Passing `node_objective` down - # instead sends a leaf the whole project: measured, an architect - # asked to design `.../cell_types/mushroom_body` while carrying the - # root objective came back with `brain, learning, environment, - # simulation`, having redesigned the library from the top at depth - # five. - queue += [(normalise(child), handles or node_objective, depth + 1) - for child, handles in parse_routes(state[key])] - continue - record, children = self._ask(state, path, node_objective, depth) + level: List[Tuple[str, str, int]] = [(self._root, objective, 0)] + while level and len(self.nodes) < self._max_nodes: + room = self._max_nodes - len(self.nodes) + batch, level = level[:room], level[room:] + + # Resume is settled before anything is dispatched: a node an earlier phase 1 + # already designed must not cost a call, and deciding that after the ask + # spends exactly the call it exists to save. + kept: List[Tuple[str, str, int]] = [] + todo: List[Tuple[str, str, int]] = [] + for path, node_objective, depth in batch: + key = f"{path}/{CONTEXT_FILE}" if path else CONTEXT_FILE + if self._resume and state.get(key): + self.reused += 1 + self.nodes.append(path) + self.depth = max(self.depth, depth) + if depth + 1 <= self._max_depth: + # The routing line's right-hand side is the objective this + # node's architect handed that child. Passing `node_objective` + # down instead sends a leaf the whole project: measured, an + # architect asked to design `.../cell_types/mushroom_body` while + # carrying the root objective came back with `brain, learning, + # environment, simulation`, having redesigned the library from + # the top at depth five. + kept += [(normalise(child), handles or node_objective, depth + 1) + for child, handles in parse_routes(state[key])] + else: + todo.append((path, node_objective, depth)) + batch = todo + snapshot = dict(state) + if not batch: + replies = [] + elif self._workers > 1 and len(batch) > 1: + from concurrent.futures import ThreadPoolExecutor + with ThreadPoolExecutor(max_workers=self._workers) as pool: + replies = list(pool.map( + lambda item: self._ask(snapshot, item[0], item[1], item[2]), + batch)) + else: + replies = [self._ask(snapshot, path, node_objective, depth) + for path, node_objective, depth in batch] + level = kept + self._apply(state, batch, replies) + level + if level: + self.truncated = len(level) + return state + + def _apply(self, state: Dict[str, str], batch: Sequence[Tuple[str, str, int]], + replies: Sequence[Tuple[Optional[str], List[Dict[str, str]]]], + ) -> List[Tuple[str, str, int]]: + """Fold one level's answers into the tree, in order, and return the next level.""" + queue: List[Tuple[str, str, int]] = [] + for (path, node_objective, depth), (record, children) in zip(batch, replies): if record is None: continue key = f"{path}/{CONTEXT_FILE}" if path else CONTEXT_FILE @@ -321,9 +364,7 @@ def design(self, given: Mapping[str, str], objective: str) -> Dict[str, str]: # the children in both directions. See `_fix_routing` for the subtree that # went missing because it only ever agreed in one. state[key] = _fix_routing(record, accepted) - if queue: - self.truncated = len(queue) - return state + return queue def refine(self, state: Mapping[str, str], path: str, objective: str, reason: str) -> Optional[str]: diff --git a/examples/genesis/_fly.py b/examples/genesis/_fly.py index 26b2bfa3..595db2a8 100644 --- a/examples/genesis/_fly.py +++ b/examples/genesis/_fly.py @@ -42,11 +42,22 @@ "reward", "suite_failures", "suite_review"] -#: 需求和驱动。人写的全部，拒绝给任何提案，评分前恢复原样。 -FROZEN = ("REQUIREMENTS.md", "fly.py") - -#: 推进每一个 brief 的东西，按这个顺序。 -CONTRACTS = ("REQUIREMENTS.md", "fly.py") +#: 人写的全部，一个文件。拒绝给任何提案，评分前恢复原样。 +#: +#: `fly.py` 曾经也在这里，而它不该在。这个 port 自己的偏差表就是这么记的 —— +#: "a frozen entry point, so the result is a program rather than a package nobody can +#: invoke. **A choice, and a defensible one**" —— 一个选择，不是上游有的东西。上游 +#: Mode B 的输入只有一句 objective，c-testsuite / LLVM / Csmith 是实验者事后拿来量的 +#: 外部基准，不在仓库里。 +#: +#: 上游其实也有契约，只是隐含在目标里："造一个 C 编译器"这句话本身就钉死了 `cc -o foo +#: foo.c`，一个人人都知道的约定，所以 c-testsuite 能跑起来。而"一个果蝇模拟器"没有任何 +#: 公认的调用方式，所以那个约定得说出来 —— 但它属于**需求**（我要怎么用它），不属于 +#: 仓库（这是给你们的代码）。现在它在 `REQUIREMENTS.md` 的第四节里，`fly.py` 由它们写。 +FROZEN = ("REQUIREMENTS.md",) + +#: 推进每一个 brief 的东西。 +CONTRACTS = ("REQUIREMENTS.md",) SCORING = "黑盒验收（agent 读不到），外加一套开跑前封存的事后测量" CASE_NOUN = "验收断言" @@ -58,11 +69,12 @@ REQUIRES_MODEL = True OBJECTIVE = ( - "实现 `REQUIREMENTS.md` 要的那个软件：一个根在 `src/` 的 Python 包，把一只果蝇的大脑" - "照真实连接组建出来，让它在竞技场里执行任务、领取奖惩、并因此改变行为，再把整个东西" - "作为一个网页端出来。冻结的 `fly.py` 只 import `src` 的四个名字 —— `make_brain`、" - "`train`、`run_episode`、`handle` —— 那四个就是公开接口。`src/` 以下怎么分层、每层叫" - "什么、谁调用谁，全部由你们决定。可以用 numpy 和其它第三方包。" + "实现 `REQUIREMENTS.md` 要的那个软件：把一只果蝇的大脑照真实连接组建出来，让它在竞技" + "场里执行任务、领取奖惩、并因此改变行为，再把整个东西作为一个网页端出来。库放在 " + "`src/` 下面。仓库根目录" + "下要有一个 `fly.py`，能跑 `train`、`probe`、`serve` 三个子命令，三个都认 " + "`--format json`；那个文件也是你们写的。除此之外 —— 分几层、每层叫什么、谁调用谁、" + "公开接口长什么样 —— 全部由你们决定。可以用 numpy 和其它第三方包。" "**每个目录都要为自己的文件写测试**：测试是“做完了”的定义，一个写了实现却没有测试的" "节点，它的上级没有任何东西可以运行来验收它。" ) @@ -82,269 +94,395 @@ #: 黑盒验收，驱动搜索。**不在仓库里** —— 只在一次评分的临时副本里出现。 _ACCEPTANCE = { - 'acceptance/test_api.py': r'''"""用户验收：只通过驱动用到的四个名字，黑盒地问"我要的东西能用吗"。 + 'acceptance/_cli.py': r'''"""跑那个程序，看它说了什么。没有一行 import 它们写的模块。 -这些断言从不进入仓库。agent 看到的是它们的名字和报错，看不到一行源码 —— -仓库里的每一个测试都是 agent 自己写的。 +上游的验证就是这个形状 —— 跑二进制、检查输出（c-testsuite / LLVM / Csmith）。这里唯一的 +契约是 `REQUIREMENTS.md` 里那三条命令行，`src/` 底下怎么分层完全不关这些断言的事。 """ import json - -from src import handle, make_brain, run_episode, train +import subprocess +import sys +import time +import urllib.error +import urllib.request TASKS = ("taxis", "avoid", "choice") - - -def rows(task, episodes=4, seed=0): - return train(task, episodes, seed)["episodes"] - - -def test_the_api_reports_where_the_animal_is_and_what_its_brain_is_doing(): - status, content_type, body = handle("GET", "/api/state") - assert status == 200 and "json" in content_type - state = json.loads(body) - assert "position" in state and len(state["position"]) == 2 - assert isinstance(state.get("brain"), dict) and len(state["brain"]) >= 4 - - -def test_the_api_can_step_reset_and_train(): - handle("POST", "/api/reset", json.dumps({"task": "avoid", "seed": 0})) - before = json.loads(handle("GET", "/api/state")[2])["step"] - handle("POST", "/api/step", json.dumps({"steps": 10})) - after = json.loads(handle("GET", "/api/state")[2])["step"] - assert after > before - status, _, _ = handle("POST", "/api/train", json.dumps({"task": "avoid", - "episodes": 2})) - assert status == 200 - assert json.loads(handle("POST", "/api/reset", - json.dumps({"task": "avoid", "seed": 0}))[2])["step"] == 0 - - -def test_an_unknown_route_is_refused_and_a_bad_body_is_too(): - assert handle("GET", "/api/nothing-here")[0] == 404 - assert handle("POST", "/api/step", "{not json")[0] == 400 +TIMEOUT = 300 + + +def run(*args, timeout=TIMEOUT): + """跑 `python fly.py ...`，返回 (returncode, stdout, stderr)。""" + done = subprocess.run([sys.executable, "fly.py", *args], + capture_output=True, text=True, timeout=timeout) + return done.returncode, done.stdout, done.stderr + + +def json_run(*args, timeout=TIMEOUT): + """同上，但要求 `--format json` 的输出是一个能解析的 JSON，且 stdout 只有它。""" + code, out, err = run(*args, "--format", "json", timeout=timeout) + assert code == 0, f"fly.py {' '.join(args)} 退出 {code}\n{err[-600:]}" + return json.loads(out.strip()) + + +def serving(port, *, timeout=40): + """起 `serve`，等它能应答，交出一个取 URL 的函数。""" + proc = subprocess.Popen([sys.executable, "fly.py", "serve", "--port", str(port)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + base = f"http://127.0.0.1:{port}" + deadline = time.time() + timeout + while time.time() < deadline: + if proc.poll() is not None: + raise AssertionError(f"serve 退出了 {proc.returncode}: " + f"{(proc.stderr.read() or '')[-600:]}") + try: + urllib.request.urlopen(base + "/", timeout=2).read() + return proc, base + except urllib.error.HTTPError: + return proc, base + except Exception: + time.sleep(0.4) + proc.kill() + raise AssertionError(f"serve 在 {timeout}s 内没有应答") + + +def get(base, path): + try: + with urllib.request.urlopen(base + path, timeout=10) as r: + return r.status, r.headers.get("content-type", ""), r.read().decode() + except urllib.error.HTTPError as e: + return e.code, e.headers.get("content-type", ""), e.read().decode() + + +def post(base, path, payload): + req = urllib.request.Request(base + path, method="POST", + data=json.dumps(payload).encode(), + headers={"content-type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return r.status, r.read().decode() + except urllib.error.HTTPError as e: + return e.code, e.read().decode() ''', - 'acceptance/test_learning.py': r'''"""用户验收：只通过驱动用到的四个名字，黑盒地问"我要的东西能用吗"。 - -这些断言从不进入仓库。agent 看到的是它们的名字和报错，看不到一行源码 —— -仓库里的每一个测试都是 agent 自己写的。 -""" + 'acceptance/test_api.py': r'''"""需求 三：后端要能取状态、单步、重置、训练。""" import json -from src import handle, make_brain, run_episode, train +from _cli import get, post, serving -TASKS = ("taxis", "avoid", "choice") +def test_the_api_reports_where_the_animal_is_and_what_its_brain_is_doing(): + proc, base = serving(8733) + try: + status, content_type, body = get(base, "/api/state") + assert status == 200 and "json" in content_type + state = json.loads(body) + assert "position" in state and len(state["position"]) == 2 + assert isinstance(state.get("brain"), dict) and len(state["brain"]) >= 3 + finally: + proc.kill() + + +def test_the_api_can_step_and_reset(): + proc, base = serving(8734) + try: + post(base, "/api/reset", {"task": "avoid", "seed": 0}) + before = json.loads(get(base, "/api/state")[2])["step"] + post(base, "/api/step", {"steps": 10}) + after = json.loads(get(base, "/api/state")[2])["step"] + assert after > before + post(base, "/api/reset", {"task": "avoid", "seed": 0}) + assert json.loads(get(base, "/api/state")[2])["step"] == 0 + finally: + proc.kill() + + +def test_the_api_can_train(): + proc, base = serving(8735) + try: + status, _ = post(base, "/api/train", {"task": "avoid", "episodes": 3}) + assert status == 200 + finally: + proc.kill() + + +def test_an_unknown_route_is_refused(): + proc, base = serving(8736) + try: + assert get(base, "/api/nothing-here")[0] == 404 + finally: + proc.kill() +''', + 'acceptance/test_learning.py': r'''"""需求 二：学习必须改变行为，而且要能被看见。""" -def rows(task, episodes=4, seed=0): - return train(task, episodes, seed)["episodes"] +from _cli import json_run def test_a_fresh_animal_is_punished_in_the_avoidance_assay(): """没学过的动物会一头撞进惩罚区 —— 否则这个实验什么也测不到。""" - naive = make_brain({"seed": 0}) - assert sum(run_episode(naive, "avoid", 1000 + i, learning=False)["shocks"] - for i in range(3)) > 0 + result = json_run("probe", "--task", "avoid", "--episodes", "1", "--trials", "3") + assert result["naive"] > 0 def test_training_reduces_the_punishment_the_animal_takes(): - shocks = [r["shocks"] for r in rows("avoid", 12)] + rows = json_run("train", "--task", "avoid", "--episodes", "12") + shocks = [r["shocks"] for r in rows] assert sum(shocks[:3]) > 0 assert sum(shocks[-3:]) < sum(shocks[:3]) def test_what_was_learned_transfers_to_arenas_it_never_saw(): """关掉可塑性、用没训练过的种子 —— 唯一诚实的学习度量。""" - trained = train("avoid", 10, 0)["brain"] - naive = make_brain({"seed": 0}) - unseen = range(2000, 2005) - before = sum(run_episode(naive, "avoid", s, learning=False)["shocks"] for s in unseen) - after = sum(run_episode(trained, "avoid", s, learning=False)["shocks"] for s in unseen) - assert before > 0 and after < before - - -def test_an_episode_with_learning_off_leaves_the_animal_unchanged(): - brain = make_brain({"seed": 0}) - for s in range(3): - run_episode(brain, "avoid", s, learning=False) - fresh = make_brain({"seed": 0}) - before = [run_episode(fresh, "avoid", 3000 + i, learning=False)["shocks"] - for i in range(3)] - after = [run_episode(brain, "avoid", 3000 + i, learning=False)["shocks"] - for i in range(3)] - assert before == after -''', - 'acceptance/test_page.py': r'''"""用户验收：只通过驱动用到的四个名字，黑盒地问"我要的东西能用吗"。 + result = json_run("probe", "--task", "avoid", "--episodes", "10", "--trials", "5") + assert result["naive"] > 0 + assert result["trained"] < result["naive"] -这些断言从不进入仓库。agent 看到的是它们的名字和报错，看不到一行源码 —— -仓库里的每一个测试都是 agent 自己写的。 -""" - -import json - -from src import handle, make_brain, run_episode, train - -TASKS = ("taxis", "avoid", "choice") +def test_learning_shows_up_in_the_choice_assay_as_well(): + result = json_run("probe", "--task", "choice", "--episodes", "10", "--trials", "5") + assert result["trained"] <= result["naive"] +''', + 'acceptance/test_page.py': r'''"""需求 三：一个网页，打开就能看。""" -def rows(task, episodes=4, seed=0): - return train(task, episodes, seed)["episodes"] +from _cli import get, serving def test_the_page_is_served_at_the_root(): - status, content_type, body = handle("GET", "/") - assert status == 200 and "html" in content_type.lower() - assert body.lstrip().lower().startswith("= 0 + assert row["shocks"] >= 0 assert isinstance(row["reward"], (int, float)) assert row["steps"] > 0 -def test_training_returns_the_animal_it_trained(): - result = train("avoid", 3, 0) - assert result["brain"] is not None - assert run_episode(result["brain"], "avoid", 7, learning=False)["shocks"] >= 0 +def test_the_same_seed_gives_the_same_run(): + one = json_run("train", "--task", "avoid", "--episodes", "4", "--seed", "0") + two = json_run("train", "--task", "avoid", "--episodes", "4", "--seed", "0") + assert [r["shocks"] for r in one] == [r["shocks"] for r in two] -def test_the_same_seed_gives_the_same_run(): - assert [r["shocks"] for r in rows("avoid", 5)] == [r["shocks"] for r in rows("avoid", 5)] +def test_json_output_is_only_json(): + """我要拿它画曲线，所以 stdout 上不能混别的东西。""" + code, out, err = run("train", "--task", "taxis", "--episodes", "1", "--format", "json") + assert code == 0 + assert out.strip().startswith(("[", "{")) ''', } #: 封存的事后测量。开跑前写死，全程不进仓库，跑完只用来报一个数。 _SEALED = { - 'sealed/test_sealed.py': r'''"""封存的事后测量：写在开跑之前，全程不进仓库，任何 agent 读不到，跑完只用来报一个数。 + 'sealed/_cli.py': r'''"""跑那个程序，看它说了什么。没有一行 import 它们写的模块。 -这是上游的协议 —— Genesis 拿 c-testsuite、LLVM、Csmith 验证，那些是实验者事后量的外部 -基准，不是 agent 的内循环。它们和 `acceptance/` 问的是同一批要求，但换了问法、换了种子、 -换了任务，所以"验收全过"和"封存套件全过"之间的差距，就是它把东西做对了还是只对着验收写的 -度量。 - -同样只用驱动那四个名字。有一条要求是这套测不了的：**大脑是不是真的照连接组建的**。黑盒 -问不出来，只能靠父节点评审读 diff、和人读结果。这条限制写在这里，不藏着。 +上游的验证就是这个形状 —— 跑二进制、检查输出（c-testsuite / LLVM / Csmith）。这里唯一的 +契约是 `REQUIREMENTS.md` 里那三条命令行，`src/` 底下怎么分层完全不关这些断言的事。 """ import json +import subprocess +import sys +import time +import urllib.error +import urllib.request + +TASKS = ("taxis", "avoid", "choice") +TIMEOUT = 300 + + +def run(*args, timeout=TIMEOUT): + """跑 `python fly.py ...`，返回 (returncode, stdout, stderr)。""" + done = subprocess.run([sys.executable, "fly.py", *args], + capture_output=True, text=True, timeout=timeout) + return done.returncode, done.stdout, done.stderr + + +def json_run(*args, timeout=TIMEOUT): + """同上，但要求 `--format json` 的输出是一个能解析的 JSON，且 stdout 只有它。""" + code, out, err = run(*args, "--format", "json", timeout=timeout) + assert code == 0, f"fly.py {' '.join(args)} 退出 {code}\n{err[-600:]}" + return json.loads(out.strip()) + + +def serving(port, *, timeout=40): + """起 `serve`，等它能应答，交出一个取 URL 的函数。""" + proc = subprocess.Popen([sys.executable, "fly.py", "serve", "--port", str(port)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + base = f"http://127.0.0.1:{port}" + deadline = time.time() + timeout + while time.time() < deadline: + if proc.poll() is not None: + raise AssertionError(f"serve 退出了 {proc.returncode}: " + f"{(proc.stderr.read() or '')[-600:]}") + try: + urllib.request.urlopen(base + "/", timeout=2).read() + return proc, base + except urllib.error.HTTPError: + return proc, base + except Exception: + time.sleep(0.4) + proc.kill() + raise AssertionError(f"serve 在 {timeout}s 内没有应答") + + +def get(base, path): + try: + with urllib.request.urlopen(base + path, timeout=10) as r: + return r.status, r.headers.get("content-type", ""), r.read().decode() + except urllib.error.HTTPError as e: + return e.code, e.headers.get("content-type", ""), e.read().decode() + + +def post(base, path, payload): + req = urllib.request.Request(base + path, method="POST", + data=json.dumps(payload).encode(), + headers={"content-type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return r.status, r.read().decode() + except urllib.error.HTTPError as e: + return e.code, e.read().decode() +''', + 'sealed/test_sealed.py': r'''"""封存的事后测量：开跑前写死，全程不进仓库，任何 agent 读不到，跑完只用来报一个数。 + +上游就是这个协议 —— c-testsuite / LLVM / Csmith 是实验者事后拿来量的外部基准，从不进 +agent 的内循环。这套问的是和验收同一批要求，但换了种子、换了任务、换了问法。"验收全过" +和"这套全过"之间的差距，就是它把东西做出来了、还是只对着验收写的度量。 -from src import handle, make_brain, run_episode, train +有一条这套测不了，写在这里不藏着：**大脑是不是真的照连接组建的**。跑二进制问不出来，只能 +靠父节点读 diff 和人读结果。 +""" +import json -def test_learning_shows_up_in_the_choice_assay_too(): - trained = train("choice", 10, 1)["brain"] - naive = make_brain({"seed": 1}) - unseen = range(4000, 4006) - before = sum(run_episode(naive, "choice", s, learning=False)["shocks"] for s in unseen) - after = sum(run_episode(trained, "choice", s, learning=False)["shocks"] for s in unseen) - assert before > 0 and after < before +from _cli import get, json_run, post, run, serving def test_the_gain_from_training_survives_a_different_seed(): - trained = train("avoid", 10, 5)["brain"] - naive = make_brain({"seed": 5}) - unseen = range(5000, 5006) - before = sum(run_episode(naive, "avoid", s, learning=False)["shocks"] for s in unseen) - after = sum(run_episode(trained, "avoid", s, learning=False)["shocks"] for s in unseen) - assert after < 0.6 * before + result = json_run("probe", "--task", "avoid", "--episodes", "10", + "--trials", "6", "--seed", "5") + assert result["naive"] > 0 + assert result["trained"] < 0.6 * result["naive"] def test_more_training_is_not_worse_than_less(): - short = train("avoid", 4, 2)["brain"] - long = train("avoid", 16, 2)["brain"] - unseen = range(6000, 6006) - a = sum(run_episode(short, "avoid", s, learning=False)["shocks"] for s in unseen) - b = sum(run_episode(long, "avoid", s, learning=False)["shocks"] for s in unseen) - assert b <= a + short = json_run("probe", "--task", "avoid", "--episodes", "4", "--trials", "5", "--seed", "2") + long = json_run("probe", "--task", "avoid", "--episodes", "16", "--trials", "5", "--seed", "2") + assert long["trained"] <= short["trained"] + + +def test_the_taxis_assay_is_actually_reachable(): + """趋向实验要真的能到达 —— 每局都跑满上限说明动物根本没找到源。""" + rows = json_run("train", "--task", "taxis", "--episodes", "6") + steps = [r["steps"] for r in rows] + assert min(steps) < max(steps) or min(steps) < 400 + + +def test_a_different_seed_is_a_different_animal(): + one = json_run("train", "--task", "avoid", "--episodes", "4", "--seed", "11") + two = json_run("train", "--task", "avoid", "--episodes", "4", "--seed", "22") + assert [r["shocks"] for r in one] != [r["shocks"] for r in two] -def test_an_animal_trained_on_one_task_still_works_on_another(): +def test_training_on_one_task_does_not_break_another(): """不应该为了通过一个实验把另一个弄坏。""" - trained = train("avoid", 10, 3)["brain"] - assert run_episode(trained, "taxis", 7000, learning=False)["steps"] > 0 + rows = json_run("train", "--task", "choice", "--episodes", "3", "--seed", "7") + assert all(r["steps"] > 0 for r in rows) -def test_two_animals_with_different_seeds_are_not_the_same_animal(): - one = [run_episode(make_brain({"seed": 11}), "avoid", 8000 + i, learning=False)["shocks"] - for i in range(4)] - two = [run_episode(make_brain({"seed": 22}), "avoid", 8000 + i, learning=False)["shocks"] - for i in range(4)] - assert one != two +def test_every_task_survives_a_longer_run(): + for task in ("taxis", "avoid", "choice"): + rows = json_run("train", "--task", task, "--episodes", "8", "--seed", "3") + assert len(rows) == 8, task -def test_the_taxis_assay_ends_sooner_than_it_times_out(): - """趋向实验要真的能到达 —— 每局都跑满上限说明动物根本没找到源。""" - steps = [r["steps"] for r in train("taxis", 6, 0)["episodes"]] - assert min(steps) < max(steps) or min(steps) < 400 +def test_a_bad_task_name_is_refused_rather_than_pretended(): + code, out, err = run("train", "--task", "fly-to-the-moon", "--episodes", "1") + assert code != 0 def test_the_state_the_page_draws_changes_as_the_animal_moves(): - handle("POST", "/api/reset", json.dumps({"task": "taxis", "seed": 0})) - first = json.loads(handle("GET", "/api/state")[2]) - handle("POST", "/api/step", json.dumps({"steps": 25})) - later = json.loads(handle("GET", "/api/state")[2]) - assert first["position"] != later["position"] + proc, base = serving(8741) + try: + post(base, "/api/reset", {"task": "taxis", "seed": 0}) + first = json.loads(get(base, "/api/state")[2]) + post(base, "/api/step", {"steps": 25}) + later = json.loads(get(base, "/api/state")[2]) + assert first["position"] != later["position"] + finally: + proc.kill() def test_the_page_exposes_more_than_one_layer_of_the_brain(): """需求里写了"要能看见大脑各层此刻的活动"，所以状态里不能只有一个标量。""" - handle("POST", "/api/reset", json.dumps({"task": "avoid", "seed": 0})) - handle("POST", "/api/step", json.dumps({"steps": 5})) - brain = json.loads(handle("GET", "/api/state")[2])["brain"] - populations = [k for k, v in brain.items() if isinstance(v, (list, dict)) and v] - assert len(populations) >= 3, sorted(brain) - - -def test_a_new_brain_through_the_api_has_forgotten_everything(): - handle("POST", "/api/reset", json.dumps({"task": "avoid", "seed": 0, "brain": "new"})) - handle("POST", "/api/train", json.dumps({"task": "avoid", "episodes": 6})) - handle("POST", "/api/reset", json.dumps({"task": "avoid", "seed": 0, "brain": "new"})) - handle("POST", "/api/step", json.dumps({"steps": 1})) - fresh = json.loads(handle("GET", "/api/state")[2]) - assert fresh["step"] >= 1 - - -def test_the_driver_is_reproducible_across_two_calls_in_one_process(): - one = [r["reward"] for r in train("choice", 5, 9)["episodes"]] - two = [r["reward"] for r in train("choice", 5, 9)["episodes"]] - assert one == two + proc, base = serving(8742) + try: + post(base, "/api/step", {"steps": 5}) + brain = json.loads(get(base, "/api/state")[2])["brain"] + populations = [k for k, v in brain.items() if isinstance(v, (list, dict)) and v] + assert len(populations) >= 3, sorted(brain) + finally: + proc.kill() + + +def test_the_server_says_where_it_is_listening(): + """`serve --format json` 要打出一行 URL，否则脚本没法自动接上它。""" + import subprocess, sys, time + proc = subprocess.Popen([sys.executable, "fly.py", "serve", "--port", "8743", + "--format", "json"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + deadline = time.time() + 30 + line = "" + while time.time() < deadline and not line.strip(): + line = proc.stdout.readline() + if proc.poll() is not None: + break + assert "8743" in line and json.loads(line.strip()).get("url") + finally: + proc.kill() ''', } -#: 用户的要求，原话。 +#: 用户的要求，原话。人写的唯一一个文件。 _REQUIREMENTS = r'''# 需求：一个果蝇大脑的模拟软件 我要一个软件，把一只果蝇（*Drosophila melanogaster*）建模出来：它的大脑、它的身体、 @@ -388,19 +526,34 @@ def test_the_driver_is_reproducible_across_two_calls_in_one_process(): 手机宽度下要能用，跟随系统的明暗配色。不要依赖 CDN，不要在页面里访问外网。 -## 四、怎么算做完 +## 四、我要怎么用它 -`fly.py` 是冻结的，你们不能改它。它就是这份需求的可执行形式 —— 我要的是一个**能跑的程序**， -不是一个没人能调用的包： +我要的是一个**能跑的程序**，不是一个没人能调用的包。仓库根目录下要有一个 `fly.py`， +我用它就能做完下面三件事 —— 这个文件也是你们写的，我只说我要怎么用： ``` -python fly.py train --task avoid --episodes 20 # 训练，打出每集的电击数 -python fly.py probe --task avoid --trials 5 # 关掉可塑性，朴素 vs 训练过 -python fly.py serve --port 8000 # 起服务，浏览器打开 +python fly.py train --task avoid --episodes 20 +python fly.py probe --task avoid --episodes 10 --trials 5 +python fly.py serve --port 8000 ``` -它只通过 `src` 这一个包调用你们写的东西。`fly.py` 里 import 了哪些名字，那些就是公开接口， -其余全是你们的内部事务。 +**train** 跑若干局并把每一局打出来：第几局、挨了多少次惩罚、拿了多少回报、走了多少步。 + +**probe** 训练一只，再把它和一只没学过的放进同样的、没训练过的竞技场，**关掉可塑性**， +各跑几局，报出两边各挨了多少惩罚。这是唯一诚实的学习度量：同样的环境、没见过的种子、 +测试期间什么都不改变。 + +**serve** 起一个 HTTP 服务，浏览器打开就能看。 + +三个子命令都要认 `--format json`：这时候**只往 stdout 打一个 JSON**，别的什么都不打。 +我要拿它画学习曲线、做对比，所以得是机器能读的： + +- `train --format json` → 一个数组，每局一个对象，至少含 `episode`（从 0 数）、 + `shocks`、`reward`、`steps` +- `probe --format json` → 一个对象，至少含 `naive` 和 `trained`，各是那一边挨的总惩罚数 +- `serve --format json` → 打出一行 `{"url": "http://..."}` 然后继续服务 + +任务名至少要有 `taxis`、`avoid`、`choice` 三个。 ## 五、工程要求 @@ -438,134 +591,9 @@ def test_the_driver_is_reproducible_across_two_calls_in_one_process(): - Westeinde 等，*Transforming a head direction signal into a goal-oriented steering command*，Nature 2024 ''' -#: 那份需求的可执行形式。冻结：它是需求，不是实现。 -_DRIVER = r'''#!/usr/bin/env python3 -"""fly -- 一只果蝇，一个大脑，和一个能看着它学习的网页。 - -这个文件是**冻结的**：没有 agent 可以写它。它是 `REQUIREMENTS.md` 的可执行形式 —— -我要的是一个能跑的程序，而不是一个没人能调用的包。 - - python fly.py train --task avoid --episodes 20 - python fly.py probe --task avoid --episodes 10 --trials 5 - python fly.py serve --port 8000 - -它只通过 `src` 这一个包调用你们写的东西，并且**只用到下面四个名字**。其余一切 —— -分几层、每层叫什么、谁调用谁、写哪些测试 —— 都是你们的内部事务。 - - make_brain(config=None) -> brain - 一只没有任何经验的动物。`config` 是个字典，至少认得 `seed`。 - - train(task, episodes, seed) -> {"episodes": [...], "brain": brain} - 跑 `episodes` 局，每局一行，按顺序放在 "episodes" 里。每行至少要有 - `episode`（从 0 数）、`shocks`（这一局挨了多少次惩罚）、`reward`、`steps`。 - "brain" 是训练完的那只动物。 - - run_episode(brain, task, seed, learning=True) -> {"shocks": ..., ...} - 用给定的动物跑一局。`learning=False` 时不得改变它 —— 这是唯一能诚实 - 比较"学过"和"没学过"的办法。 - - handle(method, path, body=None) -> (status, content_type, body) - HTTP 的全部逻辑，一个纯函数，不开 socket。下面那层 http.server 是我写的。 - `GET /` 必须返回那个网页。 - -任务名至少要有 `taxis`、`avoid`、`choice` 三个。 -""" - -import argparse -import sys - - -def cmd_train(args): - from src import train - rows = train(args.task, args.episodes, args.seed)["episodes"] - print(f"{'ep':>4} {'shocks':>7} {'reward':>9} {'steps':>6}") - for row in rows: - print(f"{row['episode']:>4} {row['shocks']:>7} {row['reward']:>9.2f} " - f"{row['steps']:>6}") - head = sum(r["shocks"] for r in rows[:3]) / max(1, len(rows[:3])) - tail = sum(r["shocks"] for r in rows[-3:]) / max(1, len(rows[-3:])) - print(f"\n前三局平均 {head:.1f} 次惩罚，后三局 {tail:.1f}") - return 0 - - -def cmd_probe(args): - """训练一只，然后把它和一只没学过的放在同样的、没见过的竞技场里，关掉可塑性。 - - 这是唯一诚实的学习度量：同样的环境、没训练过的种子、测试期间什么都不改变。 - """ - from src import make_brain, run_episode, train - trained = train(args.task, args.episodes, args.seed)["brain"] - naive = make_brain({"seed": args.seed}) - totals = {} - for label, brain in (("naive", naive), ("trained", trained)): - shocks = [run_episode(brain, args.task, 1000 + i, learning=False)["shocks"] - for i in range(args.trials)] - totals[label] = sum(shocks) - print(f"{label:>8}: {sum(shocks):>5} 次惩罚 / {args.trials} 局 {shocks}") - print(f"\n学习带来的减少：{totals['naive'] - totals['trained']}") - return 0 - - -def cmd_serve(args): - from http.server import BaseHTTPRequestHandler, HTTPServer - - from src import handle - - class Handler(BaseHTTPRequestHandler): - def _respond(self, method): - length = int(self.headers.get("content-length") or 0) - body = self.rfile.read(length).decode("utf-8") if length else None - status, content_type, payload = handle(method, self.path, body) - data = payload.encode("utf-8") if isinstance(payload, str) else payload - self.send_response(status) - self.send_header("content-type", content_type) - self.send_header("content-length", str(len(data))) - self.end_headers() - self.wfile.write(data) - - def do_GET(self): - self._respond("GET") - - def do_POST(self): - self._respond("POST") - - def log_message(self, *a): - pass - - server = HTTPServer((args.host, args.port), Handler) - print(f"http://{args.host}:{args.port}/") - sys.stdout.flush() - server.serve_forever() - return 0 - - -def main(argv=None): - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - sub = parser.add_subparsers(dest="cmd", required=True) - for name, fn in (("train", cmd_train), ("probe", cmd_probe), ("serve", cmd_serve)): - p = sub.add_parser(name) - p.set_defaults(fn=fn) - if name in ("train", "probe"): - p.add_argument("--task", default="avoid") - p.add_argument("--episodes", type=int, default=20) - p.add_argument("--seed", type=int, default=0) - if name == "probe": - p.add_argument("--trials", type=int, default=5) - if name == "serve": - p.add_argument("--host", default="127.0.0.1") - p.add_argument("--port", type=int, default=8000) - args = parser.parse_args(argv) - return args.fn(args) - - -if __name__ == "__main__": - raise SystemExit(main()) -''' - - def initial_files() -> Dict[str, str]: - """整个仓库的起点：一份需求、一个驱动、一份写模块的须知。没有测试，没有记录。""" - return {"REQUIREMENTS.md": _REQUIREMENTS, "fly.py": _DRIVER, + """整个仓库的起点：一份需求，和一份写模块的须知。没有驱动，没有测试，没有记录。""" + return {"REQUIREMENTS.md": _REQUIREMENTS, f"src/{SKILLS_DIR}/python-modules.md": PYTHON_MODULE_SKILL} diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index aa984f11..f0ccb0ca 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -498,7 +498,12 @@ def main(argv=None) -> None: root_path=root, # A phase 1 that ran out of budget is worth # continuing rather than paying for twice. - resume=bool(args.continue_from)) + resume=bool(args.continue_from), + # Siblings are independent by construction, and + # upstream *spawns* sub-architects rather than + # running them one after another. Serial cost the + # fly domain 32 minutes for 71 nodes. + workers=args.workers) initial = architect.design(initial, objective_for(args.domain)) print(f"\nPhase 1 : architect {architect.summary()}") for path in architect.nodes: diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index 70dbd249..5f5278b0 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -11,6 +11,7 @@ import json import posixpath +import time import pytest @@ -1692,6 +1693,50 @@ def test_a_node_may_not_be_named_after_one_of_its_own_ancestors(): assert parse_routing(tree["src/a/" + CONTEXT_FILE]) == ["src/a/b"] +def test_phase_one_designs_a_level_at_a_time_and_siblings_cannot_see_each_other(): + """Upstream an Architect *spawns* sub-architects, which is a statement about + independence: every node at a depth inherits the chain down to its own parent, + designed a level ago, so nothing in a level can depend on anything else in it. + Running them one after another was this port's choice, and it cost the fly domain + 32 minutes for 71 nodes. + + The asks read a snapshot rather than the live tree, so a sibling can never see + another sibling's record even if it finishes first and the result does not depend + on which call returns when. + """ + import threading + + seen, together, lock = [], [], threading.Lock() + live = [0] + plan = {"": {"record": "# root\n", "children": [{"path": "a", "objective": "A"}, + {"path": "b", "objective": "B"}, + {"path": "c", "objective": "C"}]}} + + def complete(prompt): + with lock: + live[0] += 1 + together.append(live[0]) + seen.append(prompt) + time.sleep(0.05) + with lock: + live[0] -= 1 + for path, reply in plan.items(): + if f"repository path `{path or './'}`" in prompt: + return json.dumps(reply) + return json.dumps({"record": "# leaf\n", "children": []}) + + phase = ArchitectPhase(complete, max_depth=2, max_nodes=10, workers=3) + phase.design({}, "o") + assert len(phase.nodes) == 4 # root and three children + assert max(together) >= 2, "the three siblings were designed one after another" + # No sibling saw another's record: each child's prompt carries the root's chain + # and nothing from `a`, `b` or `c`. + children = [p for p in seen if "repository path `./`" not in p] + assert len(children) == 3 + for prompt in children: + assert sum(prompt.count(f"`{n}/") for n in "abc") <= 1 + + def test_a_resumed_phase_one_hands_each_child_its_own_objective(): """The routing line's right-hand side is the objective, not decoration. From 64f98317cde354440bfbe34b26aa66a7337416ac Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Sep 2026 14:06:01 +0000 Subject: [PATCH 11/42] Genesis: an executor session runs the model the run asked for A Claude Code episode goes through the local CLI whatever `--provider` says, and the CLI was handed no model at all unless the provider happened to be `claude-cli` -- so a run whose architect was on an API model had every executor session on the CLI's default instead, two models, in silence. It now takes the run's model, and `--executor-model` is there for the case where they should deliberately differ. The CLI reads ANTHROPIC_BASE_URL and ANTHROPIC_API_KEY from the environment, so pointing the SDK and the CLI at one endpoint is all it takes to put both halves of a run on the same model. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/genesis/genesis_recursive_worlds.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index f0ccb0ca..565b7695 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -276,6 +276,11 @@ def build_parser() -> argparse.ArgumentParser: "domain's empty one. With --mode a this is the Context " "Tree extraction upstream runs on an existing codebase; " "the frozen contract is always the domain's own") + parser.add_argument("--executor-model", default="", + help="the model an --executor claude-code session runs. The " + "sessions go through the local CLI whatever --provider " + "says, so with an API provider the architect and every " + "episode are otherwise on different models by default") parser.add_argument("--no-refine", action="store_true", help="do not re-spawn an architect on a node whose record has " "drifted from its files (upstream's architect Phase 3)") @@ -535,13 +540,16 @@ def main(argv=None) -> None: # The failure template is the domain's, not a constant: a blind domain must # not hand the session the assertion's source, which is the whole point of it. # And the session runs the model the run asked for -- left unset it takes the - # CLI's default, which on a `--provider claude-cli` run means the architect is - # on one model and every executor episode on another. + # CLI's default -- a different model from the one the run asked for, silently. + # The CLI reads ANTHROPIC_BASE_URL and ANTHROPIC_API_KEY from the environment, + # so pointing both the SDK and the CLI at one endpoint is all it takes to put + # the architect and every executor session on the same model; the flag is there + # for the case where they should deliberately differ. sessions = ClaudeCodeExecutor(frozen=spec.FROZEN, failure=getattr(spec, "FAILURE", TEST_FAILURE), - model=(args.model if args.provider == "claude-cli" - else None), + model=args.executor_model or args.model, max_turns=args.max_turns) + def _refine(path, state): """Upstream's architect Phase 3, as a hook on the accountability pass. From cbe49aa9350c9b27b43ab6b7794b606636993f3a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Sep 2026 14:15:03 +0000 Subject: [PATCH 12/42] Genesis: phase 1 says what it is doing while it does it It printed nothing until the whole phase finished. On the fly domain that was 82 minutes of silence, and answering "how far in is it" meant parsing the Claude CLI's own session log out of ~/.claude/projects. Then the architect moved to the SDK, the subprocess and its log went away, and the question had no answer at all -- six sockets open and a thread count. Each node now prints as it lands, with the running count against the budget. `--quiet-phase1` turns it off. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/genesis/_architect.py | 10 +++++++++- examples/genesis/genesis_recursive_worlds.py | 9 ++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/examples/genesis/_architect.py b/examples/genesis/_architect.py index 027afc75..5b624dfd 100644 --- a/examples/genesis/_architect.py +++ b/examples/genesis/_architect.py @@ -211,7 +211,7 @@ class ArchitectPhase: def __init__(self, complete, *, contracts: Sequence[str] = (), max_depth: int = 3, max_nodes: int = 12, root_path: str = "", - resume: bool = False, workers: int = 1): + resume: bool = False, workers: int = 1, on_node=None): self._complete = complete self._contracts = tuple(contracts) self._max_depth = max_depth @@ -227,6 +227,12 @@ def __init__(self, complete, *, contracts: Sequence[str] = (), #: How many siblings to design at once. Serial by default so a #: run that did not ask for concurrency does not get it. self._workers = max(1, int(workers)) + #: Called with `(path, routes, designed_so_far)` as each node lands. Phase 1 + #: printed nothing until the whole phase finished, which on the fly domain was + #: 82 minutes of silence -- and once the architect went through the SDK rather + #: than the CLI there was no session log to read either, so "how far in is it" + #: had no answer at all. + self._on_node = on_node #: Nodes it designed, and the deepest it went. self.nodes: List[str] = [] self.depth = 0 @@ -364,6 +370,8 @@ def _apply(self, state: Dict[str, str], batch: Sequence[Tuple[str, str, int]], # the children in both directions. See `_fix_routing` for the subtree that # went missing because it only ever agreed in one. state[key] = _fix_routing(record, accepted) + if self._on_node is not None: + self._on_node(path, len(accepted), len(self.nodes)) return queue def refine(self, state: Mapping[str, str], path: str, objective: str, diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index 565b7695..cb55fe96 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -281,6 +281,8 @@ def build_parser() -> argparse.ArgumentParser: "sessions go through the local CLI whatever --provider " "says, so with an API provider the architect and every " "episode are otherwise on different models by default") + parser.add_argument("--quiet-phase1", action="store_true", + help="do not print each node as phase 1 designs it") parser.add_argument("--no-refine", action="store_true", help="do not re-spawn an architect on a node whose record has " "drifted from its files (upstream's architect Phase 3)") @@ -508,7 +510,12 @@ def main(argv=None) -> None: # upstream *spawns* sub-architects rather than # running them one after another. Serial cost the # fly domain 32 minutes for 71 nodes. - workers=args.workers) + workers=args.workers, + on_node=(None if args.quiet_phase1 else + lambda path, routes, n: print( + f" phase 1 [{n:>3}/{args.nodes}] " + f"{path or './':<58} {routes} routes", + flush=True))) initial = architect.design(initial, objective_for(args.domain)) print(f"\nPhase 1 : architect {architect.summary()}") for path in architect.nodes: From f49b1dda045d535b503339e493f69bb59531332e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Sep 2026 14:22:24 +0000 Subject: [PATCH 13/42] Genesis: a child the architect calls two files is not a directory Upstream says both things in the same breath -- decompose MORE aggressively when the objective feels large, *and* single responsibility, shared capability at the lowest common ancestor, a directory holding one short function is a directory that did not want splitting. The first is a command. The second needs judgement, and a model handed both executes the first. Measured: the fly domain's architect designed 600 nodes for one simulator -- 921 before the budget cut it -- three quarters of them pure routing, at depth 8. It gave `apl`, which is a single GABAergic neuron, two children and one of those three more. Upstream's 123-hour C compiler run, 750 files and 249 000 physical lines, has **26** nodes and bottomed out at depth 5. So the counterweight is enforced rather than stated. Each child now declares `files` -- how many source files the architect expects that directory to hold, counting everything below it -- and a child of two or fewer is refused: it is two files in this node's API Surface, not a directory. A node of three or fewer is told to return no children at all. An undeclared count is not a refusal; the guard reads what is there rather than inventing a number. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/genesis/_architect.py | 36 ++++++++++++++++++++++++++++++-- tests/test_genesis_example.py | 38 ++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/examples/genesis/_architect.py b/examples/genesis/_architect.py index 5b624dfd..7b72319a 100644 --- a/examples/genesis/_architect.py +++ b/examples/genesis/_architect.py @@ -96,9 +96,24 @@ a node whose files one agent can write in one turn; anything larger has children, and \ anything smaller belongs in its parent's API Surface as a file. +**Before you open a child, say how big it is.** Each entry carries `files`: how many \ +source files you expect that directory to hold, counting everything below it. Then: + +- **A child of two files or fewer is not a directory.** It is two files in *this* \ +node's API Surface. Do not open it; list the files here instead. +- **If this whole node is three files or fewer, return no children at all.** Write \ +them here and be a leaf. + +Count honestly. The counterweight above is not advice -- a tree whose nodes are mostly \ +routers is a tree where almost nothing is built: measured on this port, an architect \ +that ignored it produced 600 nodes for one simulator, three quarters of them pure \ +routing, at depth 8. Upstream's 123-hour C compiler run -- 750 files and 249 000 lines \ +-- has **26** nodes and bottomed out at depth 5. + Reply with ONE JSON object and nothing else: {{"record": "", - "children": [{{"path": "{path_prefix}", "objective": ""}}]}} + "children": [{{"path": "{path_prefix}", "objective": "", + "files": }}]}} An empty `children` list means this node is a leaf and its files are written here.""" @@ -247,6 +262,9 @@ def __init__(self, complete, *, contracts: Sequence[str] = (), self.reused = 0 #: Children refused for being named after one of their ancestors. self.repeated = 0 + #: Children refused because the architect itself expected them to + #: hold two files or fewer. + self.too_small = 0 def design(self, given: Mapping[str, str], objective: str) -> Dict[str, str]: """``given`` plus one ``CONTEXT.md`` per node the architect decided on. @@ -346,6 +364,13 @@ def _apply(self, state: Dict[str, str], batch: Sequence[Tuple[str, str, int]], # file there is refused to every proposal. self.refused += 1 continue + if 0 < int(child.get("files") or 0) <= 2: + # A directory the architect itself expects to hold one or two files + # is two files in this node's API Surface. Refusing it here is the + # only place the counterweight to "decompose MORE aggressively" can + # be enforced rather than merely stated. + self.too_small += 1 + continue if repeats_an_ancestor(child["path"]): # "decompose MORE aggressively" has a counterweight and upstream # states both: shared capability belongs at the lowest common @@ -408,6 +433,8 @@ def summary(self) -> str: if self.mistaken_nodes else "") + (f", {self.repeated} repeated an ancestor's name" if self.repeated else "") + + (f", {self.too_small} too small to be a directory" + if self.too_small else "") + (f", {self.truncated} left undesigned at the node budget" if self.truncated else "") + (f", {self.unparsed} replies unusable" if self.unparsed else "")) @@ -462,9 +489,14 @@ def _parse(reply: str, prefix: str) -> Tuple[Optional[str], List[Dict[str, str]] # counts it instead. if prefix and "/" not in child: child = prefix + child + try: + files = int(item.get("files")) + except (TypeError, ValueError): + files = 0 # not declared; the guard lets it pass children.append({"path": child, "objective": str(item.get("objective", "")).strip() - or f"implement {child}"}) + or f"implement {child}", + "files": files}) # The record is the map the run delegates by, so a child the architect named and # did not route to would be unreachable. Add the entry it forgot. if children and ROUTING_HEADING.lower() not in record.lower(): diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index 5f5278b0..54ede149 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -1664,6 +1664,44 @@ def test_the_routing_table_is_made_to_agree_with_the_children_it_opened(): assert parse_routing(fixed) == ["a/b"] +def test_a_child_the_architect_calls_two_files_is_not_a_directory(): + """The counterweight to "decompose MORE aggressively", enforced rather than stated. + + Upstream says both in the same breath -- decompose harder when the objective feels + large, *and* single responsibility, shared capability at the lowest common ancestor, + a directory holding one short function is a directory that did not want splitting. + The first is a command; the second needs judgement, and a model given both executes + the first. Measured on the fly domain: 600 nodes for one simulator, three quarters + of them pure routing, at depth 8. Upstream's 123-hour C compiler run -- 750 files, + 249 000 lines -- has **26** nodes and bottomed out at depth 5. + + So a child now declares how many files it expects to hold, and one that says two or + fewer is refused: it is two files in this node's API Surface. + """ + reply = {"record": "# node\n", + "children": [{"path": "big", "objective": "a lot", "files": 9}, + {"path": "tiny", "objective": "a function", "files": 1}, + {"path": "pair", "objective": "two things", "files": 2}, + {"path": "silent", "objective": "undeclared"}]} + phase = ArchitectPhase(lambda prompt: json.dumps(reply) + if "`./`" in prompt or "path `./`" in prompt + else json.dumps({"record": "# leaf\n", "children": []}), + max_depth=2, max_nodes=10) + tree = phase.design({CONTEXT_FILE: "# root\n"}, "o") + assert phase.too_small == 2 + assert "big/" + CONTEXT_FILE in tree + assert "tiny/" + CONTEXT_FILE not in tree and "pair/" + CONTEXT_FILE not in tree + # An undeclared count is not a refusal -- the guard reads what is there and does + # not invent a number the architect never gave. + assert "silent/" + CONTEXT_FILE in tree + # And the refused ones are not left advertised in the routing table either. + assert parse_routing(tree[CONTEXT_FILE]) == ["big", "silent"] + + prompt = ARCHITECT_PROMPT.format(path="", path_prefix="", objective="o", context="c") + assert '"files"' in prompt + assert "three files or fewer, return no children" in prompt + + def test_a_node_may_not_be_named_after_one_of_its_own_ancestors(): """"Decompose MORE aggressively" has a counterweight and upstream states both. From 894566e39a93234e9e3cb39a1de4835dc1cb05ed Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Sep 2026 14:23:11 +0000 Subject: [PATCH 14/42] Genesis: the routed-to test carries the child's declared file count `_parse` now returns `files` on every child, undeclared as 0, and the assertion that pinned the child dict verbatim had to say so. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- tests/test_genesis_example.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index 54ede149..1ce4065c 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -1338,7 +1338,8 @@ def test_a_child_the_architect_named_is_always_routed_to(): record, children = parse_architect_reply( json.dumps({"record": "# src\n\n## Intent\nthings\n", "children": [{"path": "core", "objective": "geometry"}]}), "src/") - assert children == [{"path": "src/core", "objective": "geometry"}] + # `files` rides along undeclared, which the size guard reads as "no claim made". + assert children == [{"path": "src/core", "objective": "geometry", "files": 0}] assert parse_routing(record) == ["src/core"] From 0b3fe4fd1e50cdf8e02605a835f26c25bbb79757 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 00:08:42 +0000 Subject: [PATCH 15/42] Genesis: an executor session can reach an endpoint this session does not `--executor claude-code` spawned the CLI with an inherited environment. Inside a managed Claude Code session that environment carries CLAUDE_CODE_REMOTE, which puts the CLI on the host's session ingress and makes it ignore the ANTHROPIC_BASE_URL and key it was handed. So every episode sent the host's token to a third-party endpoint and got 401: one fly run opened 28 sessions, failed 23, and wrote no files at all. What the CLI reports for this is "Authentication error - This may be a temporary network issue, please try again", which is neither, and the stderr beside it names an unrecognized model for query_source generate_session_title -- the session-title side query, not the agent turn, and printed just the same on runs that work. I read that line as the cause and was wrong. `cli_env` drops the host-provider variables, but only when the run actually points somewhere else; a session on the host's own provider is untouched. The credential still only ever travels from the environment into a child's env. This also corrects 611c4af, which claimed pointing the SDK and the CLI at one endpoint was all it took to put both halves of a run on the same model. It is what the CLI documents, and it is true on a laptop. It was not true here, and the port's own fly runs are what proved otherwise. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/_common.py | 66 +++++++++++++++++++++++++++++++- examples/genesis/_claude_code.py | 10 ++++- 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/examples/_common.py b/examples/_common.py index ec31199f..0ec03bd4 100644 --- a/examples/_common.py +++ b/examples/_common.py @@ -18,9 +18,10 @@ from __future__ import annotations import argparse +import os import sys import threading -from typing import Callable, Optional +from typing import Callable, Dict, Mapping, Optional from agentdescent.agents import Usage, claude, openai_compatible @@ -397,6 +398,67 @@ def worker_count(args: argparse.Namespace, requested: int) -> int: return 1 +#: Variables that hand the Claude Code CLI a provider the *host* chose. +#: +#: A managed Claude Code session -- the web app, a CI runner, a cloud container -- does +#: not authenticate the way a laptop does. Instead of an API key it is handed a session +#: ingress: a URL and a short-lived token the harness holds. `CLAUDE_CODE_REMOTE` is +#: what puts the CLI in that mode, and in that mode it uses the ingress and **ignores +#: whatever credentials are in the environment**. Which is the right default -- it stops +#: a stray `ANTHROPIC_API_KEY` from silently redirecting a managed session's traffic. +#: +#: It also makes it impossible to point a *child* CLI somewhere else from inside such a +#: session, which is exactly what `--executor claude-code` does. The child inherits the +#: variable, ignores the `ANTHROPIC_BASE_URL` and key it was given, and 401s. What it +#: reports is "Authentication error · This may be a temporary network issue, please try +#: again" -- neither temporary nor a network issue. A run of 28 sessions failed 23 of +#: them and wrote zero files this way, and the accompanying stderr +#: (`[claude-code:unrecognized_model] ... "query_source":"generate_session_title"`) is a +#: red herring: that is the session-title side query, not the agent turn, and it is +#: emitted just the same on runs that work. +#: +#: `CLAUDE_CODE_REMOTE` alone is the switch -- measured, by bisection, against a +#: third-party Anthropic-compatible endpoint. The rest is the same host's plumbing for +#: the same decision; dropping it too costs nothing and does not leave a child half in +#: one mode and half in the other. +HOST_PROVIDER_VARS = ( + "CLAUDE_CODE_REMOTE", + "CLAUDE_CODE_PROVIDER_MANAGED_BY_HOST", + "CLAUDE_SESSION_INGRESS_TOKEN_FILE", + "SESSION_INGRESS_URL", + "CLAUDE_CODE_POST_FOR_SESSION_INGRESS_V2", +) + + +def cli_env(base_url: Optional[str] = None, api_key: Optional[str] = None, + env: Optional[Mapping[str, str]] = None) -> Dict[str, str]: + """The environment to spawn a Claude Code CLI child in. + + Pass through unchanged unless an endpoint is actually being overridden. When one + is -- either explicitly here, or because the surrounding environment already names + a non-Anthropic `ANTHROPIC_BASE_URL` -- drop :data:`HOST_PROVIDER_VARS` so the + child reads the key it was given rather than the host's session ingress, and set + both `ANTHROPIC_API_KEY` and `ANTHROPIC_AUTH_TOKEN`, since which one a given + endpoint wants is not something worth discovering twice. + + Nothing here writes a credential anywhere but this process's child: the key comes + from the environment and goes into an `env=` dict. + """ + out = dict(os.environ if env is None else env) + url = base_url or out.get("ANTHROPIC_BASE_URL", "") + key = api_key or out.get("ANTHROPIC_API_KEY") or out.get("ANTHROPIC_AUTH_TOKEN") + third_party = url and "api.anthropic.com" not in url + if not third_party: + return out + for name in HOST_PROVIDER_VARS: + out.pop(name, None) + out["ANTHROPIC_BASE_URL"] = url + if key: + out["ANTHROPIC_API_KEY"] = key + out["ANTHROPIC_AUTH_TOKEN"] = key + return out + + def claude_cli(model: str = "haiku", *, usage: Optional[Usage] = None, timeout: float = 300.0, binary: str = "claude"): """A ``Completion`` that shells out to the locally authenticated Claude Code CLI. @@ -424,7 +486,7 @@ def complete(prompt: str) -> str: started = time.time() try: done = subprocess.run(argv, input=prompt, capture_output=True, text=True, - timeout=timeout) + timeout=timeout, env=cli_env()) except subprocess.TimeoutExpired: if usage is not None: usage.record(failed=True, seconds=time.time() - started) diff --git a/examples/genesis/_claude_code.py b/examples/genesis/_claude_code.py index 71b17ca4..13624bac 100644 --- a/examples/genesis/_claude_code.py +++ b/examples/genesis/_claude_code.py @@ -47,6 +47,8 @@ from agentdescent.filetree import match_any, materialize +from .._common import cli_env + from ._delegation import Brief, Edit from ._spatial import SITUATED_EDIT_PROTOCOL # noqa: F401 (documented sibling) from ._world import normalise, owns @@ -84,7 +86,7 @@ def claude_code_available(binary: str = "claude") -> bool: return False try: out = subprocess.run([binary, "--version"], stdout=subprocess.PIPE, - stderr=subprocess.PIPE, timeout=30) + stderr=subprocess.PIPE, timeout=30, env=cli_env()) return out.returncode == 0 except Exception: # noqa: BLE001 - anything at all means "no" return False @@ -185,9 +187,13 @@ def _run(self, workspace: str, owner: str, brief: Brief) -> bool: reward=brief.reward) if self._failure else "") try: turns = self._root_turns if brief.depth == 0 else self._max_turns + # `cli_env` is a no-op unless this run points ANTHROPIC_BASE_URL at a + # third-party endpoint. When it does, it drops the variables by which a + # managed session tells the CLI to use the *host's* provider and ignore + # the environment -- inherited, they make every session here 401. out = subprocess.run(self._command(prompt, turns), cwd=workspace, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - timeout=self._timeout) + timeout=self._timeout, env=cli_env()) except Exception: # noqa: BLE001 - a dead session costs its episode return False try: From 11b811f8bf5cd868ffb55dda91f9d98ad73b2b8f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 00:08:50 +0000 Subject: [PATCH 16/42] Genesis: an empty suite is not a passing suite `own_test_failures` returns [] when the repository holds no test files, which reads as "everything passes" -- and that answer is the precondition for `complete_task`. A fly root agent used it: sixteen implementation files, zero tests, acceptance reward 0.000, and it claimed the objective delivered five times and was believed every time. That is backwards from the rule the same class already enforces on every child in `own_review`: code with no test is not finished. `require_tests` applies it to the whole repository, and the blind domain's `suite_failures` passes it, because that path is a gate. It stays off where the question really is "did anything the agents wrote break" -- a parent reviewing a child that has legitimately returned no code yet should not be told its tests failed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/genesis/_fly.py | 6 +++++- examples/genesis/_suite.py | 25 ++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/examples/genesis/_fly.py b/examples/genesis/_fly.py index 595db2a8..a1d437b3 100644 --- a/examples/genesis/_fly.py +++ b/examples/genesis/_fly.py @@ -611,9 +611,13 @@ def suite_failures(state: Mapping[str, str], *, audit: bool = False) -> List[str 问的不是“人的套件过没过” —— 人的套件它看不见 —— 而是**它自己那套过没过**，这正是 上游 manager 跑 `mix test` 的那一步。`audit` 在这里没有意义：仓库里不存在保留测试， 保留的那套在仓库外面，由评分路径注入。保留这个形参只是为了和别的域同签名。 + + `require_tests=True` 是因为这条路是**闸门**：一套空套件不算跑过。没有它，根 agent + 写了十六个实现文件、零个测试，也能喊 `complete_task` 并且被信 —— 这跟同一个类对每个 + 子节点执行的规矩正好相反。 """ del audit - return FLY.own_test_failures(state) + return FLY.own_test_failures(state, require_tests=True) reward = reward_test HELD_OUT_FRAC = FLY.held_out_frac() diff --git a/examples/genesis/_suite.py b/examples/genesis/_suite.py index 9e165840..55c2fb30 100644 --- a/examples/genesis/_suite.py +++ b/examples/genesis/_suite.py @@ -956,7 +956,17 @@ def _requirement(self, stem: str, func: str) -> str: heading = self.requirements.get(stem, "") return (f"{heading}\n\n" if heading else "") + f"ACCEPTANCE: {sentence}." - def own_test_failures(self, state, under: str = "") -> List[str]: + @staticmethod + def _implementation_files(state, under: str = "") -> List[str]: + """Python the agents wrote that is neither a test nor a package marker.""" + return [path for path in state + if path.endswith(".py") + and not path.rsplit("/", 1)[-1].startswith("test_") + and "__init__" not in path + and (not under or path == under or path.startswith(under + "/"))] + + def own_test_failures(self, state, under: str = "", + *, require_tests: bool = False) -> List[str]: """The tests **the agents wrote**, run in one interpreter. `mix test`, theirs. Everything in `state` is the agents' own work by construction: the acceptance @@ -967,12 +977,25 @@ def own_test_failures(self, state, under: str = "") -> List[str]: `under` restricts it to one node's subtree, which is what a parent reviewing one child's work wants. + + `require_tests` closes the hole that an empty suite is a passing suite. With no + test files at all the loop below has nothing to fail on and returns `[]`, which + reads as "everything passes" -- so a root agent that wrote sixteen + implementation files and zero tests could call `complete_task` and be believed. + That is exactly backwards from the rule the same class enforces on every child + in :meth:`own_review`: code with no test is not finished. Pass this wherever the + answer gates something (the completion precondition, the closing report) and + leave it off where the question really is "did anything the agents wrote break", + such as a parent reviewing a child that legitimately returned no code yet. """ owned = {path: body for path, body in state.items() if path.endswith(".py") and path.rsplit("/", 1)[-1].startswith("test_") and (not under or path == under or path.startswith(under + "/"))} if not owned: + if require_tests and self._implementation_files(state, under): + return ["no test file anywhere in the repository; " + "tests are the definition of done"] return [] try: plan = [(path, func) for path, func, _ in _discover_tests(owned)] From b19a6962a929dcd5324a2a9897830b0e4b0d2c4a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 02:43:10 +0000 Subject: [PATCH 17/42] Genesis: a reasoning model's thinking is spent from --max-tokens too The architect asked for a repository design and got 1708 characters of a JSON object that stopped mid-word. The run reported "architect designed 0 nodes, 1 replies unusable" and then grew a whole phase 2 with no tree at all, which is a shape that looks like a bad model or a parser bug and is neither. The cap was the default 4096, and this backend spends it on thinking first, so what was left for visible content did not hold the record. The same file already warns about this ("a 1024 cap returned nothing at all for 4 of 8 reflection prompts") -- it just had no way to raise the number from a command line. Turning thinking off is the other lever and the worse one: it changes what the model produces, not only how much. Measured on the architect prompt: 4096 emits ~15k output tokens across attempts in 148s and truncates about half the time; 16000 takes 264s and parses. `--max-tokens` and `--timeout` are declared and honoured here, which is what this module exists for. They come in together because raising one without the other only moves the failure: the call that now fits took 264s, and claude()'s 120s default would have timed it out three times over. Both are opt-in, by naming a default. Nine ports already declare `--max-tokens` themselves, each with a number it measured -- 4096, 16000, 32000 -- so declaring it here unconditionally is an argparse conflict that takes those entry points down at import, which is how I found them: six tests across porous, openevolve and metasearch. A port that wants the shared flag asks for it, the way `include_val_cap` withholds one from a port whose splits are already frozen. The method runner asks, and keeps its measured 1024/180. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/_common.py | 40 ++++++++++++++++++++ examples/_method_runner.py | 3 +- examples/genesis/genesis_recursive_worlds.py | 4 ++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/examples/_common.py b/examples/_common.py index 0ec03bd4..92642ed7 100644 --- a/examples/_common.py +++ b/examples/_common.py @@ -38,6 +38,8 @@ def add_standard_args( model_default: Optional[str] = DEFAULT_MODEL, model_help: str = "model id", max_seconds_default: float = 30.0, + max_tokens_default: Optional[int] = None, + timeout_default: Optional[float] = None, async_ratio_default: int = 3, eval_concurrency_default: Optional[int] = 8, include_val_cap: bool = True, @@ -162,6 +164,30 @@ def add_standard_args( "endpoints only). A throughput knob, and a quality one -- see " "completion_for"), ) + # Opt-in, by naming a default. Nine ports already declare `--max-tokens` + # themselves, each with a number it measured -- 4096, 16000, 32000 -- and + # declaring it here unconditionally is an argparse conflict that takes those + # ports' entry points down at import. So a port that wants the shared flag asks + # for it, the way `include_val_cap` withholds one from a port whose splits are + # already frozen. + if max_tokens_default is not None: + parser.add_argument( + "--max-tokens", + type=int, + default=max_tokens_default, + help=(f"output-token cap for one model call (default " + f"{max_tokens_default}). Raise it for a reasoning model, whose " + f"thinking is spent from the same budget -- see completion_for"), + ) + if timeout_default is not None: + parser.add_argument( + "--timeout", + type=float, + default=timeout_default, + help=(f"seconds for one model call (default {timeout_default:g}). A " + f"reasoning model given a large --max-tokens legitimately exceeds " + f"120s, and the retry wrapper turns each timeout into three"), + ) return parser @@ -591,14 +617,28 @@ def completion_for(args: argparse.Namespace, *, usage: Optional[Usage] = None, # run would report thinking off while the CLI kept it on. print("note: --no-thinking has no effect with --provider claude-cli; the " "CLI decides its own reasoning budget", file=sys.stderr) + if getattr(args, "max_tokens", None): + print("note: --max-tokens has no effect with --provider claude-cli; the " + "CLI sets its own output cap", file=sys.stderr) built = claude_cli(model=args.model, usage=usage, timeout=float(getattr(args, "timeout", None) or 300.0)) return gauge.wrap(built) if gauge is not None else built if is_openai_compatible(args): + if getattr(args, "max_tokens", None): + kwargs.setdefault("max_tokens", int(args.max_tokens)) built = openai_compatible(model=args.model, usage=usage, **kwargs) return gauge.wrap(built) if gauge is not None else built if getattr(args, "no_thinking", False): kwargs.setdefault("thinking", {"type": "disabled"}) + # A reasoning model spends `max_tokens` on thinking *first* and emits visible + # content from what is left, so the default 4096 is not 4096 of answer. Measured: + # deepseek-v4-flash answering this package's architect prompt at the default + # returned 1708 characters of a JSON object and stopped mid-word -- unparsable, + # and the run reported "architect designed 0 nodes, 1 replies unusable" rather + # than anything that looked like a token cap. Turning thinking off is the other + # lever and a worse one: it changes what the model produces, not just how much. + if getattr(args, "max_tokens", None): + kwargs.setdefault("max_tokens", int(args.max_tokens)) # --timeout reaches this path too: claude()'s 120s default assumes Claude # latencies, and a thinking model behind an Anthropic-shaped endpoint # (GLM-5.2 on GSM-Hard) legitimately exceeds it -- a baseline eval died at diff --git a/examples/_method_runner.py b/examples/_method_runner.py index 3595daac..681702d7 100644 --- a/examples/_method_runner.py +++ b/examples/_method_runner.py @@ -632,6 +632,7 @@ def build_parser( """ parser = argparse.ArgumentParser() add_standard_args(parser, model_default="glm-5.2", + max_tokens_default=1024, timeout_default=180.0, async_ratio_default=DEFAULT_ASYNC_RATIO, # `None`, not 8: an explicit flag wins, and without one the # runner keeps one-at-a-time gate scoring in `--serial`. @@ -657,10 +658,8 @@ def build_parser( "prompt that works the arithmetic out scores 1.000 / 0.958 / 0.875 " "at 0.0 / 0.7 / 1.0, so it is second-order next to what the prompt " "says -- but it is not nothing, and it was not reportable")) - parser.add_argument("--max-tokens", type=int, default=1024) if extra_args is not None: extra_args(parser) - parser.add_argument("--timeout", type=float, default=180.0) return parser diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index cb55fe96..bb2cf2a5 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -180,7 +180,11 @@ def build_parser() -> argparse.ArgumentParser: # that arm; barrier-free by default, that dormant number became the thing # that ended every run -- a 4 000-episode budget stopping after 12 rollouts. # `--episodes` is the budget; this is a stopwatch for when you want one. + # The backend factories' own defaults, made reachable from a command + # line: a reasoning model spends `max_tokens` on thinking before it + # emits anything, and 4096 was not enough for one architect reply. parser, model_default=None, max_seconds_default=0.0, + max_tokens_default=4096, timeout_default=120.0, model_help=("optional: let a model be the manager and executor agents " "(else rule-based offline actors -- see the selected " "--domain module)")) From b0e0e875f69b4e9591cdf58c8064527f9a9ff34f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 07:52:25 +0000 Subject: [PATCH 18/42] Genesis: phase 1's architect can be a session that writes the record Upstream's Architect is not a question with an answer. `agents/architect.ex` is `use EvoGit.Agent` -- "an agent session loop template that manages a single agent session, handling tool loops" -- with `agent_type :read_write`, which in `agent/tools.ex` means file_read/create/write/edit, make_dir, context_read/ write/edit, run_bash, ripgrep, glob, list_dir. CONTEXT.md is written with a tool. Nothing comes back as a structured reply and there is nothing to parse. So is the Executor, and so is the Manager: every agent there is a session. This port had the Executor right already (`--executor claude-code`) and the architect as one completion per node returning {"record", "children"}. That is the shape a reasoning model is slowest at -- one reply holding a whole record and a whole child list, thought about once, at length -- and a truncated reply is not a short record but an empty phase 1: one run reported "architect designed 0 nodes, 1 replies unusable" and grew a phase 2 against no tree. `--architect-session` designs a node the way the executor implements one: a session in a throwaway copy, tools Read/Write/Edit/Glob/Grep, and what it did recovered by reading the file it wrote. No Bash -- the executor gets a shell because it must run the suite it is judged by, and an architect that can run things is an architect that starts implementing. One node per session, with the phase still driving the levels. Upstream's architect spawns sub-architects and recurses inside the session; keeping the recursion in the driver keeps phase 1's level-parallelism, keeps a session small enough to watch, and keeps the spatial contract checkable per node. That is a stated difference, not an oversight. Two things fall out of the record being a file: The design rules now live in ARCHITECT_RULES, and each path appends its own delivery -- "reply with JSON" or "write the file". Duplicating them is how the two paths would grow different trees. A routing line may declare its child's size, `(12 files)`, because the file-count threshold that refuses a child too small to be a directory has nowhere else to live once there is no JSON. Records without counts still route. And one defect this found, which was costing the completion path too: the routing section was matched by an exact prefix on "## Routing Table", so "### Routing Table", "## Routing" and "## 4. Routing Table" all opened nothing and the node silently became a leaf. A session wrote a complete five-child table under such a heading and the tree recorded zero children. Measured on one fly node against a coding-plan endpoint: session 497s uncapped, 203-242s with a 2048-token reasoning cap, same 4.6k record and the same children. `--thinking-tokens` exposes that cap, following upstream, which carries `reasoning_effort` per model profile beside `max_tokens` and `concurrency` rather than fixing it. It bounds reasoning; it does not disable it, which changes what the model produces and not only how long it takes. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/genesis/_architect.py | 39 ++- examples/genesis/_architect_session.py | 259 +++++++++++++++++++ examples/genesis/_world.py | 57 +++- examples/genesis/genesis_recursive_worlds.py | 31 +++ tests/test_genesis_example.py | 77 ++++++ 5 files changed, 457 insertions(+), 6 deletions(-) create mode 100644 examples/genesis/_architect_session.py diff --git a/examples/genesis/_architect.py b/examples/genesis/_architect.py index 7b72319a..49d62ce8 100644 --- a/examples/genesis/_architect.py +++ b/examples/genesis/_architect.py @@ -34,10 +34,14 @@ STANDARD_SECTIONS, looks_like_file, normalise, parse_routes, parse_routing, shadowed_by_module) -__all__ = ["ARCHITECT_PROMPT", "REFINE_PROMPT", "ArchitectPhase", +__all__ = ["ARCHITECT_PROMPT", "ARCHITECT_RULES", "REFINE_PROMPT", "ArchitectPhase", "harness_record", "misaligned"] -ARCHITECT_PROMPT = """You are an architect agent in a recursive software world, \ +#: The design rules, without a delivery instruction. Both paths use these: the +#: completion architect appends "reply with JSON", the session architect appends +#: "write the file". Splitting them is what keeps the two from drifting -- the +#: rules here are the ones every measured tree in this port came out of. +ARCHITECT_RULES = """You are an architect agent in a recursive software world, \ situated at the repository path `{path}`. You design; you do not implement. {context} @@ -110,6 +114,12 @@ routing, at depth 8. Upstream's 123-hour C compiler run -- 750 files and 249 000 lines \ -- has **26** nodes and bottomed out at depth 5. +Count honestly.""" + + +#: The completion path's delivery: one JSON object carrying record and children. +ARCHITECT_PROMPT = ARCHITECT_RULES + """ + Reply with ONE JSON object and nothing else: {{"record": "", "children": [{{"path": "{path_prefix}", "objective": "", @@ -226,8 +236,16 @@ class ArchitectPhase: def __init__(self, complete, *, contracts: Sequence[str] = (), max_depth: int = 3, max_nodes: int = 12, root_path: str = "", - resume: bool = False, workers: int = 1, on_node=None): + resume: bool = False, workers: int = 1, on_node=None, + session=None): self._complete = complete + #: An :class:`~examples.genesis._architect_session.ArchitectSession`, or None. + #: Given one, a node is designed by an agent session that *writes* its record + #: with a file tool -- upstream's shape (`agents/architect.ex` is + #: `use EvoGit.Agent`, `agent_type :read_write`, and `CONTEXT.md` is written + #: with `context_write`) -- instead of a completion returning JSON. The phase + #: drives the levels either way; only how one node is produced changes. + self._session = session self._contracts = tuple(contracts) self._max_depth = max_depth self._max_nodes = max_nodes @@ -445,10 +463,23 @@ def _ask(self, state: Mapping[str, str], path: str, objective: str, depth: int) -> Tuple[Optional[str], List[Dict[str, str]]]: world = LocalWorld(version=0, path=path, readonly=self._contracts) prefix = f"{path}/" if path else "" + situated = world.situate(state, contracts=self._contracts) + if self._session is not None: + rules = ARCHITECT_RULES.format(path=path or "./", path_prefix=prefix, + objective=objective, context=situated) + try: + record, children = self._session(state, path, rules) + except Exception: # noqa: BLE001 - one dead session costs one node + record, children = None, [] + if record is None: + # A session that wrote no record and a reply that did not parse are + # the same event to the tree: a node nobody designed. + self.unparsed += 1 + return record, children try: reply = self._complete(ARCHITECT_PROMPT.format( path=path or "./", path_prefix=prefix, objective=objective, - context=world.situate(state, contracts=self._contracts))) or "" + context=situated)) or "" except Exception: # noqa: BLE001 - one dead call costs one node self.unparsed += 1 return None, [] diff --git a/examples/genesis/_architect_session.py b/examples/genesis/_architect_session.py new file mode 100644 index 00000000..de9cce1c --- /dev/null +++ b/examples/genesis/_architect_session.py @@ -0,0 +1,259 @@ +"""Phase 1's architect as a Claude Code session, the way upstream runs it. + +Upstream's Architect is not a question with an answer -- it is an agent session with +tools. ``agents/architect.ex`` is ``use EvoGit.Agent``, whose moduledoc calls itself +"an agent session loop template that manages a single agent session, handling tool +loops, timeouts, and graceful recovery", and its ``agent_type`` is ``:read_write``, +which in ``agent/tools.ex`` means the full set: ``file_read``, ``file_create``, +``file_write``, ``file_edit``, ``make_dir``, ``context_read``/``context_write``/ +``context_edit``, ``run_bash``, ``ripgrep``, ``glob``, ``list_dir``. **`CONTEXT.md` is +written with a tool.** Nothing is returned as a structured reply, and there is nothing +to parse. + +This port had the architect as one completion per node returning +``{"record": ..., "children": [...]}``. That is a real simplification and it cost +twice: + +* **It is the shape a reasoning model is slowest at.** One reply has to hold a whole + record and a whole child list, so the model thinks once, at length, and emits once, + at length. Measured against a coding-plan endpoint: ~15 000 output tokens and 148-264 + s for a single node, where the same domain on an endpoint that emitted ~800 tokens + per call designed 23 nodes in under three minutes. A session spends the same design + on many small turns instead. +* **A truncated reply is an empty phase 1.** One run reported + ``architect designed 0 nodes, 1 replies unusable`` and then grew an entire phase 2 + against no tree, because the JSON stopped mid-word. A session has no JSON: the record + is a file, and a file that is written is a file that parses. + +What is *not* upstream here, and is a deliberate choice: upstream's architect spawns +``subagent_architect`` for each child and the recursion happens inside the session. +Here one session designs **one node**, the port reads the routing table it wrote, and +the next level is driven by the phase, a level at a time. That keeps phase 1's existing +level-parallelism, keeps every session small enough to watch, and keeps the spatial +contract checkable per node -- at the cost of the recursion living in the driver rather +than in the agent. :class:`~examples.genesis._claude_code.ClaudeCodeExecutor` already +works this way for implementation episodes, and this is the same trade applied to +design. + +The other deliberate difference: **no Bash.** The executor gets a shell because the +point of a session rather than a completion is that it can run the suite it is judged +by. An architect designs and does not implement, so there is nothing for it to run, and +a shell is how a design session turns into an implementation session by accident. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import tempfile +from typing import Dict, List, Mapping, Optional, Sequence, Tuple + +from agentdescent.filetree import materialize + +from .._common import cli_env +from ._world import CONTEXT_FILE, normalise, parse_route_sizes, parse_routes + +__all__ = ["ARCHITECT_SESSION_BRIEF", "ArchitectSession"] + + +#: What the session is told. The design rules themselves stay in +#: :data:`examples.genesis._architect.ARCHITECT_PROMPT` -- they are the same rules +#: whichever way the record is produced, and duplicating them is how the two paths +#: would drift apart. This wrapper says only what a *session* has to be told: which +#: file to write, and that writing it is the whole job. +ARCHITECT_SESSION_BRIEF = """{rules} + +--- HOW TO DELIVER THIS --- + +You are in a scratch copy of the repository, and you have file tools. Do not reply \ +with the record: **write it to `{record_path}`** and stop. What you wrote is read from \ +the working tree, so a reply is not read at all. + +Write that one file. Create no directories, no source files, no stubs, and no other \ +`CONTEXT.md` -- a child's record belongs to the agent situated there, and anything you \ +write outside `{record_path}` is discarded. + +Your routing table carries each child's size, so the line is: + + - `./{path_prefix}/` ( files) -> + +When you have written the file, you are done.""" + + +class ArchitectSession: + """``session(state, path, objective) -> (record, children)``. + + Drop-in for the completion the architect phase otherwise calls: same inputs, same + return shape, so :class:`~examples.genesis._architect.ArchitectPhase` does not know + which one it has. Failure is the same shape too -- ``(None, [])`` -- and is counted + by the phase as an unusable reply, because from the tree's point of view a session + that wrote nothing and a reply that did not parse are the same event. + """ + + #: A design turn is small -- read the contract, read the parent records, write one + #: file -- so this is a bound on going wrong rather than a budget to spend. The + #: implementation executor's 2048/128 are upstream's numbers for *implementation* + #: sessions; there is no upstream number for a single-node design session, because + #: upstream's architect session designs a whole subtree. + TURNS = 40 + + def __init__(self, *, frozen: Sequence[str] = (), binary: str = "claude", + model: str = "", max_turns: int = 0, timeout: float = 900.0, + root: Optional[str] = None, thinking_tokens: int = 0): + self._frozen = tuple(frozen) + self._binary = binary + self._model = model + self._max_turns = max_turns or self.TURNS + self._timeout = timeout + self._root = root + #: A per-turn reasoning cap for the session, or 0 to leave the CLI's own. + #: Upstream treats reasoning strength as a per-model setting rather than a + #: constant -- a model profile carries `reasoning_effort` beside `max_tokens` + #: and `concurrency` (`config/schema/definitions.ex`) -- and on a coding-plan + #: endpoint this is the difference between a design turn that thinks for + #: seconds and one that thinks for minutes. It bounds thinking; it does not + #: disable it, which is a different and worse lever: turning reasoning off + #: changes what the model produces, not only how long it takes. + self._thinking_tokens = max(0, int(thinking_tokens)) + #: Sessions run, and what came back. + self.sessions = 0 + self.failed = 0 + self.turns = 0 + #: Files a session wrote that were not its own record, discarded. An architect + #: that starts writing code is the failure mode this counts. + self.strays = 0 + + def __call__(self, state: Mapping[str, str], path: str, rules: str, + ) -> Tuple[Optional[str], List[Dict[str, object]]]: + path = normalise(path) + record_path = f"{path}/{CONTEXT_FILE}" if path else CONTEXT_FILE + prefix = f"{path}/" if path else "" + workspace = tempfile.mkdtemp(prefix="genesis-arch-", dir=self._root) + try: + before = dict(state) + materialize(before, workspace) + self._write_settings(workspace) + self.sessions += 1 + prompt = ARCHITECT_SESSION_BRIEF.format( + rules=rules, record_path=record_path, path_prefix=prefix) + if not self._run(workspace, prompt): + self.failed += 1 + record = self._read(workspace, record_path, before) + finally: + shutil.rmtree(workspace, ignore_errors=True) + if record is None: + return None, [] + return record, _children(record, prefix) + + # -- internals --------------------------------------------------------- + + def _write_settings(self, workspace: str) -> None: + """Deny the frozen globs by name, as the implementation executor does.""" + deny: List[str] = [] + for pattern in self._frozen: + deny += [f"Write({pattern})", f"Edit({pattern})", + f"NotebookEdit({pattern})"] + settings = {"permissions": {"deny": deny, "additionalDirectories": []}} + target = os.path.join(workspace, ".claude") + os.makedirs(target, exist_ok=True) + with open(os.path.join(target, "settings.local.json"), "w", + encoding="utf-8") as handle: + json.dump(settings, handle) + + def _command(self, prompt: str) -> List[str]: + # No Bash, no TodoWrite: this session reads and writes one file. Glob and Grep + # stay because a node is designed against what is already in the tree. + command = [self._binary, "-p", prompt, + "--output-format", "json", + "--permission-mode", "acceptEdits", + "--max-turns", str(self._max_turns), + "--allowedTools", "Read,Write,Edit,Glob,Grep", + "--disallowedTools", "Bash,WebFetch,WebSearch,Task"] + if self._model: + command += ["--model", self._model] + return command + + def _run(self, workspace: str, prompt: str) -> bool: + try: + env = cli_env() + if self._thinking_tokens: + env["MAX_THINKING_TOKENS"] = str(self._thinking_tokens) + out = subprocess.run(self._command(prompt), cwd=workspace, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=self._timeout, env=env) + except Exception: # noqa: BLE001 - a dead session costs its node + return False + try: + report = json.loads(out.stdout.decode("utf-8", "replace") or "{}") + self.turns += int(report.get("num_turns") or 0) + return not report.get("is_error", out.returncode != 0) + except Exception: # noqa: BLE001 - no JSON is not a reason to lose a record + return out.returncode == 0 + + def _read(self, workspace: str, record_path: str, + before: Mapping[str, str]) -> Optional[str]: + """The record the session wrote, or `None`; everything else it wrote is a stray. + + A session reported as failed may still have written the file before it died -- + the turn limit, in particular, lands *after* the work -- so the tree is read + either way and the file is what decides. + """ + try: + with open(os.path.join(workspace, record_path), encoding="utf-8") as handle: + record = handle.read() + except OSError: + record = "" + self.strays += _strays(workspace, record_path, before) + record = record.strip() + return (record + "\n") if record else None + + def summary(self) -> str: + return (f"sessions={self.sessions} failed={self.failed} " + f"turns={self.turns} strays={self.strays}") + + +def _children(record: str, prefix: str) -> List[Dict[str, object]]: + """The routing table, as the child dicts the architect phase folds. + + Same shape the JSON path produces, so the phase's guards -- the file-count + threshold, the module-shadowing refusal, the ancestor-repeat check -- apply + unchanged. A bare name is relative to this node; a path with a separator is taken + as written, so a route out of the subtree stays a violation the caller counts + rather than a relocation this function performs silently. + """ + sizes = parse_route_sizes(record) + out: List[Dict[str, object]] = [] + for child, handles in parse_routes(record): + # The size is keyed by the path *as the record wrote it*, so it is read + # before the bare name is resolved against this node -- `web/` is `web` in + # the sizes map and `src/web` in the tree, and looking it up after the + # rewrite silently reported every bare-named child as undeclared. + declared = int(sizes.get(normalise(child), 0)) + if prefix and "/" not in child: + child = prefix + child + out.append({"path": child, + "objective": handles or f"implement {child}", + "files": declared or int(sizes.get(normalise(child), 0))}) + return out + + +def _strays(workspace: str, record_path: str, before: Mapping[str, str]) -> int: + """How many files the session wrote that were not its own record.""" + count = 0 + for base, dirs, files in os.walk(workspace): + dirs[:] = [d for d in dirs if d not in (".git", ".claude", "__pycache__")] + for name in files: + rel = os.path.relpath(os.path.join(base, name), workspace) + rel = rel.replace(os.sep, "/") + if rel == record_path: + continue + try: + with open(os.path.join(base, name), encoding="utf-8") as handle: + body = handle.read() + except (UnicodeDecodeError, OSError): + continue + if before.get(rel) != body: + count += 1 + return count diff --git a/examples/genesis/_world.py b/examples/genesis/_world.py index e4184a05..46363557 100644 --- a/examples/genesis/_world.py +++ b/examples/genesis/_world.py @@ -42,6 +42,7 @@ "EpisodeRecord", "LocalWorld", "ROUTING_HEADING", + "is_routing_heading", "SKILLS_DIR", "TRUNCATED", "WorldLog", @@ -91,10 +92,33 @@ _ROUTE_LINE = re.compile( r"""^\s*[-*]\s* # a markdown list item `?\s*(?P\.?/?[A-Za-z0-9._\-/]+?)\s*/?`?\s* # the path, backticks optional + (?:\(\s*(?P\d+)\s*files?\s*\)\s*)? # optional "(12 files)" (?:$|[-=]+>|\u2192|:) # end of line, '->', an arrow, or a colon """, re.VERBOSE) +#: A heading that opens the routing section, however it is written. +#: +#: The exact string is what this port *asks* for, and an agent writing markdown by +#: hand does not always give it back: `### Routing Table` under a deeper document, +#: `## Routing`, `## 4. Routing Table` under a numbered outline. A prefix match on the +#: canonical heading rejects all three, and a rejected heading is not a formatting +#: nit -- the routing table is the map delegation runs on, so the node silently +#: becomes a leaf. Measured: one session architect wrote a complete five-child table +#: and the tree recorded zero children. +#: +#: Matching is deliberately narrow all the same. The text after the hashes, after an +#: optional `1.` / `1)` ordinal, has to *start* with "routing" -- so "## Routing Table", +#: "### Routing", "## 4. Routing Table (children)" are the section, and "## Non-routing +#: notes" or a sentence mentioning routing is not. +_ROUTING_HEADING_LINE = re.compile(r"^\s*#{1,6}\s*(?:\d+[.)]\s*)?routing\b", re.I) + + +def is_routing_heading(line: str) -> bool: + """Does this line open the routing section?""" + return bool(_ROUTING_HEADING_LINE.match(line or "")) + + def parse_routing(body: str) -> List[str]: """The child paths a ``CONTEXT.md`` routes to, in the order it lists them. @@ -107,7 +131,7 @@ def parse_routing(body: str) -> List[str]: for line in lines: stripped = line.strip() if stripped.startswith("##"): - inside = stripped.lower().startswith(ROUTING_HEADING.lower()) + inside = is_routing_heading(stripped) continue if not inside: continue @@ -119,6 +143,35 @@ def parse_routing(body: str) -> List[str]: return out + +def parse_route_sizes(body: str) -> Dict[str, int]: + """``{path: how many files the architect said it holds}`` for one record. + + The size is what refuses a child too small to be a directory, and a + *session* architect has nowhere to put it but the routing line -- it writes + `CONTEXT.md` with a tool and returns no structured reply at all, the way + upstream's Architect does (`agents/architect.ex`, `context_write`). A record + that omits the count yields nothing here, and the caller's guard reads a + missing count as "not declared" rather than as zero. + """ + if not body: + return {} + out: Dict[str, int] = {} + inside = False + for line in body.splitlines(): + stripped = line.strip() + if stripped.startswith("##"): + inside = is_routing_heading(stripped) + continue + if not inside: + continue + match = _ROUTE_LINE.match(line) + if match and match.group("files"): + path = normalise(match.group("path")) + if path: + out.setdefault(path, int(match.group("files"))) + return out + def parse_routes(body: str) -> List[Tuple[str, str]]: """``(path, what it handles)`` for every routing-table entry, in order. @@ -138,7 +191,7 @@ def parse_routes(body: str) -> List[Tuple[str, str]]: for line in body.splitlines(): stripped = line.strip() if stripped.startswith("##"): - inside = stripped.lower().startswith(ROUTING_HEADING.lower()) + inside = is_routing_heading(stripped) continue if not inside: continue diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index bb2cf2a5..b1f55d01 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -75,6 +75,7 @@ from __future__ import annotations import argparse +import sys import posixpath from agentdescent import Policies, evolve @@ -95,6 +96,7 @@ from . import _stackvm as stackvm from ._architect import (ArchitectPhase, harness_record, misaligned, missing_sections) +from ._architect_session import ArchitectSession from ._claude_code import ClaudeCodeExecutor, claude_code_available from ._extract import ExtractPhase from ._judge import ParentJudge @@ -294,6 +296,22 @@ def build_parser() -> argparse.ArgumentParser: help="how many CONTEXT.md records phase 1 may write. The fly " "domain's architect named 17 children it never reached at " "the default, so the tree came out two deep and truncated") + parser.add_argument("--thinking-tokens", type=int, default=0, + help="per-turn reasoning cap for an --architect-session " + "design turn (MAX_THINKING_TOKENS); 0 leaves the CLI's " + "own. Upstream carries reasoning strength per model " + "profile (`reasoning_effort`) rather than as a constant. " + "Measured on one fly node against a coding-plan endpoint: " + "uncapped 497s, capped at 2048 203-242s, same record") + parser.add_argument("--architect-session", action="store_true", + help="run phase 1's architect as a Claude Code session that " + "WRITES each CONTEXT.md with a file tool, rather than as " + "one completion returning JSON. This is upstream's shape " + "-- `agents/architect.ex` is `use EvoGit.Agent` with " + "`agent_type :read_write`, and the record is written with " + "`context_write` -- and it removes the whole class of " + "failure where a truncated reply leaves phase 1 with no " + "tree. Needs --architect and the claude CLI") parser.add_argument("--architect", action="store_true", help="upstream's Phase 1: an agent designs the CONTEXT.md tree " "-- intent, API surface, constraints, routing tables -- " @@ -493,6 +511,7 @@ def main(argv=None) -> None: print(f" {path or './':<24} {len(parse_routing(record))} routes") architect = None + designer = None if args.mode == "b": if complete is None: print("--mode b needs --model: there is no offline architect, and a " @@ -504,7 +523,17 @@ def main(argv=None) -> None: root = package_root(args.domain) initial[CONTEXT_FILE] = harness_record(root, spec.FROZEN, objective_for(args.domain)) + if args.architect_session: + if not claude_code_available(): + print("--architect-session needs the `claude` CLI on PATH; phase 1 " + "falls back to one completion per node", file=sys.stderr) + else: + designer = ArchitectSession( + frozen=spec.FROZEN, model=(args.executor_model or args.model or ""), + timeout=float(getattr(args, "timeout", None) or 900.0), + thinking_tokens=args.thinking_tokens) architect = ArchitectPhase(complete, contracts=spec.CONTRACTS, + session=designer, max_depth=args.depth, max_nodes=args.nodes, root_path=root, # A phase 1 that ran out of budget is worth @@ -720,6 +749,8 @@ def _superseded(rendered, task, output, reward_): print(f"workspace : {ledger.summary()}") if sessions is not None: print(f"claude code : {sessions.summary()}") + if designer is not None and designer.sessions: + print(f"architect : {designer.summary()}") if code_review is not None: print(f"parent review : read={code_review.reviewed} " f"rejected={code_review.rejected} unparsed={code_review.unparsed}") diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index 1ce4065c..dea82589 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -2152,3 +2152,80 @@ def superseded(rendered, task, output, reward): # replaced by the bundle assert result.final_reward > 0.9, result.outcomes() assert log.observed_depth() >= 2, "the run never delegated past the first level" assert log.contract_violations == 0 + + +# -- phase 1's architect as a session ------------------------------------------- + +def test_a_routing_heading_is_read_at_any_level_or_numbering(): + """The routing table is the map delegation runs on, so a heading written the way + an agent writes markdown -- deeper, numbered, lowercased -- must still open it. + + Measured: a session architect wrote a complete five-child routing table under a + heading the exact-prefix match rejected, and the tree recorded the node as a leaf. + """ + from examples.genesis._world import parse_route_sizes, parse_routes + + for heading in ("## Routing Table", "### Routing Table", "## Routing table", + "## Routing", "#### Routing", "## 4. Routing Table", + "## Routing Table (children)"): + body = f"{heading}\n\n- `./src/brain/` (3 files) -> the circuits\n" + assert parse_routes(body) == [("src/brain", "the circuits")], heading + assert parse_route_sizes(body) == {"src/brain": 3}, heading + # and a heading that only mentions routing is still not the section + assert parse_routes("## Non-routing notes\n\n- `./src/brain/` -> x\n") == [] + + +def test_a_routing_line_may_declare_the_child_s_size(): + """A session architect writes `CONTEXT.md` and returns no structured reply, so the + file-count that refuses a child too small to be a directory has to live in the + routing line. A record without counts still routes; it just declares nothing. + """ + from examples.genesis._world import parse_route_sizes, parse_routes + + sized = ("## Routing Table\n\n" + "- `./src/brain/` (12 files) -> circuits\n" + "- `./src/web/` -> the page\n") + assert parse_routes(sized) == [("src/brain", "circuits"), ("src/web", "the page")] + assert parse_route_sizes(sized) == {"src/brain": 12} + assert parse_route_sizes("## Routing Table\n\n- `./src/brain/` -> x\n") == {} + + +def test_the_session_architect_returns_what_the_completion_one_does(): + """`ArchitectSession` is a drop-in for the completion: same return shape, so the + phase's guards -- the file-count threshold, the shadowing refusals -- apply to + both. A record the session did not write is `(None, [])`, which the phase counts + as an unusable reply, because a node nobody designed is one event either way. + """ + from examples.genesis._architect_session import _children + + record = ("## Routing Table\n\n" + "- `./src/brain/` (12 files) -> circuits\n" + "- `web/` (4 files) -> the page\n") + kids = _children(record, "src/") + assert [k["path"] for k in kids] == ["src/brain", "src/web"] + assert [k["files"] for k in kids] == [12, 4] + assert all(k["objective"] for k in kids) + + +def test_the_design_rules_are_shared_by_both_delivery_paths(): + """One set of rules, two ways to deliver the record. Duplicating them is how the + completion path and the session path would silently grow different trees. + """ + from examples.genesis._architect import ARCHITECT_PROMPT, ARCHITECT_RULES + + assert ARCHITECT_PROMPT.startswith(ARCHITECT_RULES) + assert "ONE JSON object" in ARCHITECT_PROMPT + assert "ONE JSON object" not in ARCHITECT_RULES + + +def test_a_design_session_is_given_no_shell(): + """The implementation executor gets Bash because it has to run the suite it is + judged by. An architect designs and does not implement, and a shell is how a + design session becomes an implementation session by accident. + """ + from examples.genesis._architect_session import ArchitectSession + + command = ArchitectSession(model="m")._command("hi") + allowed = command[command.index("--allowedTools") + 1] + assert "Bash" not in allowed + assert "Bash" in command[command.index("--disallowedTools") + 1] From 6ae420784e2b696948c99d99c000fced2563fc0d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 08:32:48 +0000 Subject: [PATCH 19/42] Genesis: the manager and the context extractor are sessions too `agents/manager.ex` opens with "The Manager does NOT implement features directly" and lists five jobs: analyse, plan, delegate, **validate results**, report completion. `agents/context_extractor.ex` is Mode A's root agent and the reason Mode A exists -- a repository with code and no CONTEXT.md cannot be worked on by recursive delegation, because the routing table is the map. Both are sessions with tools upstream. Both were one completion returning JSON here. Two of the Manager's five jobs are model calls in this port, and each now has a session behind it. Planning and delegation. The completion decided from the routing table and the record alone. The routing table exists so a parent *need not* investigate the subtree -- upstream says so -- but need not is not cannot, and the Manager is the role upstream gives read tools to precisely so it can check. Validating results. This is the one that was actually wrong. The completion was handed a diff rendered and truncated at 12 000 characters; a reviewer judging a truncated rendering of the work is not reviewing the work. `ReviewSession` runs over the parent's state with the child's edits applied, so Read, Glob and Grep reach the files the child really wrote. The extractor's gap was already written down in `_extract._ask`: "Upstream it reads what it needs with a tool; an agent here has none, so the node's own files travel in the prompt" -- eight files of six thousand characters, and the run before that cap invented an API surface off the file names. It reads the code now. Its record can also carry the sections upstream lists for it and an architect has no use for: Design Decisions, Notes for Agents, Dependencies, Test Strategy, on upstream's own rule that a section earns its place if it "would save an agent from re-investigating or re-discovering something". The session machinery moves to `_session.AgentSession` and the architect sits on it, so there is one copy of materialize / run / read-back rather than two that drift. Tool sets follow `agent/tools.ex`: `:read` roles get Read/Glob/Grep plus the one write a read-only agent does make (its own record), `:read_write` adds Edit. Nobody but the executor gets Bash -- the implementer has to run the suite it is judged by, and a design or review session that can run things is one that starts implementing. Deliverables are files, and lines rather than JSON: a plan of five children whose fourth line is malformed should delegate four, not none. A verdict that does not say ACCEPT or REJECT is not a rejection -- the child did the work, and a reviewer that cannot speak is not evidence against it. Still this port's standing difference, stated rather than hidden: the recursion lives in the driver. Upstream's agents spawn subagents and recurse inside the session; here one session does one node and the phase or the delegation policy drives the next. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/genesis/_architect_session.py | 136 ++------- examples/genesis/_extract.py | 29 +- examples/genesis/_roles.py | 300 +++++++++++++++++++ examples/genesis/_session.py | 192 ++++++++++++ examples/genesis/genesis_recursive_worlds.py | 50 +++- tests/test_genesis_example.py | 116 ++++++- 6 files changed, 705 insertions(+), 118 deletions(-) create mode 100644 examples/genesis/_roles.py create mode 100644 examples/genesis/_session.py diff --git a/examples/genesis/_architect_session.py b/examples/genesis/_architect_session.py index de9cce1c..0a2cc610 100644 --- a/examples/genesis/_architect_session.py +++ b/examples/genesis/_architect_session.py @@ -43,16 +43,9 @@ from __future__ import annotations -import json -import os -import shutil -import subprocess -import tempfile from typing import Dict, List, Mapping, Optional, Sequence, Tuple -from agentdescent.filetree import materialize - -from .._common import cli_env +from ._session import READ_WRITE_TOOLS, AgentSession from ._world import CONTEXT_FILE, normalise, parse_route_sizes, parse_routes __all__ = ["ARCHITECT_SESSION_BRIEF", "ArchitectSession"] @@ -102,116 +95,47 @@ class ArchitectSession: def __init__(self, *, frozen: Sequence[str] = (), binary: str = "claude", model: str = "", max_turns: int = 0, timeout: float = 900.0, root: Optional[str] = None, thinking_tokens: int = 0): - self._frozen = tuple(frozen) - self._binary = binary - self._model = model - self._max_turns = max_turns or self.TURNS - self._timeout = timeout - self._root = root - #: A per-turn reasoning cap for the session, or 0 to leave the CLI's own. - #: Upstream treats reasoning strength as a per-model setting rather than a - #: constant -- a model profile carries `reasoning_effort` beside `max_tokens` - #: and `concurrency` (`config/schema/definitions.ex`) -- and on a coding-plan - #: endpoint this is the difference between a design turn that thinks for - #: seconds and one that thinks for minutes. It bounds thinking; it does not - #: disable it, which is a different and worse lever: turning reasoning off - #: changes what the model produces, not only how long it takes. - self._thinking_tokens = max(0, int(thinking_tokens)) - #: Sessions run, and what came back. - self.sessions = 0 - self.failed = 0 - self.turns = 0 + # `:read_write` upstream, minus the shell -- see `_session.READ_WRITE_TOOLS` + # for why the executor gets Bash and a design session does not. + self.session = AgentSession( + tools=READ_WRITE_TOOLS, frozen=frozen, binary=binary, model=model, + max_turns=max_turns or self.TURNS, timeout=timeout, root=root, + thinking_tokens=thinking_tokens) #: Files a session wrote that were not its own record, discarded. An architect - #: that starts writing code is the failure mode this counts. + #: that starts writing code is the failure mode this counts. The session runs + #: in a throwaway copy, so a stray costs nothing but is worth seeing. self.strays = 0 + # The phase reads these off the designer; keeping them as properties means one + # set of counters rather than two that can disagree. + @property + def sessions(self) -> int: + return self.session.sessions + + @property + def failed(self) -> int: + return self.session.failed + + @property + def turns(self) -> int: + return self.session.turns + def __call__(self, state: Mapping[str, str], path: str, rules: str, ) -> Tuple[Optional[str], List[Dict[str, object]]]: path = normalise(path) record_path = f"{path}/{CONTEXT_FILE}" if path else CONTEXT_FILE prefix = f"{path}/" if path else "" - workspace = tempfile.mkdtemp(prefix="genesis-arch-", dir=self._root) - try: - before = dict(state) - materialize(before, workspace) - self._write_settings(workspace) - self.sessions += 1 - prompt = ARCHITECT_SESSION_BRIEF.format( - rules=rules, record_path=record_path, path_prefix=prefix) - if not self._run(workspace, prompt): - self.failed += 1 - record = self._read(workspace, record_path, before) - finally: - shutil.rmtree(workspace, ignore_errors=True) - if record is None: + prompt = ARCHITECT_SESSION_BRIEF.format( + rules=rules, record_path=record_path, path_prefix=prefix) + got = self.session.run(state, prompt, read=[record_path]) + body = (got.get(record_path) or "").strip() + if not body: return None, [] + record = body + "\n" return record, _children(record, prefix) - # -- internals --------------------------------------------------------- - - def _write_settings(self, workspace: str) -> None: - """Deny the frozen globs by name, as the implementation executor does.""" - deny: List[str] = [] - for pattern in self._frozen: - deny += [f"Write({pattern})", f"Edit({pattern})", - f"NotebookEdit({pattern})"] - settings = {"permissions": {"deny": deny, "additionalDirectories": []}} - target = os.path.join(workspace, ".claude") - os.makedirs(target, exist_ok=True) - with open(os.path.join(target, "settings.local.json"), "w", - encoding="utf-8") as handle: - json.dump(settings, handle) - - def _command(self, prompt: str) -> List[str]: - # No Bash, no TodoWrite: this session reads and writes one file. Glob and Grep - # stay because a node is designed against what is already in the tree. - command = [self._binary, "-p", prompt, - "--output-format", "json", - "--permission-mode", "acceptEdits", - "--max-turns", str(self._max_turns), - "--allowedTools", "Read,Write,Edit,Glob,Grep", - "--disallowedTools", "Bash,WebFetch,WebSearch,Task"] - if self._model: - command += ["--model", self._model] - return command - - def _run(self, workspace: str, prompt: str) -> bool: - try: - env = cli_env() - if self._thinking_tokens: - env["MAX_THINKING_TOKENS"] = str(self._thinking_tokens) - out = subprocess.run(self._command(prompt), cwd=workspace, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - timeout=self._timeout, env=env) - except Exception: # noqa: BLE001 - a dead session costs its node - return False - try: - report = json.loads(out.stdout.decode("utf-8", "replace") or "{}") - self.turns += int(report.get("num_turns") or 0) - return not report.get("is_error", out.returncode != 0) - except Exception: # noqa: BLE001 - no JSON is not a reason to lose a record - return out.returncode == 0 - - def _read(self, workspace: str, record_path: str, - before: Mapping[str, str]) -> Optional[str]: - """The record the session wrote, or `None`; everything else it wrote is a stray. - - A session reported as failed may still have written the file before it died -- - the turn limit, in particular, lands *after* the work -- so the tree is read - either way and the file is what decides. - """ - try: - with open(os.path.join(workspace, record_path), encoding="utf-8") as handle: - record = handle.read() - except OSError: - record = "" - self.strays += _strays(workspace, record_path, before) - record = record.strip() - return (record + "\n") if record else None - def summary(self) -> str: - return (f"sessions={self.sessions} failed={self.failed} " - f"turns={self.turns} strays={self.strays}") + return f"{self.session.summary()} strays={self.strays}" def _children(record: str, prefix: str) -> List[Dict[str, object]]: diff --git a/examples/genesis/_extract.py b/examples/genesis/_extract.py index c6b6a265..b29ae5a0 100644 --- a/examples/genesis/_extract.py +++ b/examples/genesis/_extract.py @@ -77,8 +77,15 @@ class ExtractPhase: def __init__(self, complete, *, contracts: Sequence[str] = (), max_depth: int = 4, max_nodes: int = 24, root_path: str = "", skip: Sequence[str] = (), - file_chars: int = 6_000, max_files: int = 8): + file_chars: int = 6_000, max_files: int = 8, session=None): self._complete = complete + #: An :class:`~examples.genesis._roles.ExtractSession`, or None. Given one, a + #: node is described by an agent that **reads the code with a tool**, the way + #: upstream's ContextExtractor does -- which is the gap the comment in `_ask` + #: names: without tools the node's files have to travel in the prompt, capped + #: at eight files of six thousand characters, and the run before that cap + #: existed invented an API surface off the file names alone. + self._session = session self._contracts = tuple(contracts) self._max_depth = max_depth self._max_nodes = max_nodes @@ -142,14 +149,20 @@ def _ask(self, state: Mapping[str, str], path: str, objective: str, # reads what it needs with a tool; an agent here has none, so the node's own # files travel in the prompt -- and the first run without them invented an API # surface off the file names, swapping what two modules do. + situated = world.situate(state, contracts=self._contracts) + if self._session is not None: + record = self._session(state, path, objective, context=situated) + if record is None: + self.unparsed += 1 + return None + return self._note_shadowed(record, state, children) sources = "\n\n".join( f"# {key}\n```\n{_clip(state[key], self._file_chars)}\n```" for key in here[:self._max_files]) or "(this node owns no file)" try: reply = self._complete(EXTRACTOR_PROMPT.format( path=path or "./", objective=objective, listing=listing, - sources=sources, - context=world.situate(state, contracts=self._contracts))) or "" + sources=sources, context=situated)) or "" except Exception: # noqa: BLE001 - one dead call costs one node self.unparsed += 1 return None @@ -157,6 +170,16 @@ def _ask(self, state: Mapping[str, str], path: str, objective: str, if record is None: self.unparsed += 1 return None + return self._note_shadowed(record, state, children) + + def _note_shadowed(self, record: str, state: Mapping[str, str], + children: Sequence[str]) -> Optional[str]: + """The two things a record must say about the directory as found. + + Both paths -- the completion and the session -- end here, because neither a + name collision nor a child missing from the table is something to ask an agent + to remember: they are facts about `state` the caller can see. + """ shadowed = [c for c in children if shadowed_by_module(state, c)] if record and shadowed: # The guard in `_is_node` stops this being *created*; a repository handed diff --git a/examples/genesis/_roles.py b/examples/genesis/_roles.py new file mode 100644 index 00000000..a7f47b51 --- /dev/null +++ b/examples/genesis/_roles.py @@ -0,0 +1,300 @@ +"""The Manager and the ContextExtractor as sessions, on :mod:`._session`. + +Two of upstream's root-capable roles, each of which this port ran as a single +completion returning JSON. + +**Manager** (``agents/manager.ex``) is the orchestrator: its moduledoc opens with "The +Manager does NOT implement features directly" and lists five jobs -- analyse the +objective, plan the work, delegate to subagents, **validate results and handle +conflicts**, report completion. It is ``:read_write`` and ``delegation_level :high``, +and in Mode B it is the *second* root agent: the Architect designs and commits, the +Manager starts from that commit and fills the tree in. + +Two of those five jobs are model calls in this port, and both get a session here: + +* *plan and delegate* -- :class:`ManagerSession`. The completion decided where to + delegate from the routing table and the record alone. A session can **look**: the + routing table is there so a parent need not investigate (upstream's own words: "you + don't need to investigate the subtree first"), but *need not* is not *cannot*, and + the Manager is the role upstream gives read tools to precisely so it can check. +* *validate results* -- :class:`ReviewSession`. The completion was handed a rendered + diff, truncated at 12 000 characters. A session reads **the files the child actually + wrote**, in place, with Glob and Grep. That difference is the whole point of the + role: a reviewer judging a truncated rendering of work is not reviewing the work. + +**ContextExtractor** (``agents/context_extractor.ex``) is Mode A's root agent, and the +reason Mode A exists: a repository with code and no ``CONTEXT.md`` cannot be worked on +by recursive delegation at all, because the routing table *is* the map. It is ``:read`` +-- it may not change the code -- but it writes ``CONTEXT.md``, which is why upstream's +read-only tool set keeps ``context_write``/``context_edit``. :class:`ExtractSession` +is that: read tools plus permission to write exactly one record. + +Its record is richer than an architect's, and upstream says why -- the sections it +lists include Design Decisions, **Notes for Agents** ("this file is generated, don't +split it"), Dependencies, Test Strategy and Status, on the rule that a section earns +its place if it "would save an agent from re-investigating or re-discovering +something". An architect inventing a directory has nothing to record there yet; an +extractor reading one does. + +Both keep this port's standing difference: **the recursion is in the driver.** One +session does one node, and the phase or the delegation policy drives the next. +""" + +from __future__ import annotations + +import re +from typing import Callable, List, Optional, Sequence, Tuple + +from ._session import (READ_ONLY_TOOLS, SCRATCH_DIR, AgentSession) +from ._world import CONTEXT_FILE, normalise + +__all__ = ["EXTRACT_BRIEF", "MANAGER_BRIEF", "REVIEW_BRIEF", + "ExtractSession", "ManagerSession", "ReviewSession"] + + +_PLAN = f"{SCRATCH_DIR}/plan.md" +_VERDICT = f"{SCRATCH_DIR}/verdict.md" + +#: One delegation per line, `path -> objective`. A line and not JSON because the +#: failure this whole file exists to remove is "the structured reply did not parse": +#: a plan of five children where the fourth line is malformed should delegate four +#: children, not none. +_PLAN_LINE = re.compile( + r"""^\s*[-*]?\s*`?\s*(?P[A-Za-z0-9._\-/]+?)\s*/?`?\s* + (?:[-=]+>|→|:)\s*(?P.+?)\s*$""", re.VERBOSE) + +MANAGER_BRIEF = """You are a manager agent in a recursive software world, situated at \ +the repository path `{path}`. You own everything under it. You do not write code. + +{context} + +This node's routing table says its children are: {routes} + +THE OBJECTIVE +{objective} + +Decide whether to delegate to more specific paths inside your own subtree, or to \ +handle this here. You have read tools -- the routing table means you do not *have* to \ +investigate the subtree, but you may, and a delegation aimed at the wrong child costs \ +a whole episode. + +Rules: +- You are ACCOUNTABLE for all code under `{path}`, and delegating does not discharge \ +that. The files AT `{path}` itself are nobody else's to write: after your children \ +return you get one more turn to write them. +- A node is a **directory**, never a file. `src/frontend` is a node; \ +`src/frontend/lexer.py` is a file belonging to the agent situated at `src/frontend`, \ +and delegating to it is refused. +- Delegate only inside `{path}`. A path outside your subtree is refused and counted. + +--- HOW TO DELIVER THIS --- + +Write your plan to `{plan}`, one delegation per line, and stop: + + -> + +An empty file means you are handling this node yourself. Do not reply with the plan; \ +what you wrote is read from the working tree, and a reply is not read at all. Change \ +no other file.""" + +REVIEW_BRIEF = """You are the parent agent at `{path}`, reviewing what a child \ +returned before you accept it. You are ACCOUNTABLE for all code under your node. + +{context} + +THE OBJECTIVE +{objective} + +The child situated at `{child}` has written its work into this working tree. **Read \ +the files it wrote.** They are on disk, and you have Read, Glob and Grep -- do not \ +judge from the summary below, which is only a list of what changed: + +{changed} + +Reject only for something that is wrong: code that cannot run, an import that does not \ +resolve, a contract the record forbids, a file written where a child had no business \ +writing. Do not reject for style, for work that is merely incomplete -- partial \ +progress is accepted here -- or for something you would rather have been done \ +differently. + +--- HOW TO DELIVER THIS --- + +Write your verdict to `{verdict}` and stop. The first line is exactly `ACCEPT` or \ +`REJECT`; if you reject, the lines after it say why, concretely enough that the child \ +can act on it. Change no other file.""" + +EXTRACT_BRIEF = """You are a context extractor: an expert software architect reading \ +an existing codebase and writing the map others navigate it by. You are situated at \ +`{path}`. **You may not change the code** -- you read it and you describe it. + +{context} + +THE OBJECTIVE THIS REPOSITORY SERVES +{objective} + +Read what is actually in `{path}` and write its `CONTEXT.md`. You are describing what \ +is there, not what should be: if the code went somewhere no design anticipated, the \ +code is what is true. + +Sections, in this order. The first four are required; include the rest when they would \ +save a later agent from re-investigating something, and leave them out when they would \ +not: + +- `## Intent` -- what this directory is for. +- `## API Surface` -- the files this node owns and what each exposes. +- `## Constraints` -- rules for code here. Omit what the parent already says. +- `## Routing Table` -- one line per child directory that exists, written \ +`- ./{path_prefix}/ ( files) -> `. +- `## Design Decisions` -- why something is the way it is, where that is not obvious. +- `## Known Issues` -- gotchas, subtle bugs, tricky behaviour. Where two things \ +collide and one of them has to go, say so and say which. +- `## Notes for Agents` -- what would waste an agent's time. "This file is generated, \ +do not split it" is the shape. +- `## Dependencies` -- what is needed beyond the package manager. +- `## Test Strategy` -- how this directory is tested, and what is not covered. + +Only directories that **exist** go in the routing table: you are finding children, not \ +inventing them. + +--- HOW TO DELIVER THIS --- + +Write the record to `{record}` and stop. Create no other file, change no code, and do \ +not reply with the record -- what you wrote is read from the working tree.""" + + +class ManagerSession: + """``manager(brief) -> [Delegation]``, as a session that may read the subtree.""" + + def __init__(self, delegation_cls, **kwargs): + self._delegation = delegation_cls + kwargs.setdefault("tools", READ_ONLY_TOOLS + ("Write",)) + self.session = AgentSession(**kwargs) + + def __call__(self, brief) -> Sequence[object]: + path = normalise(brief.world.path) + routes = brief.world.routing(brief.state) + prompt = MANAGER_BRIEF.format( + path=path or "./", context=brief.context, plan=_PLAN, + routes=", ".join(f"`{r}/`" for r in routes) or "(none yet)", + objective=brief.objective) + got = self.session.run(brief.state, prompt, read=[_PLAN]) + return [self._delegation(p, o) for p, o in _plan(got.get(_PLAN, ""))] + + def summary(self) -> str: + return self.session.summary() + + +class ReviewSession: + """``review(parent, returned) -> ("rejected", why) | None``, reading real files. + + The rejection shape is the one :func:`examples.genesis._review.chain_reviews` + already composes, so this drops in beside the deterministic checks. + """ + + def __init__(self, *, contracts: Sequence[str] = (), **kwargs): + self._contracts = tuple(contracts) + kwargs.setdefault("tools", READ_ONLY_TOOLS + ("Write",)) + self.session = AgentSession(**kwargs) + #: How many children were read, and how many were sent back. + self.reviewed = 0 + self.rejected = 0 + self.unparsed = 0 + + def __call__(self, parent, returned) -> Optional[Tuple[str, str]]: + work = [e for e in returned + if e.kind == "work" and e.content is not None] + if not work: + return None # bookkeeping only: nothing to read + candidate = dict(parent.state) + for edit in work: + candidate[edit.path] = edit.content + child = _child_of(work) + prompt = REVIEW_BRIEF.format( + path=normalise(parent.world.path) or "./", child=child, + context=parent.world.situate(parent.state, contracts=self._contracts), + objective=parent.objective, verdict=_VERDICT, + changed="\n".join(f"- {e.path}" for e in work[:60])) + self.reviewed += 1 + got = self.session.run(candidate, prompt, read=[_VERDICT]) + verdict, reason = _verdict(got.get(_VERDICT, "")) + if verdict != "reject": + # An unparseable verdict is not a rejection: the child did the work, and + # a reviewer that cannot speak is not evidence against it. + self.unparsed += int(verdict is None) + return None + self.rejected += 1 + return ("rejected", f"parent review: {reason or 'unspecified'}") + + def summary(self) -> str: + return (f"{self.session.summary()} read={self.reviewed} " + f"rejected={self.rejected} unparsed={self.unparsed}") + + +class ExtractSession: + """``extract(state, path, objective) -> record | None`` for one existing node.""" + + def __init__(self, **kwargs): + # `:read` upstream, plus the one thing a read-only agent still writes: its + # own `CONTEXT.md` (`agent/tools.ex` keeps `context_write` in the read-only + # set for exactly this). Write is granted; the brief scopes it to one file + # and the caller reads back only that file. + kwargs.setdefault("tools", READ_ONLY_TOOLS + ("Write",)) + self.session = AgentSession(**kwargs) + + def __call__(self, state, path: str, objective: str, *, + context: str = "") -> Optional[str]: + path = normalise(path) + record = f"{path}/{CONTEXT_FILE}" if path else CONTEXT_FILE + prompt = EXTRACT_BRIEF.format( + path=path or "./", path_prefix=f"{path}/" if path else "", + objective=objective, context=context, record=record) + got = self.session.run(state, prompt, read=[record]) + body = (got.get(record) or "").strip() + return (body + "\n") if body else None + + def summary(self) -> str: + return self.session.summary() + + +# -- readers -------------------------------------------------------------------- + +def _plan(body: str) -> List[Tuple[str, str]]: + """``(path, objective)`` per usable line; a malformed line costs only itself.""" + out: List[Tuple[str, str]] = [] + seen = set() + for line in (body or "").splitlines(): + if not line.strip() or line.lstrip().startswith("#"): + continue + match = _PLAN_LINE.match(line) + if not match: + continue + path = normalise(match.group("path")) + if path and path not in seen: + seen.add(path) + out.append((path, match.group("objective").strip())) + return out + + +def _verdict(body: str) -> Tuple[Optional[str], str]: + """``("accept"|"reject"|None, why)``. The first non-empty line decides.""" + lines = [l.strip() for l in (body or "").splitlines()] + lines = [l for l in lines if l] + if not lines: + return None, "" + head = lines[0].strip("`*# ").upper() + reason = " ".join(lines[1:]).strip() + if head.startswith("REJECT"): + return "reject", reason + if head.startswith("ACCEPT"): + return "accept", reason + return None, reason + + +def _child_of(work) -> str: + """The deepest directory every edit in `work` sits under.""" + parts: Optional[List[str]] = None + for edit in work: + here = normalise(edit.path).split("/")[:-1] + parts = here if parts is None else [ + a for a, b in zip(parts, here) if a == b] + return "/".join(parts or []) or "./" diff --git a/examples/genesis/_session.py b/examples/genesis/_session.py new file mode 100644 index 00000000..21a801c4 --- /dev/null +++ b/examples/genesis/_session.py @@ -0,0 +1,192 @@ +"""Upstream's agent roles as Claude Code sessions, and the machinery they share. + +Every EvoGit agent is a session with tools. ``EvoGit.Agent`` is "an agent session loop +template that manages a single agent session, handling tool loops, timeouts, and +graceful recovery", and ``agent/tools.ex`` hands out two sets by ``agent_type``: + +* ``:read_write`` -- Architect, Manager, Executor -- gets ``file_read``, + ``file_create``, ``file_write``, ``file_edit``, ``make_dir``, ``context_read`` / + ``context_write`` / ``context_edit``, ``run_bash``, ``ripgrep``, ``glob``, + ``list_dir``, ``search_context``, ``search_history`` and the skill tools. +* ``:read`` -- ContextExtractor, Investigator -- gets the read half, *plus* + ``context_write`` / ``context_edit`` and ``run_bash``, because a read-only agent + still has to write ``CONTEXT.md`` in its own repository and still has to be able to + run the tests. + +This port had every one of them as a single completion returning JSON. That is a real +simplification with two measured costs, and they are the same for each role: + +* **It is the shape a reasoning model is slowest at.** One reply carries the whole + answer, so the model thinks once, at length. On a coding-plan endpoint a single + architect node took 148-264 s and ~15 000 output tokens, where the same domain on an + endpoint that emitted ~800 tokens per call designed 23 nodes in under three minutes. +* **A truncated reply is not a short answer, it is no answer.** One run reported + ``architect designed 0 nodes, 1 replies unusable`` and then grew a whole phase 2 + against no tree at all, because the JSON stopped mid-word. + +A session has neither problem: it works in small turns, and its answer is a **file**, +so a file that exists is an answer that parses. What it also has, and the completion +never did, is the thing that makes these roles what upstream says they are -- a Manager +that "validates results" can *read the files the child wrote* instead of a diff +someone rendered for it, and a ContextExtractor whose whole job is "read code, write +the tree" can actually read the code. + +One deliberate difference from upstream throughout: **the recursion stays in the +driver.** Upstream's agents spawn subagents and recurse inside the session; here one +session does one node's worth of work and the phase or the delegation policy drives the +next step. That keeps the level-parallelism, keeps each session small enough to watch, +and keeps the spatial contract checkable per node. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import tempfile +from typing import Dict, List, Mapping, Optional, Sequence + +from agentdescent.filetree import materialize + +from .._common import cli_env + +__all__ = ["AgentSession", "READ_ONLY_TOOLS", "READ_WRITE_TOOLS", "SCRATCH_DIR"] + + +#: Where a session that has to *answer* rather than edit puts its answer. +#: +#: A Manager decides and a reviewer judges; neither produces a file the repository +#: keeps. Upstream those answers are tool calls -- ``subagent_manager``, an accept or +#: reject -- and this port has no tool bus to carry them, so the session writes the +#: answer to a scratch path and the caller reads it back. The path is outside the +#: project by construction, so nothing here can be mistaken for work: `_read_tree` +#: skips it, and the workspace is deleted either way. +SCRATCH_DIR = ".genesis" + +#: What upstream's `:read` agents get, mapped onto the CLI's tool names. No Write: +#: a role that only has to look does not get to change anything, and the one job a +#: read-only agent does write -- `CONTEXT.md` -- is granted per role rather than here. +READ_ONLY_TOOLS = ("Read", "Glob", "Grep") + +#: What upstream's `:read_write` agents get, minus the shell. Bash is granted per role, +#: and only to the executor: the point of a session rather than a completion is that +#: the *implementer* can run the suite it is judged by. A design or review session that +#: can run things is one that starts implementing. +READ_WRITE_TOOLS = ("Read", "Write", "Edit", "Glob", "Grep") + + +class AgentSession: + """One headless Claude Code session in a throwaway copy of the repository. + + The shared half of every role below: put the state on disk, run the session with + the tools that role is entitled to, and read back the paths the caller asked for. + What the session *decided* is whatever is in those paths; what it says in its reply + is not read, because a reply is the thing that truncates. + """ + + def __init__(self, *, tools: Sequence[str] = READ_WRITE_TOOLS, + frozen: Sequence[str] = (), binary: str = "claude", + model: str = "", max_turns: int = 40, timeout: float = 900.0, + root: Optional[str] = None, thinking_tokens: int = 0, + allow_bash: bool = False): + self._tools = tuple(tools) + self._frozen = tuple(frozen) + self._binary = binary + self._model = model + self._max_turns = max(1, int(max_turns)) + self._timeout = timeout + self._root = root + #: A per-turn reasoning cap, or 0 to leave the CLI's own. Upstream carries + #: reasoning strength per model profile (`reasoning_effort`, beside + #: `max_tokens` and `concurrency`) rather than fixing it, and on a coding-plan + #: endpoint this was 497s against 203-242s for the same design, same record. + #: It bounds reasoning; it does not disable it, which is a different and worse + #: lever -- that changes what the model produces, not only how long it takes. + self._thinking_tokens = max(0, int(thinking_tokens)) + self._allow_bash = allow_bash + #: Sessions run, and what came back. + self.sessions = 0 + self.failed = 0 + self.turns = 0 + + def run(self, state: Mapping[str, str], prompt: str, *, + read: Sequence[str]) -> Dict[str, str]: + """Run one session over `state`; return the `read` paths that came back. + + A path the session did not write is simply absent from the result. A session + reported as failed is still read: the turn limit in particular lands *after* + the work, so the file is what decides, not the exit status. + """ + workspace = tempfile.mkdtemp(prefix="genesis-sess-", dir=self._root) + try: + materialize(dict(state), workspace) + self._write_settings(workspace) + self.sessions += 1 + if not self._invoke(workspace, prompt): + self.failed += 1 + out: Dict[str, str] = {} + for rel in read: + try: + with open(os.path.join(workspace, rel), encoding="utf-8") as fh: + body = fh.read() + except (OSError, UnicodeDecodeError): + continue + if body.strip(): + out[rel] = body + return out + finally: + shutil.rmtree(workspace, ignore_errors=True) + + def summary(self) -> str: + return (f"sessions={self.sessions} failed={self.failed} " + f"turns={self.turns}") + + # -- internals --------------------------------------------------------- + + def _write_settings(self, workspace: str) -> None: + """Deny the frozen globs by name, inside the sandbox as well as outside it.""" + deny: List[str] = [] + for pattern in self._frozen: + deny += [f"Write({pattern})", f"Edit({pattern})", + f"NotebookEdit({pattern})"] + settings = {"permissions": {"deny": deny, "additionalDirectories": []}} + target = os.path.join(workspace, ".claude") + os.makedirs(target, exist_ok=True) + with open(os.path.join(target, "settings.local.json"), "w", + encoding="utf-8") as handle: + json.dump(settings, handle) + + def _command(self, prompt: str) -> List[str]: + tools = list(self._tools) + if self._allow_bash and "Bash" not in tools: + tools.append("Bash") + denied = ["WebFetch", "WebSearch", "Task"] + if not self._allow_bash: + denied.insert(0, "Bash") + command = [self._binary, "-p", prompt, + "--output-format", "json", + "--permission-mode", "acceptEdits", + "--max-turns", str(self._max_turns), + "--allowedTools", ",".join(tools), + "--disallowedTools", ",".join(denied)] + if self._model: + command += ["--model", self._model] + return command + + def _invoke(self, workspace: str, prompt: str) -> bool: + env = cli_env() + if self._thinking_tokens: + env["MAX_THINKING_TOKENS"] = str(self._thinking_tokens) + try: + out = subprocess.run(self._command(prompt), cwd=workspace, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=self._timeout, env=env) + except Exception: # noqa: BLE001 - a dead session costs its turn + return False + try: + report = json.loads(out.stdout.decode("utf-8", "replace") or "{}") + self.turns += int(report.get("num_turns") or 0) + return not report.get("is_error", out.returncode != 0) + except Exception: # noqa: BLE001 - no JSON is not a reason to lose the work + return out.returncode == 0 diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index b1f55d01..59df5526 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -88,7 +88,7 @@ from examples._common import (ConcurrencyGauge, add_standard_args, budget_kwargs, completion_for, confirm, report_engine, worker_count) -from ._delegation import RecursiveDelegation +from ._delegation import Delegation, RecursiveDelegation from . import _domain as minilang from . import _fly as fly from . import _jqx as jqx @@ -97,6 +97,7 @@ from ._architect import (ArchitectPhase, harness_record, misaligned, missing_sections) from ._architect_session import ArchitectSession +from ._roles import ExtractSession, ManagerSession, ReviewSession from ._claude_code import ClaudeCodeExecutor, claude_code_available from ._extract import ExtractPhase from ._judge import ParentJudge @@ -303,6 +304,14 @@ def build_parser() -> argparse.ArgumentParser: "profile (`reasoning_effort`) rather than as a constant. " "Measured on one fly node against a coding-plan endpoint: " "uncapped 497s, capped at 2048 203-242s, same record") + parser.add_argument("--agent-sessions", action="store_true", + help="run the MANAGER and (with --mode a) the CONTEXT " + "EXTRACTOR as Claude Code sessions with tools, the way " + "upstream runs every agent. The manager may read its " + "subtree before delegating, and reviews the child's real " + "files rather than a diff truncated at 12k characters; " + "the extractor reads the code instead of being shown " + "eight files in its prompt. Needs the claude CLI") parser.add_argument("--architect-session", action="store_true", help="run phase 1's architect as a Claude Code session that " "WRITES each CONTEXT.md with a file tool, rather than as " @@ -501,6 +510,9 @@ def main(argv=None) -> None: "and a rule-based reader would be describing what it was told") return extractor = ExtractPhase(complete, contracts=spec.CONTRACTS, + session=(ExtractSession(frozen=spec.FROZEN, + **_session_kwargs()) + if use_sessions else None), max_depth=args.depth + 1, skip=tuple(p.split("/")[0] for p in spec.FROZEN if "/" in p)) @@ -510,6 +522,19 @@ def main(argv=None) -> None: record = initial[f"{path}/{CONTEXT_FILE}" if path else CONTEXT_FILE] print(f" {path or './':<24} {len(parse_routing(record))} routes") + def _session_kwargs(): + return dict(model=(args.executor_model or args.model or ""), + timeout=float(getattr(args, "timeout", None) or 900.0), + thinking_tokens=args.thinking_tokens) + + #: Whether the manager and extractor run as sessions. Decided once: a run that + #: asked for them and has no CLI should say so once, not once per node. + use_sessions = bool(args.agent_sessions) and complete is not None + if use_sessions and not claude_code_available(): + print("--agent-sessions needs the `claude` CLI on PATH; the manager and the " + "extractor fall back to one completion each", file=sys.stderr) + use_sessions = False + architect = None designer = None if args.mode == "b": @@ -528,10 +553,8 @@ def main(argv=None) -> None: print("--architect-session needs the `claude` CLI on PATH; phase 1 " "falls back to one completion per node", file=sys.stderr) else: - designer = ArchitectSession( - frozen=spec.FROZEN, model=(args.executor_model or args.model or ""), - timeout=float(getattr(args, "timeout", None) or 900.0), - thinking_tokens=args.thinking_tokens) + designer = ArchitectSession(frozen=spec.FROZEN, + **_session_kwargs()) architect = ArchitectPhase(complete, contracts=spec.CONTRACTS, session=designer, max_depth=args.depth, max_nodes=args.nodes, @@ -564,8 +587,16 @@ def main(argv=None) -> None: # results, run the tests, and reject anti-patterns it can see in the code. The # middle one is a number and was all this port had; the zero-field run is what # that cost. Both now, tests first because they are free. - code_review = (None if args.no_parent_review or complete is None else - ParentCodeReview(complete, contracts=spec.CONTRACTS)) + if args.no_parent_review or complete is None: + code_review = None + elif use_sessions: + # `manager.ex` lists "validate results" among the Manager's five jobs, and a + # reviewer handed a rendering of the work truncated at 12 000 characters is + # not validating the work. This one reads the files on disk. + code_review = ReviewSession(contracts=spec.CONTRACTS, frozen=spec.FROZEN, + **_session_kwargs()) + else: + code_review = ParentCodeReview(complete, contracts=spec.CONTRACTS) ledger = WorktreeLedger() if args.worktrees else None if ledger is not None and not git_worktrees_available(): print("Worktrees: git worktree is unavailable here -- running without it") @@ -613,7 +644,10 @@ def _refine(path, state): delegation = RecursiveDelegation( refine=None if (architect is None or args.no_refine) else _refine, rollout_factory=(None if ledger is None else lambda: Rollout(ledger)), - manager=spec.llm_manager(complete) if complete else spec.offline_manager, + manager=(ManagerSession(Delegation, frozen=spec.FROZEN, + **_session_kwargs()) if use_sessions else + spec.llm_manager(complete) if complete + else spec.offline_manager), executor=(sessions if sessions is not None else spec.llm_executor(complete) if complete else spec.offline_executor), log=log, max_depth=args.depth, max_edits=4, contracts=spec.CONTRACTS, diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index dea82589..271e561e 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -2225,7 +2225,121 @@ def test_a_design_session_is_given_no_shell(): """ from examples.genesis._architect_session import ArchitectSession - command = ArchitectSession(model="m")._command("hi") + command = ArchitectSession(model="m").session._command("hi") allowed = command[command.index("--allowedTools") + 1] assert "Bash" not in allowed assert "Bash" in command[command.index("--disallowedTools") + 1] + + +# -- the manager and the extractor as sessions ---------------------------------- + +def test_a_manager_plan_is_read_line_by_line(): + """A plan of five children whose fourth line is malformed delegates four, not + none. That is the whole reason the deliverable is lines and not JSON: the failure + these sessions exist to remove is "the structured reply did not parse". + """ + from examples.genesis._roles import _plan + + assert _plan("# heading\n" + "- `src/brain` -> build the circuits\n" + "src/world -> arena and odour\n" + "a line with no arrow at all\n" + "* src/web: the page\n") == [ + ("src/brain", "build the circuits"), + ("src/world", "arena and odour"), + ("src/web", "the page")] + assert _plan("") == [] # an empty plan means "handle it here" + + +def test_a_review_verdict_that_does_not_speak_is_not_a_rejection(): + """The child did the work; a reviewer that cannot say ACCEPT or REJECT is not + evidence against it. Same rule the completion reviewer already held. + """ + from examples.genesis._roles import _verdict + + assert _verdict("ACCEPT\nlooks fine") == ("accept", "looks fine") + assert _verdict("**REJECT**\nimport does not resolve") == ( + "reject", "import does not resolve") + assert _verdict("I think it is probably ok") == (None, "") + assert _verdict("") == (None, "") + + +def test_a_reviewing_session_is_shown_the_child_s_real_files(): + """`ReviewSession` runs over the candidate -- the parent's state with the child's + edits applied -- so Read and Grep reach the work itself. The completion reviewer + saw a diff rendered and truncated at 12 000 characters. + """ + import examples.genesis._roles as roles + + seen = {} + + class _Fake: + sessions = failed = turns = 0 + + def run(self, state, prompt, *, read): + seen["state"] = dict(state) + seen["prompt"] = prompt + return {read[0]: "ACCEPT\n"} + + def summary(self): + return "" + + review = roles.ReviewSession.__new__(roles.ReviewSession) + review._contracts = () + review.session = _Fake() + review.reviewed = review.rejected = review.unparsed = 0 + + class _World: + path = "src" + + def situate(self, state, contracts=()): + return "(context)" + + class _Parent: + world = _World() + state = {"src/CONTEXT.md": "# src\n"} + objective = "build it" + + class _Edit: + kind = "work" + + def __init__(self, path, content): + self.path, self.content = path, content + + assert review(_Parent(), [_Edit("src/brain/kc.py", "X = 1\n")]) is None + assert seen["state"]["src/brain/kc.py"] == "X = 1\n" + assert "src/brain/kc.py" in seen["prompt"] + + +def test_an_extractor_session_replaces_files_in_the_prompt(): + """Upstream's ContextExtractor reads code with a tool. Without one this port had + to put the node's own files in the prompt -- capped at eight files of six thousand + characters, and the run before that cap invented an API surface off file names. + """ + import inspect + + from examples.genesis._extract import ExtractPhase + from examples.genesis._roles import EXTRACT_BRIEF + + assert "session" in inspect.signature(ExtractPhase.__init__).parameters + # the record is a file it writes, not a reply it returns + assert "{record}" in EXTRACT_BRIEF + # and it is told it may not change the code + assert "may not change the code" in EXTRACT_BRIEF + + +def test_a_read_only_role_gets_no_shell_and_no_edit(): + """`agent/tools.ex` gives `:read` agents the read half plus `context_write`. The + one thing they write is a record; they do not edit code and they are not given a + shell here, which upstream grants for running tests and this port's extractor has + no use for. + """ + from examples.genesis._roles import ExtractSession, ManagerSession + + for role in (ExtractSession(model="m"), + ManagerSession(lambda p, o: (p, o), model="m")): + command = role.session._command("hi") + allowed = command[command.index("--allowedTools") + 1] + assert "Edit" not in allowed, allowed + assert "Bash" not in allowed, allowed + assert "Bash" in command[command.index("--disallowedTools") + 1] From 6a70ca45adeceaea2ac67236d1872ec443064a30 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 12:13:57 +0000 Subject: [PATCH 20/42] genesis: find what actually ends an executor episode, and fix it A fly formation run reported `sessions=52 failed=43` -- 83% of implementation episodes judged failed -- while the parent review rejected only three and the spatial contract dropped nothing. The CLI writes a transcript per session, and all 52 were on disk, so the cause was measurable rather than inferable: 27 ran into the wall (>=860s of a 900s limit), 25 killed mid-tool-call 17 had the endpoint drop the stream first (26s-830s, also mid-tool) 8 ended on a text turn, having finished 0 reached the 128-turn child budget -- the busiest made 97 Three defects, not a tuning problem: * The executor was the one role built without the run's session settings. Every other role got `**_session_kwargs()`; this one got three arguments of its own, so `--timeout 600` and `--thinking-tokens 2048` reached the architect, the manager, the reviewer and the extractor and not the role that writes the code. It ran the whole domain at a 900s default nobody chose, with no reasoning cap. * `--timeout` is the timeout on one model call (`_common` says so in its own help, and genesis defaults it to 120s). A session is a loop of many calls, so the wall is now its own flag, `--session-timeout`, printed in the run header beside the turn budget. * `turns=309` was not the truth. There is no JSON after a SIGKILL, so `num_turns` is never read for a session that hit the wall; the transcripts hold 2607 assistant turns. Timeouts are now counted apart from every other failure (`failed=43 timeout=27`), because the two have different fixes. Work was never discarded: the port reads the worktree, not the exit status, so a session interrupted at the wall still returns what it wrote. That was already right and is now covered by a test. Also, from the same transcripts: `subprocess.run(timeout=)` signals the CLI and nothing else. A session is told to run the suite, a suite run starts servers, and one was still listening two hours later with its working directory deleted. Sessions now lead their own process group, killed on the way out -- after the wall and after a clean finish alike. Two smaller things found while reading: `--mode a --agent-sessions` raised NameError, because the extractor read `use_sessions` and `_session_kwargs` from above their definitions; and `ArchitectSession.strays` reported 0 unconditionally from a helper that walked the workspace with a module its file never imported. The session owns the workspace and deletes it, so it does the counting now. Full suite green; six new tests, including one that asserts a session's grandchild does not outlive the episode that started it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- docs/algo-genesis.md | 46 +++++ examples/genesis/_architect_session.py | 21 +-- examples/genesis/_claude_code.py | 56 ++++-- examples/genesis/_session.py | 107 +++++++++-- examples/genesis/genesis_recursive_worlds.py | 63 ++++--- tests/test_genesis_example.py | 180 +++++++++++++++++++ 6 files changed, 408 insertions(+), 65 deletions(-) diff --git a/docs/algo-genesis.md b/docs/algo-genesis.md index 01be6250..28d02d8b 100644 --- a/docs/algo-genesis.md +++ b/docs/algo-genesis.md @@ -679,6 +679,52 @@ for the entry point as well, a session situated at `src/frontend` wrote only its node and said so, so the port's request channel never even had to fire (`requests=0`). The contract held inside the session rather than at the boundary. +### What actually ends an episode + +A formation run on the `fly` domain reported `sessions=52 failed=43` — 83% of +implementation episodes judged failed — while the review rejected only three and the +spatial contract dropped nothing. The CLI writes a transcript per session, so the +answer was on disk rather than in the counters, and it is not what the counters +suggested: + +| | sessions | | +|---|---:|---| +| ran into the wall (≥ 860 s of a 900 s limit) | **27** | 25 of them killed mid-tool-call | +| the endpoint dropped the stream first | 17 | 26 s to 830 s, also mid-tool | +| ended on a text turn, having finished | 8 | | +| reached the 128-turn child budget | **0** | the busiest made 97 | + +Three things follow, and each was a defect rather than a tuning problem. + +**The wall is the binding constraint, not the turn budget.** Upstream's 2 048/128 turns +are this port's numbers too and they are fine; what ends an episode here is wall-clock. +So it is `--session-timeout`, its own flag, printed in the run header beside the turn +budget — and *not* `--timeout`, which `examples/_common` documents as "seconds for one +model call" and defaults to 120 s. A session is a loop of many calls; sharing one number +between the two means either the call timeout is absurdly long or the session wall is +absurdly short. + +**The executor was the one role running on defaults.** The architect, the manager, the +reviewer and the extractor were all constructed from the run's session settings; the +executor was constructed from three arguments of its own, so `--timeout 600` and +`--thinking-tokens 2048` reached every role except the one that writes the code. It ran +that whole domain at a 900 s wall nobody had chosen, with no reasoning cap while every +other role had one. + +**A session that hits the wall is interrupted, not refuted.** The port reads the +worktree, not the exit status, so the work it did before the wall comes back and is +judged like any other — which is the right behaviour and was already the behaviour. What +the wall costs is the *report*: there is no JSON after a `SIGKILL`, so `num_turns` is +never read, and the run that reported `turns=309` had 2 607 assistant turns in its +transcripts. Timeouts are now counted separately (`failed=43 timeout=27`) because the +two failures have different fixes. + +One more thing the transcripts showed: `subprocess.run(timeout=)` signals the CLI and +nothing else. A session is told to run the suite, a suite run starts servers, and one of +them was still listening two hours later with its working directory already deleted. +Sessions now lead their own process group and the group is killed on the way out — +after the wall, and after a clean finish too. + ## Synchronous or barrier-free, and which one is upstream's `--async` switches `evolve()` to its barrier-free runtime, and it is a fair question diff --git a/examples/genesis/_architect_session.py b/examples/genesis/_architect_session.py index 0a2cc610..eba6fbc7 100644 --- a/examples/genesis/_architect_session.py +++ b/examples/genesis/_architect_session.py @@ -128,6 +128,7 @@ def __call__(self, state: Mapping[str, str], path: str, rules: str, prompt = ARCHITECT_SESSION_BRIEF.format( rules=rules, record_path=record_path, path_prefix=prefix) got = self.session.run(state, prompt, read=[record_path]) + self.strays += len(self.session.changed) body = (got.get(record_path) or "").strip() if not body: return None, [] @@ -161,23 +162,3 @@ def _children(record: str, prefix: str) -> List[Dict[str, object]]: "objective": handles or f"implement {child}", "files": declared or int(sizes.get(normalise(child), 0))}) return out - - -def _strays(workspace: str, record_path: str, before: Mapping[str, str]) -> int: - """How many files the session wrote that were not its own record.""" - count = 0 - for base, dirs, files in os.walk(workspace): - dirs[:] = [d for d in dirs if d not in (".git", ".claude", "__pycache__")] - for name in files: - rel = os.path.relpath(os.path.join(base, name), workspace) - rel = rel.replace(os.sep, "/") - if rel == record_path: - continue - try: - with open(os.path.join(base, name), encoding="utf-8") as handle: - body = handle.read() - except (UnicodeDecodeError, OSError): - continue - if before.get(rel) != body: - count += 1 - return count diff --git a/examples/genesis/_claude_code.py b/examples/genesis/_claude_code.py index 13624bac..4d5487e4 100644 --- a/examples/genesis/_claude_code.py +++ b/examples/genesis/_claude_code.py @@ -50,6 +50,7 @@ from .._common import cli_env from ._delegation import Brief, Edit +from ._session import run_cli from ._spatial import SITUATED_EDIT_PROTOCOL # noqa: F401 (documented sibling) from ._world import normalise, owns @@ -106,9 +107,17 @@ class ClaudeCodeExecutor: ROOT_TURNS = 2048 CHILD_TURNS = 128 + #: What actually ends an episode, measured. Across 52 sessions of one formation + #: run, 27 ran to this wall and 25 of those were killed mid-tool-call, 17 more had + #: the endpoint drop the stream before it, and **none** reached :attr:`CHILD_TURNS` + #: -- the busiest made 97 of its 128. The turn budget is upstream's and it is not + #: the binding constraint; this is, so it is a number the run gets to choose rather + #: than one buried in a default. + TIMEOUT = 900.0 + def __init__(self, *, frozen: Sequence[str] = (), binary: str = "claude", model: str = "", max_turns: int = 0, root_turns: int = 0, - timeout: float = 900.0, + timeout: float = 0.0, thinking_tokens: int = 0, allow_bash: bool = True, failure: str = "", root: Optional[str] = None): self._frozen = tuple(frozen) @@ -116,13 +125,21 @@ def __init__(self, *, frozen: Sequence[str] = (), binary: str = "claude", self._model = model self._max_turns = max_turns or self.CHILD_TURNS self._root_turns = root_turns or max_turns or self.ROOT_TURNS - self._timeout = timeout + self._timeout = float(timeout or self.TIMEOUT) + #: The same per-turn reasoning cap the other roles carry. It reaches the CLI + #: as an environment variable, and an executor left without it while every + #: other role had one is an executor thinking at a different length than the + #: run asked for -- which is exactly how it runs out of wall. + self._thinking_tokens = max(0, int(thinking_tokens)) self._allow_bash = allow_bash self._failure = failure self._root = root - #: Sessions run, and what came back. + #: Sessions run, and what came back. `failed` is the total; `timeouts` is the + #: part of it that was still working when the wall came, and `edits` is + #: counted for those too -- a session that ran out of time wrote what it wrote. self.sessions = 0 self.failed = 0 + self.timeouts = 0 self.edits = 0 self.requests = 0 self.turns = 0 @@ -185,27 +202,38 @@ def _run(self, workspace: str, owner: str, brief: Brief) -> bool: failure=self._failure.format(prompt=getattr(brief.task, "prompt", ""), output=(brief.output or "")[:400], reward=brief.reward) if self._failure else "") + # `cli_env` is a no-op unless this run points ANTHROPIC_BASE_URL at a + # third-party endpoint. When it does, it drops the variables by which a + # managed session tells the CLI to use the *host's* provider and ignore + # the environment -- inherited, they make every session here 401. + env = cli_env() + if self._thinking_tokens: + env["MAX_THINKING_TOKENS"] = str(self._thinking_tokens) try: turns = self._root_turns if brief.depth == 0 else self._max_turns - # `cli_env` is a no-op unless this run points ANTHROPIC_BASE_URL at a - # third-party endpoint. When it does, it drops the variables by which a - # managed session tells the CLI to use the *host's* provider and ignore - # the environment -- inherited, they make every session here 401. - out = subprocess.run(self._command(prompt, turns), cwd=workspace, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - timeout=self._timeout, env=cli_env()) + out, code, timed_out = run_cli(self._command(prompt, turns), + cwd=workspace, env=env, + timeout=self._timeout) except Exception: # noqa: BLE001 - a dead session costs its episode return False + if timed_out: + # There is no report after the wall, so this episode's turns are not in + # `turns` -- which is why the counter read 309 for a run whose transcripts + # hold 2 607. The diff is taken either way: the caller reads the worktree + # after this returns, and half-finished work is still work. + self.timeouts += 1 + return False try: - report = json.loads(out.stdout.decode("utf-8", "replace") or "{}") + report = json.loads(out.decode("utf-8", "replace") or "{}") self.turns += int(report.get("num_turns") or 0) - return not report.get("is_error", out.returncode != 0) + return not report.get("is_error", code != 0) except Exception: # noqa: BLE001 - no JSON is not a reason to lose the diff - return out.returncode == 0 + return code == 0 def summary(self) -> str: return (f"sessions={self.sessions} failed={self.failed} " - f"turns={self.turns} edits={self.edits} requests={self.requests}") + f"timeout={self.timeouts} turns={self.turns} " + f"edits={self.edits} requests={self.requests}") def _read_tree(workspace: str) -> Dict[str, str]: diff --git a/examples/genesis/_session.py b/examples/genesis/_session.py index 21a801c4..b63e065f 100644 --- a/examples/genesis/_session.py +++ b/examples/genesis/_session.py @@ -43,15 +43,17 @@ import json import os import shutil +import signal import subprocess import tempfile -from typing import Dict, List, Mapping, Optional, Sequence +from typing import Dict, List, Mapping, Optional, Sequence, Tuple from agentdescent.filetree import materialize from .._common import cli_env -__all__ = ["AgentSession", "READ_ONLY_TOOLS", "READ_WRITE_TOOLS", "SCRATCH_DIR"] +__all__ = ["AgentSession", "READ_ONLY_TOOLS", "READ_WRITE_TOOLS", "SCRATCH_DIR", + "run_cli"] #: Where a session that has to *answer* rather than edit puts its answer. @@ -76,6 +78,77 @@ READ_WRITE_TOOLS = ("Read", "Write", "Edit", "Glob", "Grep") +def run_cli(command: Sequence[str], *, cwd: str, env: Mapping[str, str], + timeout: float) -> Tuple[bytes, int, bool]: + """Run one session to completion or to the wall; ``(stdout, code, timed_out)``. + + The **process group** is the point, and `subprocess.run(timeout=)` does not have + one: it signals the CLI and nothing else, so a shell the session started outlives + the episode that started it. One run left a `python3 _cli.py serve` listening with + its working directory already deleted, because its parent was the only process that + got the signal. Here the session leads its own group and the group is killed on the + way out -- after the wall, and after a clean finish too, because a session that + leaves a server running has left it running either way. + + `timed_out` is returned rather than folded into the exit status because those are + different events with different fixes: a session that hit the wall was still + working, and one that died at 26 seconds was not. + """ + proc = subprocess.Popen(list(command), cwd=cwd, env=dict(env), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + start_new_session=True) + timed_out = False + try: + try: + out, _ = proc.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + _kill_group(proc) + try: + out, _ = proc.communicate(timeout=30) + except subprocess.TimeoutExpired: # a pipe a grandchild still holds + out = b"" + finally: + _kill_group(proc) + return out or b"", proc.returncode or 0, timed_out + + +def _kill_group(proc: "subprocess.Popen") -> None: + """SIGKILL everything the session started, itself included. Never raises.""" + try: + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + except (OSError, ProcessLookupError, PermissionError): + pass + + +def _strays(workspace: str, state: Mapping[str, str], + read: Sequence[str]) -> List[str]: + """Paths the session changed that were neither its answer nor already there. + + Called before the workspace is deleted, because that is the only moment it can + be. The scratch directory is not a stray -- it is where a session that has to + answer rather than edit puts its answer -- and neither is anything in `read`. + """ + skip = set(read) + out: List[str] = [] + for base, dirs, files in os.walk(workspace): + dirs[:] = [d for d in dirs + if d not in (".git", ".claude", "__pycache__", SCRATCH_DIR)] + for name in files: + full = os.path.join(base, name) + rel = os.path.relpath(full, workspace).replace(os.sep, "/") + if rel in skip: + continue + try: + with open(full, encoding="utf-8") as handle: + body = handle.read() + except (UnicodeDecodeError, OSError): + continue + if state.get(rel) != body: + out.append(rel) + return sorted(out) + + class AgentSession: """One headless Claude Code session in a throwaway copy of the repository. @@ -105,10 +178,18 @@ def __init__(self, *, tools: Sequence[str] = READ_WRITE_TOOLS, #: lever -- that changes what the model produces, not only how long it takes. self._thinking_tokens = max(0, int(thinking_tokens)) self._allow_bash = allow_bash - #: Sessions run, and what came back. + #: Sessions run, and what came back. `failed` is the total; `timeouts` is + #: the half of it that was still working when the wall arrived, which is the + #: only one of the two that `--session-timeout` can do anything about. self.sessions = 0 self.failed = 0 + self.timeouts = 0 self.turns = 0 + #: Paths the last session changed that it was not asked for -- set by + #: :meth:`run` before the workspace is deleted, because afterwards there is + #: nothing left to look at. A role that cares (an architect, which is meant + #: to write one record and no code) counts them. + self.changed: List[str] = [] def run(self, state: Mapping[str, str], prompt: str, *, read: Sequence[str]) -> Dict[str, str]: @@ -134,13 +215,14 @@ def run(self, state: Mapping[str, str], prompt: str, *, continue if body.strip(): out[rel] = body + self.changed = _strays(workspace, state, read) return out finally: shutil.rmtree(workspace, ignore_errors=True) def summary(self) -> str: return (f"sessions={self.sessions} failed={self.failed} " - f"turns={self.turns}") + f"timeout={self.timeouts} turns={self.turns}") # -- internals --------------------------------------------------------- @@ -179,14 +261,19 @@ def _invoke(self, workspace: str, prompt: str) -> bool: if self._thinking_tokens: env["MAX_THINKING_TOKENS"] = str(self._thinking_tokens) try: - out = subprocess.run(self._command(prompt), cwd=workspace, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - timeout=self._timeout, env=env) + out, code, timed_out = run_cli(self._command(prompt), cwd=workspace, + env=env, timeout=self._timeout) except Exception: # noqa: BLE001 - a dead session costs its turn return False + if timed_out: + # The wall, and it is worth telling apart: there is no report to read, + # so `turns` does not count what this session did. The file it may have + # written by then is still read back, which is why `run` reads either way. + self.timeouts += 1 + return False try: - report = json.loads(out.stdout.decode("utf-8", "replace") or "{}") + report = json.loads(out.decode("utf-8", "replace") or "{}") self.turns += int(report.get("num_turns") or 0) - return not report.get("is_error", out.returncode != 0) + return not report.get("is_error", code != 0) except Exception: # noqa: BLE001 - no JSON is not a reason to lose the work - return out.returncode == 0 + return code == 0 diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index 59df5526..fc9b0409 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -298,12 +298,20 @@ def build_parser() -> argparse.ArgumentParser: "domain's architect named 17 children it never reached at " "the default, so the tree came out two deep and truncated") parser.add_argument("--thinking-tokens", type=int, default=0, - help="per-turn reasoning cap for an --architect-session " - "design turn (MAX_THINKING_TOKENS); 0 leaves the CLI's " - "own. Upstream carries reasoning strength per model " - "profile (`reasoning_effort`) rather than as a constant. " + help="per-turn reasoning cap for every session role, executor " + "included (MAX_THINKING_TOKENS); 0 leaves the CLI's own. " + "Upstream carries reasoning strength per model profile " + "(`reasoning_effort`) rather than as a constant. " "Measured on one fly node against a coding-plan endpoint: " "uncapped 497s, capped at 2048 203-242s, same record") + parser.add_argument("--session-timeout", type=float, + default=ClaudeCodeExecutor.TIMEOUT, metavar="SECONDS", + help=(f"the wall on one agent session, in seconds (default " + f"{ClaudeCodeExecutor.TIMEOUT:g}). NOT --timeout, which " + "is one model call: a session is a loop of many. This is " + "what actually ends an episode -- of 52 executor sessions " + "in one fly run, 27 ran into this wall and none reached " + "the 128-turn budget")) parser.add_argument("--agent-sessions", action="store_true", help="run the MANAGER and (with --mode a) the CONTEXT " "EXTRACTOR as Claude Code sessions with tools, the way " @@ -396,8 +404,11 @@ def main(argv=None) -> None: f"up to {args.max_turns or ClaudeCodeExecutor.ROOT_TURNS} " f"turns at the root and " f"{args.max_turns or ClaudeCodeExecutor.CHILD_TURNS} below, " - f"frozen paths denied, network " - "tools off -- billed to the local CLI's credentials" + # The wall belongs beside the turn budget because it is the + # one of the two that ends episodes. Printed, a run that + # timed out 27 of 52 sessions says so at the top. + f"{args.session_timeout:g}s each, frozen paths denied, " + "network tools off -- billed to the local CLI's credentials" if args.executor == "claude-code" else "one model call returning whole files")) print("Workspace: " + ("a git worktree per episode, a commit per episode, and " @@ -504,6 +515,26 @@ def main(argv=None) -> None: "invent") return + def _session_kwargs(): + # `--timeout` is the timeout on one *model call* -- `_common` says so in its + # own help -- and a session is a loop of many calls, so its wall is its own + # flag. Sharing one number put the roles on whatever `--timeout` said while + # the executor, handed neither, sat on a default nobody in the run had chosen. + return dict(model=(args.executor_model or args.model or ""), + timeout=float(args.session_timeout or ClaudeCodeExecutor.TIMEOUT), + thinking_tokens=args.thinking_tokens) + + #: Whether the manager, the reviewer and the extractor run as sessions. Decided + #: once: a run that asked for them and has no CLI should say so once, not once per + #: node. Decided *here*, above every role that reads it -- the extractor is built + #: inside the `--mode a` branch below, and a name defined after its only use is a + #: NameError rather than the fallback it looks like. + use_sessions = bool(args.agent_sessions) and complete is not None + if use_sessions and not claude_code_available(): + print("--agent-sessions needs the `claude` CLI on PATH; the manager and the " + "extractor fall back to one completion each", file=sys.stderr) + use_sessions = False + if args.mode == "a": if complete is None: print("--mode a needs --model: extracting a Context Tree is reading code, " @@ -522,19 +553,6 @@ def main(argv=None) -> None: record = initial[f"{path}/{CONTEXT_FILE}" if path else CONTEXT_FILE] print(f" {path or './':<24} {len(parse_routing(record))} routes") - def _session_kwargs(): - return dict(model=(args.executor_model or args.model or ""), - timeout=float(getattr(args, "timeout", None) or 900.0), - thinking_tokens=args.thinking_tokens) - - #: Whether the manager and extractor run as sessions. Decided once: a run that - #: asked for them and has no CLI should say so once, not once per node. - use_sessions = bool(args.agent_sessions) and complete is not None - if use_sessions and not claude_code_available(): - print("--agent-sessions needs the `claude` CLI on PATH; the manager and the " - "extractor fall back to one completion each", file=sys.stderr) - use_sessions = False - architect = None designer = None if args.mode == "b": @@ -616,10 +634,13 @@ def _session_kwargs(): # so pointing both the SDK and the CLI at one endpoint is all it takes to put # the architect and every executor session on the same model; the flag is there # for the case where they should deliberately differ. + # The same session settings every other role gets. Without this the one role + # that does the work was the one role running on defaults: a wall it was never + # told about and no reasoning cap, while the architect, the manager, the + # reviewer and the extractor all carried the run's own numbers. sessions = ClaudeCodeExecutor(frozen=spec.FROZEN, failure=getattr(spec, "FAILURE", TEST_FAILURE), - model=args.executor_model or args.model, - max_turns=args.max_turns) + max_turns=args.max_turns, **_session_kwargs()) def _refine(path, state): """Upstream's architect Phase 3, as a hook on the accountability pass. diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index 271e561e..a148e774 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -9,7 +9,10 @@ from __future__ import annotations +import ast +import inspect import json +import os import posixpath import time @@ -26,6 +29,7 @@ from examples.genesis import _jqx as jqx from examples.genesis import _md as md from examples.genesis import _stackvm as stackvm +from examples.genesis import genesis_recursive_worlds as genesis from examples.genesis._delegation import (Brief, Delegation, Edit, RecursiveDelegation, render_edits) from examples.genesis._claude_code import (CLAUDE_CODE_BRIEF, @@ -2096,6 +2100,182 @@ def test_a_session_that_dies_costs_its_episode_and_nothing_else(tmp_path): assert edits == [] and executor.failed == 1 +#: A session that does some work and then never finishes. The wall, not the turn +#: budget, is what ends an episode in practice: of 52 executor sessions in one fly +#: run, 27 ran into the wall and the busiest reached 97 of its 128 turns. +SLOW = ''' +import os, sys, time +if "--version" in sys.argv: + print("0.0.0 (fake)"); raise SystemExit(0) +os.makedirs("src/core", exist_ok=True) +open("src/core/vectors.py", "w").write("def minimum_image(a, b, box):\\n return b\\n") +time.sleep(120) +''' + + +def test_a_session_that_runs_out_of_wall_keeps_what_it_wrote(tmp_path): + """The wall is an interruption, not a verdict. + + A session killed at the wall wrote what it wrote, and the port reads the worktree + rather than the exit status, so the work comes back. What the wall costs is the + report -- there is no JSON after a SIGKILL -- which is why the turn counter read + 309 for a run whose transcripts held 2 607 assistant turns, and why the timeout + is counted apart from every other way a session can fail. + """ + executor = ClaudeCodeExecutor(frozen=md.FROZEN, timeout=3.0, + binary=_fake_claude(tmp_path, SLOW)) + edits = executor(Brief(world=LocalWorld(version=1, path="src/core", + readonly=md.FROZEN), + objective="o", context="", state=dict(md.initial_files()), + task=md.build_tasks()[0], output="FAIL", reward=0.0, depth=2)) + assert [e.path for e in edits] == ["src/core/vectors.py"] + assert executor.failed == 1 and executor.timeouts == 1 + assert executor.turns == 0 # no report to read: the count is not the truth + + +#: A session that starts a server the way the brief tells it to run the suite, and +#: then hangs. `subprocess.run(timeout=)` signals only the CLI, so one run left a +#: `python3 _cli.py serve` listening with its working directory already deleted. +LEAKY = ''' +import os, subprocess, sys, time +if "--version" in sys.argv: + print("0.0.0 (fake)"); raise SystemExit(0) +subprocess.Popen([sys.executable, "-c", + "import os, sys, time\\n" + "open(sys.argv[1], 'w').write(str(os.getpid()))\\n" + "time.sleep(120)", "MARKER"]) +while not os.path.exists("MARKER") or not open("MARKER").read().strip(): + time.sleep(0.05) +time.sleep(120) +''' + + +def test_a_session_does_not_outlive_the_episode_that_started_it(tmp_path): + """The group is killed, not the process. + + An executor has a shell because it has to run the suite it is judged by, and a + suite run starts servers. The session leads its own process group and the group + goes when the session does -- otherwise the thing the session started is still + holding a port when the next 43 episodes come round. + """ + marker = tmp_path / "grandchild.pid" + binary = _fake_claude(tmp_path, LEAKY.replace("MARKER", str(marker))) + executor = ClaudeCodeExecutor(frozen=md.FROZEN, timeout=5.0, binary=binary) + executor(Brief(world=LocalWorld(version=1, path="src", readonly=md.FROZEN), + objective="o", context="", state=dict(md.initial_files()), + task=md.build_tasks()[0], output="FAIL", reward=0.0, depth=1)) + assert executor.timeouts == 1 + pid = int(marker.read_text().strip()) + for _ in range(100): # the signal is delivered, not instant + try: + os.kill(pid, 0) + except OSError: + break + time.sleep(0.05) + else: + os.kill(pid, 9) + raise AssertionError(f"the session's grandchild {pid} outlived the episode") + + +#: Reports back whatever reasoning cap reached the CLI, which is how the run's own +#: number is checked rather than the one the class would have defaulted to. +ECHO_THINKING = ''' +import json, os, sys +if "--version" in sys.argv: + print("0.0.0 (fake)"); raise SystemExit(0) +open("thinking.txt", "w").write(os.environ.get("MAX_THINKING_TOKENS", "(unset)")) +print(json.dumps({"is_error": False, "num_turns": 2})) +''' + + +def test_the_executor_gets_the_same_session_settings_as_every_other_role(tmp_path): + """The role that does the work is not the role that runs on defaults. + + The architect, the manager, the reviewer and the extractor were all built from the + run's session settings; the executor was built from three of its own arguments and + inherited neither the wall nor the reasoning cap. It ran a whole fly domain at a + 900 s default nobody had chosen while `--timeout 600` and `--thinking-tokens 2048` + applied to every other role -- so the numbers have to reach it too. + """ + executor = ClaudeCodeExecutor(frozen=md.FROZEN, timeout=17.0, thinking_tokens=2048, + binary=_fake_claude(tmp_path, ECHO_THINKING)) + edits = executor(Brief(world=LocalWorld(version=1, path="src", readonly=md.FROZEN), + objective="o", context="", state=dict(md.initial_files()), + task=md.build_tasks()[0], output="FAIL", reward=0.0, depth=1)) + assert executor._timeout == 17.0 + assert {e.path: e.content for e in edits}["thinking.txt"] == "2048" + + +def test_the_session_wall_is_not_the_model_call_timeout(): + """Two different numbers, and sharing one was how the executor lost both. + + `--timeout` is the timeout on one model call -- `examples/_common` says so in its + own help, and this port defaults it to 120 s. A session is a loop of many calls, + so a 120 s "timeout" would kill every one of them before it read a file. + """ + args = genesis.build_parser().parse_args([]) + assert args.timeout == 120.0 # one model call + assert args.session_timeout == ClaudeCodeExecutor.TIMEOUT + assert args.session_timeout > args.timeout + + +def test_the_session_decisions_are_made_before_the_roles_that_read_them(): + """`--mode a` builds the extractor, and it is the first role built. + + Both the session settings and the "is there a CLI at all" fallback were written + below it, so a `--mode a --agent-sessions` run raised `NameError: use_sessions` + where it looked like it would fall back to a completion. Order is the fix, and + order is what this checks. + """ + tree = ast.parse(inspect.getsource(genesis.main)) + defined = {} + mode_a = [] + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == "_session_kwargs": + defined["_session_kwargs"] = node.lineno + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store) \ + and node.id == "use_sessions": + defined.setdefault("use_sessions", node.lineno) + if isinstance(node, ast.Compare) and isinstance(node.left, ast.Attribute) \ + and node.left.attr == "mode" \ + and any(getattr(c, "value", None) == "a" for c in node.comparators): + mode_a.append(node.lineno) + assert set(defined) == {"_session_kwargs", "use_sessions"} + extractor = max(mode_a) # the branch that builds the extractor + assert defined["_session_kwargs"] < extractor + assert defined["use_sessions"] < extractor + + +#: An architect that starts implementing. The record is the job; the module beside +#: it is the failure mode, and it costs nothing here because the workspace is a copy. +BUSY_ARCHITECT = ''' +import json, os, sys +if "--version" in sys.argv: + print("0.0.0 (fake)"); raise SystemExit(0) +os.makedirs("src", exist_ok=True) +open("src/CONTEXT.md", "w").write("# src\\n\\n## Routing Table\\n- `./core/` (2 files) -> cores\\n") +open("src/eager.py", "w").write("# an architect that could not help itself\\n") +print(json.dumps({"is_error": False, "num_turns": 4})) +''' + + +def test_a_design_session_that_writes_code_is_counted_not_believed(tmp_path): + """`strays=0` has to mean nothing strayed. + + The counter was reported on every phase-1 line and incremented nowhere, by a + helper that walked the workspace with a module its file never imported -- so it + said zero whatever the architect did. The session owns the workspace and deletes + it, so the count is taken there, before it is gone. + """ + from examples.genesis._architect_session import ArchitectSession + + designer = ArchitectSession(binary=_fake_claude(tmp_path, BUSY_ARCHITECT)) + record, children = designer({}, "src", "design it") + assert record and [c["path"] for c in children] == ["src/core"] + assert designer.strays == 1 # src/eager.py, and not the record + assert "strays=1" in designer.summary() + + def test_stackvm_is_deeper_than_minilang_which_is_why_it_exists(): """The second domain is not "harder code" -- it is somewhere for the recursion to go. `src/vm/ops` is a node whose parent is itself a child.""" From 8dfd0d3dacdaccd49db637008a19697c479f6915 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 12:33:26 +0000 Subject: [PATCH 21/42] genesis: a session launched from a session is not that session Reading the executor transcripts for the wall turned up what else they carried. A run launched from inside a Claude Code session had every episode inheriting that session's whole situation: its MCP servers, its skill listing, its agent types, the user's email address, and a system prompt about reviewing pull requests and publishing artifacts. Same endpoint, same one-line prompt: inherited 31,850 input tokens 6.7s 224 KB transcript --bare --strict-mcp-config 1,317 input tokens 2.4s 16 KB transcript A 24x prefix on every turn of every episode, none of it about the objective and some of it competing with it -- an executor told it is accountable for a pull request has been given a second job. It also feeds the wall: 30k extra tokens per turn, ~50 turns a session, 52 sessions. Three things were shared and are now not: * Identity. CLAUDE_CODE_SESSION_ID and its siblings are dropped, so each episode is its own session. Inherited, all 52 wrote transcripts named with the host's session id, and their TodoWrite state -- keyed by that id -- landed in the host's own task list, two hundred entries of "Implement src/brain package". * State. CLAUDE_CONFIG_DIR points at one directory per run, so transcripts, todos and synced skills stay out of ~/.claude, which one run had left 685 project directories in. Reported at the end of the run and deliberately not deleted: those transcripts are the only record of what an episode did, and reading 52 of them is how the wall was found. * Context. --bare --strict-mcp-config, dropping hooks, LSP, plugin sync, commit attribution, auto-memory, MCP servers and CLAUDE.md auto-discovery. The artifact carries CONTEXT.md records the brief names, not a CLAUDE.md. Bare mode reads credentials strictly from ANTHROPIC_API_KEY, never OAuth and never the keychain, so it is used only when the run brought its own key: a session billed to the local CLI's sign-in makes no API call at all with it (measured, duration_api_ms 0), which is a worse failure than a long prompt. The three fences are untouched. A bare session still honours .claude/settings.local.json -- checked against the real CLI by asking one to append to a denied path and watching it refuse, while it read and wrote the files it was entitled to. Full suite green (2939 tests). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- docs/algo-genesis.md | 39 +++++++++ examples/genesis/_claude_code.py | 21 +++-- examples/genesis/_session.py | 88 ++++++++++++++++++-- examples/genesis/genesis_recursive_worlds.py | 7 ++ tests/test_genesis_example.py | 76 +++++++++++++++++ 5 files changed, 218 insertions(+), 13 deletions(-) diff --git a/docs/algo-genesis.md b/docs/algo-genesis.md index 28d02d8b..e2a53c56 100644 --- a/docs/algo-genesis.md +++ b/docs/algo-genesis.md @@ -725,6 +725,45 @@ them was still listening two hours later with its working directory already dele Sessions now lead their own process group and the group is killed on the way out — after the wall, and after a clean finish too. +### A session launched from a session is not that session + +The transcripts turned up a second thing, and it is the more expensive one. A run +launched from inside a Claude Code session had every episode inheriting that session's +*whole situation* — its MCP servers, its skill listing, its agent types, the user's +email address, and a system prompt about reviewing pull requests and publishing +artifacts. Measured against the same endpoint with the same one-line prompt: + +| | input tokens | latency | transcript | +|---|---:|---:|---:| +| inherited | **31 850** | 6.7 s | 224 KB | +| `--bare --strict-mcp-config` | **1 317** | 2.4 s | 16 KB | + +A 24× prefix on every turn of every episode, none of it about the objective and some of +it competing with it — an executor told it is accountable for a pull request is being +given a second job. It also feeds straight back into the wall above: 30 000 extra tokens +per turn, ~50 turns a session, 52 sessions. + +Three things were shared and are now not: + +* **Identity.** `CLAUDE_CODE_SESSION_ID` and its siblings are dropped, so each episode + is its own session. Inherited, all 52 wrote transcripts named with the *host's* + session id, and their `TodoWrite` state — keyed by that id — landed in the host's own + task list, two hundred entries of "Implement src/brain package". +* **State.** `CLAUDE_CONFIG_DIR` points at one directory per run, so transcripts, todos + and synced skills stay out of `~/.claude`, which one run had left 685 project + directories in. The directory is reported at the end of the run and deliberately not + deleted: those transcripts are the only record of what an episode did. +* **Context.** `--bare --strict-mcp-config`, which drops hooks, LSP, plugin sync, commit + attribution, auto-memory, MCP servers and `CLAUDE.md` auto-discovery — the artifact + carries `CONTEXT.md` records the brief names, not a `CLAUDE.md`. + +Bare mode reads credentials strictly from `ANTHROPIC_API_KEY`, never OAuth and never the +keychain, so it is used **only when the run brought its own key**. A session billed to +the local CLI's sign-in makes no API call at all with it — measured, `duration_api_ms: +0` — which is a worse failure than a long prompt. The three fences are untouched: a bare +session still honours `.claude/settings.local.json`, checked by asking one to append to +a denied path and watching it refuse. + ## Synchronous or barrier-free, and which one is upstream's `--async` switches `evolve()` to its barrier-free runtime, and it is a fair question diff --git a/examples/genesis/_claude_code.py b/examples/genesis/_claude_code.py index 4d5487e4..a687a5d1 100644 --- a/examples/genesis/_claude_code.py +++ b/examples/genesis/_claude_code.py @@ -50,7 +50,7 @@ from .._common import cli_env from ._delegation import Brief, Edit -from ._session import run_cli +from ._session import isolation_flags, run_cli, session_env from ._spatial import SITUATED_EDIT_PROTOCOL # noqa: F401 (documented sibling) from ._world import normalise, owns @@ -179,7 +179,8 @@ def _write_settings(self, workspace: str) -> None: encoding="utf-8") as handle: json.dump(settings, handle) - def _command(self, prompt: str, turns: int) -> List[str]: + def _command(self, prompt: str, turns: int, + env: Optional[Mapping[str, str]] = None) -> List[str]: tools = ["Read", "Write", "Edit", "Glob", "Grep", "TodoWrite"] if self._allow_bash: # The point of a session rather than a completion is that it can run the @@ -191,6 +192,9 @@ def _command(self, prompt: str, turns: int) -> List[str]: "--max-turns", str(turns), "--allowedTools", ",".join(tools), "--disallowedTools", "WebFetch,WebSearch,Task"] + # The fourth fence, and it faces the other way: the first three keep the + # session out of the repository, this one keeps the host out of the session. + command += isolation_flags(env or {}) if self._model: command += ["--model", self._model] return command @@ -202,16 +206,17 @@ def _run(self, workspace: str, owner: str, brief: Brief) -> bool: failure=self._failure.format(prompt=getattr(brief.task, "prompt", ""), output=(brief.output or "")[:400], reward=brief.reward) if self._failure else "") - # `cli_env` is a no-op unless this run points ANTHROPIC_BASE_URL at a - # third-party endpoint. When it does, it drops the variables by which a - # managed session tells the CLI to use the *host's* provider and ignore - # the environment -- inherited, they make every session here 401. - env = cli_env() + # `session_env` carries the credentials and leaves the identity: `cli_env` + # underneath it drops the variables by which a managed session tells the CLI + # to use the *host's* provider and ignore the environment -- inherited, they + # make every session here 401 -- and `session_env` then drops the ones that + # say *which* session this is, and points the CLI's state somewhere private. + env = session_env() if self._thinking_tokens: env["MAX_THINKING_TOKENS"] = str(self._thinking_tokens) try: turns = self._root_turns if brief.depth == 0 else self._max_turns - out, code, timed_out = run_cli(self._command(prompt, turns), + out, code, timed_out = run_cli(self._command(prompt, turns, env), cwd=workspace, env=env, timeout=self._timeout) except Exception: # noqa: BLE001 - a dead session costs its episode diff --git a/examples/genesis/_session.py b/examples/genesis/_session.py index b63e065f..02aedc16 100644 --- a/examples/genesis/_session.py +++ b/examples/genesis/_session.py @@ -46,14 +46,90 @@ import signal import subprocess import tempfile +import threading from typing import Dict, List, Mapping, Optional, Sequence, Tuple from agentdescent.filetree import materialize from .._common import cli_env -__all__ = ["AgentSession", "READ_ONLY_TOOLS", "READ_WRITE_TOOLS", "SCRATCH_DIR", - "run_cli"] +__all__ = ["AgentSession", "HOST_SESSION_VARS", "READ_ONLY_TOOLS", + "READ_WRITE_TOOLS", "SCRATCH_DIR", "isolation_flags", "run_cli", + "session_env", "session_home"] + + +#: How the host tells a CLI it spawns who it is. Inherited, every session in a run +#: *is* the host: one fly run's 52 episodes each wrote a transcript named with the +#: host's own session id, and their `TodoWrite` state -- keyed by that id -- landed in +#: the host's task list, two hundred entries of "Implement src/brain package". +HOST_SESSION_VARS = ( + "CLAUDE_CODE_SESSION_ID", + "CLAUDE_CODE_REMOTE_SESSION_ID", + "CLAUDE_CODE_CHILD_SESSION", + "CLAUDE_CODE_SESSION_ATTENDED", + "CLAUDE_CODE_MESSAGING_SOCKET", + "CLAUDE_CODE_MESSAGING_TOKEN", + "CLAUDE_CODE_DIAGNOSTICS_FILE", + "CLAUDE_PID", +) + +_HOME_LOCK = threading.Lock() +_HOME: Optional[str] = None + + +def session_home() -> str: + """One CLI state directory for this process's sessions, and not the host's. + + `CLAUDE_CONFIG_DIR` moves everything the CLI keeps per user -- transcripts, todos, + synced skills, settings -- out of `~/.claude`. Without it a run writes into the + state of whatever session launched it; one left 685 project directories behind. + + It is **not** deleted with the workspace, deliberately. The per-session transcripts + are the only record of what an episode actually did, and reading 52 of them is how + the wall was found to be what ends an episode. They are in the system temp + directory, so they are transient without being gone before they can be read. + """ + global _HOME + with _HOME_LOCK: + if _HOME is None: + _HOME = tempfile.mkdtemp(prefix="genesis-home-") + return _HOME + + +def session_env(base: Optional[Mapping[str, str]] = None) -> Dict[str, str]: + """The environment a session runs in: the host's credentials, not its identity.""" + env = dict(cli_env() if base is None else base) + for name in HOST_SESSION_VARS: + env.pop(name, None) + env["CLAUDE_CONFIG_DIR"] = session_home() + return env + + +def isolation_flags(env: Mapping[str, str]) -> List[str]: + """`--bare --strict-mcp-config` when the run brings its own key, else nothing. + + A session launched from inside a Claude Code session inherits that session's whole + situation: the MCP servers it has connected, the skills it can call, the agents it + can spawn, the user's email address, and a system prompt about reviewing pull + requests and publishing artifacts. Measured against the same endpoint with the same + one-line prompt: **31 850 input tokens plain, 1 317 bare** -- a 24x prefix on every + turn of every episode, none of it about the objective, some of it competing with + it. Latency followed, 6.7 s to 2.4 s. + + Bare mode reads credentials strictly from `ANTHROPIC_API_KEY` -- never OAuth, never + the keychain -- so it is used only when the run has one. A session billed to the + local CLI's own sign-in would make no API call at all with it (measured: + `duration_api_ms: 0`), which is a worse failure than a long prompt. + + What it drops is what a sandboxed executor has no use for: hooks, LSP, plugin sync, + commit attribution, auto-memory, and `CLAUDE.md` auto-discovery -- the artifact + carries `CONTEXT.md` records the brief names, not a `CLAUDE.md`. The fences are + unaffected: a bare session still honours `.claude/settings.local.json`, verified by + asking one to append to a denied path and watching it refuse. + """ + if not env.get("ANTHROPIC_API_KEY"): + return [] + return ["--bare", "--strict-mcp-config"] #: Where a session that has to *answer* rather than edit puts its answer. @@ -239,7 +315,8 @@ def _write_settings(self, workspace: str) -> None: encoding="utf-8") as handle: json.dump(settings, handle) - def _command(self, prompt: str) -> List[str]: + def _command(self, prompt: str, env: Optional[Mapping[str, str]] = None, + ) -> List[str]: tools = list(self._tools) if self._allow_bash and "Bash" not in tools: tools.append("Bash") @@ -252,16 +329,17 @@ def _command(self, prompt: str) -> List[str]: "--max-turns", str(self._max_turns), "--allowedTools", ",".join(tools), "--disallowedTools", ",".join(denied)] + command += isolation_flags(env or {}) if self._model: command += ["--model", self._model] return command def _invoke(self, workspace: str, prompt: str) -> bool: - env = cli_env() + env = session_env() if self._thinking_tokens: env["MAX_THINKING_TOKENS"] = str(self._thinking_tokens) try: - out, code, timed_out = run_cli(self._command(prompt), cwd=workspace, + out, code, timed_out = run_cli(self._command(prompt, env), cwd=workspace, env=env, timeout=self._timeout) except Exception: # noqa: BLE001 - a dead session costs its turn return False diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index fc9b0409..abfcdf17 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -102,6 +102,7 @@ from ._extract import ExtractPhase from ._judge import ParentJudge from ._review import CompletionJudge, ParentCodeReview, chain_reviews +from ._session import session_home from ._suite import TEST_FAILURE from ._suite import cold_start, preflight from ._octopus import OctopusConflict, git_available @@ -814,6 +815,12 @@ def _superseded(rendered, task, output, reward_): if not args.engine_gate: print(f"gate : accepted={judge.accepted} rejected={judge.rejected} " f"partial={judge.partial}") + if sessions is not None or use_sessions: + # Where to go and read what an episode actually did, and the promise that it + # is not in `~/.claude`: a run used to write its transcripts and its TodoWrite + # state into the state of whatever session launched it. + print(f"session state : {session_home()} " + f"(isolated from the host's ~/.claude)") if log.pending_rework: print(f"open rework : {sorted(log.pending_rework)}") print(f"model usage : {usage.summary()}" diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index a148e774..306f204a 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -2276,6 +2276,82 @@ def test_a_design_session_that_writes_code_is_counted_not_believed(tmp_path): assert "strays=1" in designer.summary() +#: Reports back the session's own situation: what it was invoked with, and where the +#: CLI was told to keep its state. +ECHO_ENV = ''' +import json, os, sys +if "--version" in sys.argv: + print("0.0.0 (fake)"); raise SystemExit(0) +open("argv.txt", "w").write("\\n".join(sys.argv)) +open("home.txt", "w").write(os.environ.get("CLAUDE_CONFIG_DIR", "(unset)")) +open("ident.txt", "w").write(",".join(k for k in HOST if os.environ.get(k))) +print(json.dumps({"is_error": False, "num_turns": 1})) +''' + + +def test_a_session_does_not_inherit_the_host_s_identity_or_its_state(tmp_path, + monkeypatch): + """A session launched from a session is not that session. + + Inherited, the identity variables make every episode in a run *be* the host: one + fly run's 52 episodes each wrote a transcript named with the host's session id, and + their TodoWrite state -- keyed by that id -- landed in the host's own task list. + `CLAUDE_CONFIG_DIR` then moves transcripts, todos and synced skills out of + `~/.claude`, which one run had left 685 project directories in. + """ + from examples.genesis._session import (HOST_SESSION_VARS, session_env, + session_home) + + for name in HOST_SESSION_VARS: + monkeypatch.setenv(name, "the-host") + env = session_env() + assert not [k for k in HOST_SESSION_VARS if k in env] + assert env["CLAUDE_CONFIG_DIR"] == session_home() + assert not session_home().startswith(os.path.expanduser("~/.claude")) + + script = ECHO_ENV.replace("HOST", repr(list(HOST_SESSION_VARS))) + executor = ClaudeCodeExecutor(frozen=md.FROZEN, + binary=_fake_claude(tmp_path, script)) + edits = executor(Brief(world=LocalWorld(version=1, path="src", readonly=md.FROZEN), + objective="o", context="", state=dict(md.initial_files()), + task=md.build_tasks()[0], output="FAIL", reward=0.0, depth=1)) + wrote = {e.path: e.content for e in edits} + assert wrote["ident.txt"] == "" # none of them reached the session + assert wrote["home.txt"] == session_home() + + +def test_a_session_with_its_own_key_is_not_given_the_host_s_whole_situation(tmp_path, + monkeypatch): + """31 850 input tokens plain against 1 317 bare, same endpoint, same prompt. + + A CLI launched from inside a Claude Code session inherits that session's MCP + servers, skills, agent list, the user's email and a system prompt about reviewing + pull requests -- a 24x prefix on every turn of every episode, none of it about the + objective. `--bare` drops it, and reads credentials strictly from + `ANTHROPIC_API_KEY`, which is why it is used only when the run brought one: a + session billed to the local CLI's own sign-in makes no API call at all with it. + """ + from examples.genesis._session import isolation_flags + + assert isolation_flags({}) == [] + assert isolation_flags({"ANTHROPIC_AUTH_TOKEN": "t"}) == [] # OAuth-shaped: no + assert isolation_flags({"ANTHROPIC_API_KEY": "k"}) == ["--bare", + "--strict-mcp-config"] + + monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://example.invalid/anthropic") + monkeypatch.setenv("ANTHROPIC_API_KEY", "k") + script = ECHO_ENV.replace("HOST", "[]") + executor = ClaudeCodeExecutor(frozen=md.FROZEN, + binary=_fake_claude(tmp_path, script)) + edits = executor(Brief(world=LocalWorld(version=1, path="src", readonly=md.FROZEN), + objective="o", context="", state=dict(md.initial_files()), + task=md.build_tasks()[0], output="FAIL", reward=0.0, depth=1)) + argv = {e.path: e.content for e in edits}["argv.txt"].splitlines() + assert "--bare" in argv and "--strict-mcp-config" in argv + # the fences are not what bare drops: the deny list is still on the command line + assert "--disallowedTools" in argv + + def test_stackvm_is_deeper_than_minilang_which_is_why_it_exists(): """The second domain is not "harder code" -- it is somewhere for the recursion to go. `src/vm/ops` is a node whose parent is itself a child.""" From a80fc705d7a83b7a25812faa6b4287a956fb2547 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 12:49:24 +0000 Subject: [PATCH 22/42] docs: say what the inherited prefix actually is The previous commit described it as "a system prompt about reviewing pull requests and publishing artifacts" and an executor "given a second job". That was wrong, and the transcript's own prompt_snapshot says so: the system prompt is 5,720 characters, and the tool schemas are 178,742. 26 of them, of which Artifact alone is 64,168, then Monitor at 14,335 and DesignSync at 13,255. The executor is allowed six tools; Read and Bash together are 6,887 characters of that list. Beside it ride a 13.5 KB skill listing, a 3 KB agent listing, 900 bytes of deferred tool names, and the user's email. And one measurement that was missing, which is the whole mechanism: inherited 31,850 input tokens with the port's --allowedTools/--disallowedTools 31,099 --bare --strict-mcp-config 1,317 Permission flags do not shorten the request. They say what the session may *call*; every other schema is sent regardless, and the port had been passing those flags all along. Nobody instructed the episode to review a pull request -- it was handed the equipment of the session that launched it, and equipment is priced per turn whether or not it is reachable. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- docs/algo-genesis.md | 26 +++++++++++++++++++------- examples/genesis/_session.py | 16 +++++++++------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/docs/algo-genesis.md b/docs/algo-genesis.md index e2a53c56..95d91241 100644 --- a/docs/algo-genesis.md +++ b/docs/algo-genesis.md @@ -729,19 +729,31 @@ after the wall, and after a clean finish too. The transcripts turned up a second thing, and it is the more expensive one. A run launched from inside a Claude Code session had every episode inheriting that session's -*whole situation* — its MCP servers, its skill listing, its agent types, the user's -email address, and a system prompt about reviewing pull requests and publishing -artifacts. Measured against the same endpoint with the same one-line prompt: +*situation*. Measured against the same endpoint with the same one-line prompt: | | input tokens | latency | transcript | |---|---:|---:|---:| | inherited | **31 850** | 6.7 s | 224 KB | +| the run's own `--allowedTools` / `--disallowedTools` | 31 099 | | | | `--bare --strict-mcp-config` | **1 317** | 2.4 s | 16 KB | -A 24× prefix on every turn of every episode, none of it about the objective and some of -it competing with it — an executor told it is accountable for a pull request is being -given a second job. It also feeds straight back into the wall above: 30 000 extra tokens -per turn, ~50 turns a session, 52 sessions. +The middle row is the surprise and the whole mechanism. **Permission flags do not +shorten the request.** `--allowedTools Read,Write,Edit,Glob,Grep,Bash` says what the +session may *call*; every other tool's schema is still sent, and the port had been +passing those flags all along. + +What is actually in there, from the transcript's own `prompt_snapshot`: the system +prompt is 5 720 characters, and the **tool schemas are 178 742** — 26 of them, of which +`Artifact` alone is 64 168, then `Monitor` at 14 335 and `DesignSync` at 13 255. The +executor is allowed six tools; `Read` and `Bash` together are 6 887 characters of that +list. Beside it ride a 13.5 KB skill listing, a 3 KB agent listing, 900 bytes of +deferred tool names, and the user's email address. + +So it is not that the episode was given competing instructions — nobody told it to go +review a pull request. It is that a CLI launched inside a managed session is handed that +session's *equipment*, and equipment is priced per turn whether or not it is reachable: +30 000 tokens before the brief, ~50 turns a session, 52 sessions. That feeds straight +back into the wall above. Three things were shared and are now not: diff --git a/examples/genesis/_session.py b/examples/genesis/_session.py index 02aedc16..31406960 100644 --- a/examples/genesis/_session.py +++ b/examples/genesis/_session.py @@ -108,13 +108,15 @@ def session_env(base: Optional[Mapping[str, str]] = None) -> Dict[str, str]: def isolation_flags(env: Mapping[str, str]) -> List[str]: """`--bare --strict-mcp-config` when the run brings its own key, else nothing. - A session launched from inside a Claude Code session inherits that session's whole - situation: the MCP servers it has connected, the skills it can call, the agents it - can spawn, the user's email address, and a system prompt about reviewing pull - requests and publishing artifacts. Measured against the same endpoint with the same - one-line prompt: **31 850 input tokens plain, 1 317 bare** -- a 24x prefix on every - turn of every episode, none of it about the objective, some of it competing with - it. Latency followed, 6.7 s to 2.4 s. + A session launched from inside a Claude Code session is handed that session's + *equipment*, and equipment is priced per turn whether or not it is reachable. + Measured against the same endpoint with the same one-line prompt: **31 850 input + tokens plain, 31 099 with this port's own `--allowedTools`/`--disallowedTools`, + 1 317 bare.** The middle number is the mechanism: permission flags say what may be + *called*, and every other schema is sent anyway. From the transcript's own + `prompt_snapshot`, the system prompt is 5 720 characters and the tool schemas are + 178 742, of which `Artifact` alone is 64 168 -- beside a 13.5 KB skill listing, an + agent listing, and the user's email address. Latency followed, 6.7 s to 2.4 s. Bare mode reads credentials strictly from `ANTHROPIC_API_KEY` -- never OAuth, never the keychain -- so it is used only when the run has one. A session billed to the From 8eb405ace09017183590a962f717866ec8419196 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 13:50:49 +0000 Subject: [PATCH 23/42] genesis: ask bare mode only for the tools it carries, and say what the fences are Reading one executor's full trajectory from the isolated run turned up two things the code was wrong about. **--bare does not expose Write.** Neither --tools nor --allowedTools brings it back: a bare session given `--tools Read,Write,Glob,Grep` answers "I only have a file Read tool available". One executor spent a turn finding out -- `No such tool available: Write. Write is disabled for this session` -- and then wrote every file through `cat > f << EOF`, while the same code without --bare had made 1,910 successful Write calls. Edit is there and creates a file that does not exist, which is how all seven phase-1 records in that run were written by sessions with no shell at all. So the tool list now drops Write when bare is on and keeps a write path that exists. **--allowedTools is not a fence.** It is the auto-approve list; under --permission-mode acceptEdits a session reaches for whatever built-in tool it likes. Measured over that run's role sessions: the manager used Edit 41 times and the reviewer 6, and Edit was in neither one's allowed tools. What does hold is --disallowedTools -- no role session ran a shell or reached the network -- and AgentSession.run, which returns the paths the caller named and drops everything else. The comments and the test that claimed otherwise now say this, and the test asserts the fences that exist rather than the one that does not. **The worktree bounds what survives, not what is seen.** Four of twelve episodes ran `find /` and read a previous run's output from /tmp; one opened the very _cli.py that answered the acceptance failure it had been handed to reproduce. The brief now forbids reading outside the checkout, which is a rule and not a wall: enforcing it takes a container, and short of that a machine should be cleaned of earlier runs' output before starting one. Said plainly in the module header and in the doc rather than left as an implied guarantee. Full suite green. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- docs/algo-genesis.md | 9 ++++ examples/genesis/_claude_code.py | 37 +++++++++++----- examples/genesis/_roles.py | 6 +-- examples/genesis/_session.py | 41 +++++++++++++++--- tests/test_genesis_example.py | 73 ++++++++++++++++++++++++++++---- 5 files changed, 138 insertions(+), 28 deletions(-) diff --git a/docs/algo-genesis.md b/docs/algo-genesis.md index 95d91241..824c307c 100644 --- a/docs/algo-genesis.md +++ b/docs/algo-genesis.md @@ -665,6 +665,15 @@ The contract does not move. Three fences, none of which replaces the others: It signs in with the local `claude` CLI's credentials rather than the `--model` endpoint, which is why it is opt-in and why the run header says so. +**What the sandbox is not.** The worktree bounds what *survives*; it does not bound what +can be *seen*. A session with a shell reads whatever the process can read, and in one +run four of twelve episodes ran `find /` and opened a previous run's output from `/tmp` +— one of them read the very `_cli.py` that answered the acceptance failure it had been +handed to reproduce. The brief now forbids it, which is a rule and not a wall: the blind +property is enforced by the prompt and by what is *in* the worktree, not by the +operating system. Enforcing it takes a container; short of that, clean the machine of +earlier runs' output before starting one. + Exercised once against a real session rather than only the stand-in binary the tests drive. One leaf episode at `src/frontend` on `minilang`, twelve turns: it wrote `lexer.py` and the package marker, nothing else, and the tokens it emits are diff --git a/examples/genesis/_claude_code.py b/examples/genesis/_claude_code.py index a687a5d1..0b916663 100644 --- a/examples/genesis/_claude_code.py +++ b/examples/genesis/_claude_code.py @@ -19,10 +19,12 @@ **Permissions, and why they are set the way they are.** Three fences, none of which replaces the others: -1. *The session's own*: writes are allowed only through `Write`/`Edit`, `Bash` is off - by default, and the network tools are off outright. A settings file denies the - frozen globs by name, so the suite and the specification cannot be edited even by - accident -- a system that can edit its own tests has no tests. +1. *The session's own*: the network tools are off and `Bash` is off by default, both + through `--disallowedTools`, which is the half of the CLI's tool flags that is + actually a fence -- `--allowedTools` only auto-approves, and a session reaches for + built-in tools that are not on it. A settings file denies the frozen globs by name, + so the suite and the specification cannot be edited even by accident -- a system + that can edit its own tests has no tests. 2. *The sandbox*: the session runs with its cwd inside a throwaway worktree holding a copy of the accepted version, never the real repository. Nothing it does survives except through the diff. @@ -31,6 +33,15 @@ frozen path is dropped and counted. Both were already enforced on every edit and still are -- this module hands them edits and nothing else. +**What the sandbox is not.** It bounds what *survives*, not what can be *seen*. A +session that has a shell can read anything the process can read, and four of twelve +episodes in one run ran `find /` and read a previous run's output from `/tmp` -- one of +them opened the very `_cli.py` that answered the acceptance failure it had been asked +to reproduce. The brief now says not to, which is a rule and not a wall: the blind +property here is enforced by the prompt and by what is *in* the worktree, not by the +operating system. A run that needs it enforced needs a container, and a run on a machine +with an earlier run's output still on it should clean that up first. + The session is billed to whatever credentials the local `claude` CLI is signed in with, which is *not* the `--model` endpoint the rest of the run uses. That is a deliberate restriction: it is opt-in with `--executor claude-code`, and the header says so. @@ -50,7 +61,7 @@ from .._common import cli_env from ._delegation import Brief, Edit -from ._session import isolation_flags, run_cli, session_env +from ._session import available_tools, isolation_flags, run_cli, session_env from ._spatial import SITUATED_EDIT_PROTOCOL # noqa: F401 (documented sibling) from ._world import normalise, owns @@ -74,6 +85,11 @@ it: say so in your final message and the agent responsible for that path will be asked. - These paths are the contract and are read-only: {frozen}. The suite is what you are \ judged by; editing it is not a way to pass it. +- Everything you need is in this checkout. Do not read outside it -- no `find /`, no \ +other directory on this machine. The acceptance suite is not here and never will be, \ +and another copy of this project that you find elsewhere is some other run's work, not \ +this repository's state: reading it tells you nothing true and copying it is not the \ +task. - The tests that already pass have to keep passing. Run them. - Make the change and stop. Do not refactor beyond the objective. @@ -181,7 +197,11 @@ def _write_settings(self, workspace: str) -> None: def _command(self, prompt: str, turns: int, env: Optional[Mapping[str, str]] = None) -> List[str]: - tools = ["Read", "Write", "Edit", "Glob", "Grep", "TodoWrite"] + # The fourth fence, and it faces the other way: the first three keep the + # session out of the repository, this one keeps the host out of the session. + flags = isolation_flags(env or {}) + tools = available_tools(["Read", "Write", "Edit", "Glob", "Grep", "TodoWrite"], + flags) if self._allow_bash: # The point of a session rather than a completion is that it can run the # suite it is judged by. Bash is the only way to do that. @@ -191,10 +211,7 @@ def _command(self, prompt: str, turns: int, "--permission-mode", "acceptEdits", "--max-turns", str(turns), "--allowedTools", ",".join(tools), - "--disallowedTools", "WebFetch,WebSearch,Task"] - # The fourth fence, and it faces the other way: the first three keep the - # session out of the repository, this one keeps the host out of the session. - command += isolation_flags(env or {}) + "--disallowedTools", "WebFetch,WebSearch,Task"] + flags if self._model: command += ["--model", self._model] return command diff --git a/examples/genesis/_roles.py b/examples/genesis/_roles.py index a7f47b51..54d238de 100644 --- a/examples/genesis/_roles.py +++ b/examples/genesis/_roles.py @@ -167,7 +167,7 @@ class ManagerSession: def __init__(self, delegation_cls, **kwargs): self._delegation = delegation_cls - kwargs.setdefault("tools", READ_ONLY_TOOLS + ("Write",)) + kwargs.setdefault("tools", READ_ONLY_TOOLS + ("Write", "Edit")) self.session = AgentSession(**kwargs) def __call__(self, brief) -> Sequence[object]: @@ -193,7 +193,7 @@ class ReviewSession: def __init__(self, *, contracts: Sequence[str] = (), **kwargs): self._contracts = tuple(contracts) - kwargs.setdefault("tools", READ_ONLY_TOOLS + ("Write",)) + kwargs.setdefault("tools", READ_ONLY_TOOLS + ("Write", "Edit")) self.session = AgentSession(**kwargs) #: How many children were read, and how many were sent back. self.reviewed = 0 @@ -238,7 +238,7 @@ def __init__(self, **kwargs): # own `CONTEXT.md` (`agent/tools.ex` keeps `context_write` in the read-only # set for exactly this). Write is granted; the brief scopes it to one file # and the caller reads back only that file. - kwargs.setdefault("tools", READ_ONLY_TOOLS + ("Write",)) + kwargs.setdefault("tools", READ_ONLY_TOOLS + ("Write", "Edit")) self.session = AgentSession(**kwargs) def __call__(self, state, path: str, objective: str, *, diff --git a/examples/genesis/_session.py b/examples/genesis/_session.py index 31406960..09a97a52 100644 --- a/examples/genesis/_session.py +++ b/examples/genesis/_session.py @@ -54,8 +54,8 @@ from .._common import cli_env __all__ = ["AgentSession", "HOST_SESSION_VARS", "READ_ONLY_TOOLS", - "READ_WRITE_TOOLS", "SCRATCH_DIR", "isolation_flags", "run_cli", - "session_env", "session_home"] + "READ_WRITE_TOOLS", "SCRATCH_DIR", "available_tools", + "isolation_flags", "run_cli", "session_env", "session_home"] #: How the host tells a CLI it spawns who it is. Inherited, every session in a run @@ -134,6 +134,27 @@ def isolation_flags(env: Mapping[str, str]) -> List[str]: return ["--bare", "--strict-mcp-config"] +def available_tools(tools: Sequence[str], flags: Sequence[str]) -> List[str]: + """The requested tools, minus the one bare mode does not carry. + + **`--bare` does not expose `Write`**, and neither `--tools` nor `--allowedTools` + brings it back: a bare session given `--tools Read,Write,Glob,Grep` answers "I only + have a file Read tool available". Asking for it anyway is asking for a tool the + session will be told does not exist -- one executor spent a turn discovering that + and then wrote every file through `cat > f << EOF` instead. + + `Edit` is there and it creates a file that does not exist, which is how all seven + phase-1 records in one run were written by architect sessions that have no shell at + all. So under bare the write path is Edit, and the tool list should say so. + """ + if "--bare" not in flags: + return list(tools) + out = [t for t in tools if t != "Write"] + if out != list(tools) and "Edit" not in out: + out.append("Edit") # a role that could write must still be able to + return out + + #: Where a session that has to *answer* rather than edit puts its answer. #: #: A Manager decides and a reviewer judges; neither produces a file the repository @@ -144,9 +165,16 @@ def isolation_flags(env: Mapping[str, str]) -> List[str]: #: skips it, and the workspace is deleted either way. SCRATCH_DIR = ".genesis" -#: What upstream's `:read` agents get, mapped onto the CLI's tool names. No Write: -#: a role that only has to look does not get to change anything, and the one job a +#: What upstream's `:read` agents get, mapped onto the CLI's tool names. The one job a #: read-only agent does write -- `CONTEXT.md` -- is granted per role rather than here. +#: +#: Read this as a *request*, not a fence. `--allowedTools` is the CLI's auto-approve +#: list, and under `--permission-mode acceptEdits` a session reaches for whatever +#: built-in tool it likes: measured over one run, the manager used `Edit` 41 times and +#: the reviewer 6, and neither had `Edit` in its allowed tools. The fences that do hold +#: are `--disallowedTools` -- no role session in that run ran a shell or reached the +#: network -- and :meth:`AgentSession.run`, which returns the paths the caller named and +#: silently drops everything else the session touched. READ_ONLY_TOOLS = ("Read", "Glob", "Grep") #: What upstream's `:read_write` agents get, minus the shell. Bash is granted per role, @@ -325,13 +353,14 @@ def _command(self, prompt: str, env: Optional[Mapping[str, str]] = None, denied = ["WebFetch", "WebSearch", "Task"] if not self._allow_bash: denied.insert(0, "Bash") + flags = isolation_flags(env or {}) + tools = available_tools(tools, flags) command = [self._binary, "-p", prompt, "--output-format", "json", "--permission-mode", "acceptEdits", "--max-turns", str(self._max_turns), "--allowedTools", ",".join(tools), - "--disallowedTools", ",".join(denied)] - command += isolation_flags(env or {}) + "--disallowedTools", ",".join(denied)] + flags if self._model: command += ["--model", self._model] return command diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index 306f204a..b4643b1a 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -2352,6 +2352,50 @@ def test_a_session_with_its_own_key_is_not_given_the_host_s_whole_situation(tmp_ assert "--disallowedTools" in argv +def test_bare_mode_is_asked_only_for_the_tools_it_carries(): + """`--bare` does not expose `Write`, and no flag brings it back. + + Measured against the real CLI: a bare session given `--tools Read,Write,Glob,Grep` + answers "I only have a file Read tool available". An executor in one run spent a + turn discovering it -- `No such tool available: Write. Write is disabled for this + session` -- and then wrote every file through `cat > f << EOF` instead, while the + same code without `--bare` had made 1,910 successful `Write` calls. `Edit` is there + and creates a file that does not exist, which is how all seven phase-1 records in + that run were written by sessions with no shell at all. + """ + from examples.genesis._session import READ_WRITE_TOOLS, available_tools + + bare = ["--bare", "--strict-mcp-config"] + assert available_tools(READ_WRITE_TOOLS, []) == list(READ_WRITE_TOOLS) + kept = available_tools(READ_WRITE_TOOLS, bare) + assert "Write" not in kept and "Edit" in kept + # a role whose only job is to write one file keeps a way to write it + assert "Edit" in available_tools(("Read", "Glob", "Grep", "Write"), bare) + + executor = ClaudeCodeExecutor(frozen=md.FROZEN, binary="claude") + command = executor._command("do it", 8, {"ANTHROPIC_API_KEY": "k"}) + allowed = command[command.index("--allowedTools") + 1].split(",") + assert "--bare" in command and "Write" not in allowed and "Edit" in allowed + plain = executor._command("do it", 8, {}) + assert "--bare" not in plain + assert "Write" in plain[plain.index("--allowedTools") + 1].split(",") + + +def test_the_brief_keeps_the_session_inside_the_checkout(): + """The blind property is a rule, not a wall, and the rule has to be stated. + + Four of twelve episodes in one run ran `find /` and read a previous run's output + from `/tmp`; one opened the very `_cli.py` that answered the acceptance failure it + had been handed to reproduce. A shell can read whatever the process can, so the + worktree bounds what survives and not what is seen. + """ + brief = CLAUDE_CODE_BRIEF.format(path="src", objective="o", frozen="REQUIREMENTS.md", + failure="") + assert "Do not read outside it" in brief + assert "find /" in brief + assert "not this repository's state" in brief + + def test_stackvm_is_deeper_than_minilang_which_is_why_it_exists(): """The second domain is not "harder code" -- it is somewhere for the recursion to go. `src/vm/ops` is a node whose parent is itself a child.""" @@ -2584,18 +2628,29 @@ def test_an_extractor_session_replaces_files_in_the_prompt(): assert "may not change the code" in EXTRACT_BRIEF -def test_a_read_only_role_gets_no_shell_and_no_edit(): - """`agent/tools.ex` gives `:read` agents the read half plus `context_write`. The - one thing they write is a record; they do not edit code and they are not given a - shell here, which upstream grants for running tests and this port's extractor has - no use for. +def test_a_read_only_role_gets_no_shell_and_keeps_only_what_it_was_asked_for(tmp_path): + """The fence that holds is the deny list and the readback, not the allow list. + + `--allowedTools` is an auto-approve list, not a whitelist: measured over one run's + role sessions, the manager used `Edit` 41 times and the reviewer 6, and `Edit` was + never in either one's allowed tools. `--disallowedTools` *is* a fence -- no role + session in that run ran a shell, and none reached the network tools -- and the + containment for everything else is :meth:`AgentSession.run`, which returns the paths + the caller named and nothing else. `agent/tools.ex` gives `:read` agents the read + half plus `context_write`; that is what this comes to here. """ from examples.genesis._roles import ExtractSession, ManagerSession for role in (ExtractSession(model="m"), ManagerSession(lambda p, o: (p, o), model="m")): command = role.session._command("hi") - allowed = command[command.index("--allowedTools") + 1] - assert "Edit" not in allowed, allowed - assert "Bash" not in allowed, allowed - assert "Bash" in command[command.index("--disallowedTools") + 1] + denied = command[command.index("--disallowedTools") + 1] + assert "Bash" in denied and "WebFetch" in denied and "Task" in denied + + # ...and a session that writes where it was not asked is not believed. + from examples.genesis._session import AgentSession + + session = AgentSession(binary=_fake_claude(tmp_path, BUSY_ARCHITECT)) + got = session.run({}, "design it", read=["src/CONTEXT.md"]) + assert set(got) == {"src/CONTEXT.md"} # src/eager.py was written, not read + assert session.changed == ["src/eager.py"] # seen and counted, never returned From abc16b504d086fbb07ac5d249911c951ca8b47c1 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 14:28:03 +0000 Subject: [PATCH 24/42] genesis: run every agent session inside the engine's own sandbox The blind property was a line in a prompt. Four of twelve episodes in one run ran `find /` and opened a previous run's output from /tmp; one of them read the very _cli.py that answered the acceptance failure it had been handed to reproduce. The worktree bounds what a session's work *becomes* -- an edit outside the node is a request, a frozen write is dropped, nothing survives but the diff -- and none of that bounds what it can read. agentdescent already owns the fix and I had missed it. sandbox.py manages *lifetime* and says so ("It does not manage isolation"); sandbox_container.py is titled "A sandbox that is actually a boundary" and gives exactly what is wanted: only the workspace visible, read-only root, no capabilities, no new privileges, resource ceilings, network off unless the spec asks. The first look here reported no engine and that was wrong -- dockerd was installed and simply not running. examples/genesis/_sandbox.py subclasses ContainerProvider and adds the mounts an *agent* session needs that a candidate's test run does not: the claude binary's own install (the image has no agent in it), the proxy's CA bundle (a session that cannot verify a TLS-inspecting proxy spends its turns on certificate errors -- a probe lost two to pip's CERTIFICATE_VERIFY_FAILED), and a per-session CLI state directory so the transcript outlives the container. Nothing else of the host. Asked from inside, on this machine: ls /home/user/agentdescent No such file or directory find / -name 'algo-genesis.md' (nothing) ls /tmp | wc -l 0 ls /work CONTEXT.md md.py spec src tests touch /etc/x Read-only file system grep CapEff /proc/self/status CapEff: 0000000000000000 Four things this needed that the flags alone did not give: * `network="inherit"` leaves the engine's default bridge, which is not the host's network. The endpoint here is reached through a proxy on the host's loopback, and on a bridge 127.0.0.1 is the container. The subclass adds --network host. * The binary under the mount is the *resolved* path: `claude` is a symlink out of a node install, and exec'ing the link's own path inside is "stat: no such file or directory". * Only the agent's argv[0] is rewritten. A shell probe through the same workspace is not the agent, and rewriting it turns `sh -c 'ls /'` into "Please run /login". * The provider's own .agentdescent-mount marker is not the session's work; the first sandboxed diff reported it as an edit the node never made. Also dropped MAX_THINKING_TOKENS from the inherited environment. It is not identity but it leaks the same way: the reasoning cap the *host* session runs under silently overrode --thinking-tokens. Two honest limits, both stated in the doc: the network is on, so this is a boundary against contamination and not against hostile code; and where no engine answers the run says so once and falls back to a plain directory rather than pretending. Full suite green. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- docs/algo-genesis.md | 37 ++- examples/genesis/_claude_code.py | 32 +- examples/genesis/_sandbox.py | 311 +++++++++++++++++++ examples/genesis/_session.py | 40 ++- examples/genesis/genesis_recursive_worlds.py | 30 +- tests/test_genesis_example.py | 117 +++++++ 6 files changed, 533 insertions(+), 34 deletions(-) create mode 100644 examples/genesis/_sandbox.py diff --git a/docs/algo-genesis.md b/docs/algo-genesis.md index 824c307c..3cdf1df2 100644 --- a/docs/algo-genesis.md +++ b/docs/algo-genesis.md @@ -665,14 +665,35 @@ The contract does not move. Three fences, none of which replaces the others: It signs in with the local `claude` CLI's credentials rather than the `--model` endpoint, which is why it is opt-in and why the run header says so. -**What the sandbox is not.** The worktree bounds what *survives*; it does not bound what -can be *seen*. A session with a shell reads whatever the process can read, and in one -run four of twelve episodes ran `find /` and opened a previous run's output from `/tmp` -— one of them read the very `_cli.py` that answered the acceptance failure it had been -handed to reproduce. The brief now forbids it, which is a rule and not a wall: the blind -property is enforced by the prompt and by what is *in* the worktree, not by the -operating system. Enforcing it takes a container; short of that, clean the machine of -earlier runs' output before starting one. +**What the worktree does not bound: `--sandbox`.** The worktree bounds what *survives*, +not what can be *seen*. A session with a shell reads whatever the process can read, and +in one run four of twelve episodes ran `find /` and opened a previous run's output from +`/tmp` — one of them read the very `_cli.py` that answered the acceptance failure it had +been handed to reproduce. The blind property was a line in a prompt. + +The engine already owns the fix. `agentdescent/sandbox_container.py` is titled "A +sandbox that is actually a boundary", and `--sandbox` (on by default where an engine +answers) runs every agent session inside it. `examples/genesis/_sandbox.py` subclasses +its `ContainerProvider` and adds the mounts an *agent* session needs and a candidate's +test run does not: the `claude` binary's own install, the proxy's CA bundle, and a +per-session CLI state directory so the transcript outlives the container. Nothing else +of the host. Asked from inside, on this machine: + +``` +ls /home/user/agentdescent No such file or directory +find / -name 'algo-genesis.md' (nothing) +ls /tmp | wc -l 0 +ls /work CONTEXT.md md.py spec src tests +touch /etc/x Read-only file system +grep CapEff /proc/self/status CapEff: 0000000000000000 +``` + +Two honest limits. **The network is on** — `SandboxSpec.network="inherit"`, because a +session's whole job is to reach a model endpoint — so this is a boundary against +contamination, which is the failure that happened, and not against hostile code, which +could still send what it read. And **when no engine answers** the run says so once and +falls back to a plain directory rather than pretending; the brief's "do not read outside +this checkout" rule is what is left, and a rule is not a wall. Exercised once against a real session rather than only the stand-in binary the tests drive. One leaf episode at `src/frontend` on `minilang`, twelve turns: it wrote diff --git a/examples/genesis/_claude_code.py b/examples/genesis/_claude_code.py index 0b916663..063b40ca 100644 --- a/examples/genesis/_claude_code.py +++ b/examples/genesis/_claude_code.py @@ -53,7 +53,6 @@ import os import shutil import subprocess -import tempfile from typing import Callable, Dict, List, Mapping, Optional, Sequence from agentdescent.filetree import match_any, materialize @@ -61,6 +60,7 @@ from .._common import cli_env from ._delegation import Brief, Edit +from ._sandbox import PROVIDER_FILES, LocalSandbox, Workspace from ._session import available_tools, isolation_flags, run_cli, session_env from ._spatial import SITUATED_EDIT_PROTOCOL # noqa: F401 (documented sibling) from ._world import normalise, owns @@ -135,7 +135,10 @@ def __init__(self, *, frozen: Sequence[str] = (), binary: str = "claude", model: str = "", max_turns: int = 0, root_turns: int = 0, timeout: float = 0.0, thinking_tokens: int = 0, allow_bash: bool = True, failure: str = "", - root: Optional[str] = None): + root: Optional[str] = None, sandbox=None): + #: Where an episode runs and what it can see -- see `._sandbox`. The executor is + #: the role this matters most for: it is the only one with a shell. + self.sandbox = sandbox if sandbox is not None else LocalSandbox(root or "") self._frozen = tuple(frozen) self._binary = binary self._model = model @@ -162,22 +165,22 @@ def __init__(self, *, frozen: Sequence[str] = (), binary: str = "claude", def __call__(self, brief: Brief) -> Sequence[Edit]: owner = normalise(brief.world.path) - workspace = tempfile.mkdtemp(prefix="genesis-cc-", dir=self._root) + space = self.sandbox.open("genesis-cc-") try: before = dict(brief.state) - materialize(before, workspace) - self._write_settings(workspace) + materialize(before, space.path) + self._write_settings(space.path) self.sessions += 1 - ok = self._run(workspace, owner, brief) + ok = self._run(space, owner, brief) if not ok: self.failed += 1 - after = _read_tree(workspace) + after = _read_tree(space.path) edits = _diff(before, after, owner) self.edits += sum(1 for e in edits if owns(owner, e.path)) self.requests += sum(1 for e in edits if not owns(owner, e.path)) return edits finally: - shutil.rmtree(workspace, ignore_errors=True) + space.close() # -- internals --------------------------------------------------------- @@ -216,7 +219,7 @@ def _command(self, prompt: str, turns: int, command += ["--model", self._model] return command - def _run(self, workspace: str, owner: str, brief: Brief) -> bool: + def _run(self, space: Workspace, owner: str, brief: Brief) -> bool: prompt = CLAUDE_CODE_BRIEF.format( path=owner or ".", objective=brief.objective, frozen=", ".join(self._frozen) or "(none)", @@ -233,12 +236,15 @@ def _run(self, workspace: str, owner: str, brief: Brief) -> bool: env["MAX_THINKING_TOKENS"] = str(self._thinking_tokens) try: turns = self._root_turns if brief.depth == 0 else self._max_turns - out, code, timed_out = run_cli(self._command(prompt, turns, env), - cwd=workspace, env=env, - timeout=self._timeout) + out, code, timed_out = run_cli( + space.command(self._command(prompt, turns, env), env), + cwd=space.path, env=env, timeout=self._timeout) except Exception: # noqa: BLE001 - a dead session costs its episode return False if timed_out: + # The process group that gets killed is the `exec`'s, and inside a container + # the session is a different tree on the same machine: stop it explicitly. + space.kill() # There is no report after the wall, so this episode's turns are not in # `turns` -- which is why the counter read 309 for a run whose transcripts # hold 2 607. The diff is taken either way: the caller reads the worktree @@ -266,6 +272,8 @@ def _read_tree(workspace: str) -> Dict[str, str]: for name in files: full = os.path.join(base, name) rel = os.path.relpath(full, workspace).replace(os.sep, "/") + if rel in PROVIDER_FILES: + continue # the sandbox's own bookkeeping, not the node's work try: with open(full, encoding="utf-8") as handle: out[rel] = handle.read() diff --git a/examples/genesis/_sandbox.py b/examples/genesis/_sandbox.py new file mode 100644 index 00000000..7613e254 --- /dev/null +++ b/examples/genesis/_sandbox.py @@ -0,0 +1,311 @@ +"""Where a session runs, and what it can see from there. + +The port's three fences all bound what a session's work *becomes*: an edit outside the +node is a request, a write to a frozen path is dropped, and the worktree is a throwaway +copy so nothing survives except through the diff. None of them bounds what a session can +*read*, and a session with a shell reads whatever the process can. Measured, in one fly +run: four of twelve episodes ran ``find /`` and opened a previous run's output from +``/tmp``, and one of those read the very ``_cli.py`` that answered the acceptance failure +it had been handed to reproduce. The blind property was a rule in a prompt. + +The engine already owns the fix. :mod:`agentdescent.sandbox_container` is titled "A +sandbox that is actually a boundary" and says what it changes: only the workspace is +visible, the network is off unless the spec asks, no capabilities, no new privileges, +resource ceilings. This module uses it, and adds the one thing an *agent* session needs +that a candidate's test run does not -- **the toolchain that runs the agent**. + +Three read-only mounts and one writable one, and each is there for a reason: + +* the ``claude`` binary's own install, because the image has no agent in it; +* the proxy's CA bundle when there is one, because a session behind a TLS-inspecting + proxy that cannot verify it spends its turns on certificate errors rather than on the + objective (measured: a probe session lost two turns to `pip`'s + ``CERTIFICATE_VERIFY_FAILED`` before working around it); +* nothing else of the host, which is the point; +* and a per-session CLI state directory, so the transcript outlives the container. + Reading 52 of those transcripts is how the wall was found to be what ends an episode, + and a boundary that also destroys the evidence is a bad trade. + +**The network is on, and that is a real weakening.** ``SandboxSpec.network="inherit"`` +is required: the session's whole job is to talk to a model endpoint. So this is not a +boundary against hostile code -- a session that can reach the network can send whatever +it read. It is a boundary against *contamination*, which is the failure that actually +happened, and against that it is complete: the filesystem the session sees contains the +work and nothing else. + +**When there is no engine** the run says so once and falls back to a plain directory. +A fallback that pretended to isolate would be worse than none. +""" + +from __future__ import annotations + +import os +import shutil +import tempfile +from typing import List, Mapping, Optional, Sequence + +from agentdescent.policies import SandboxSpec +from agentdescent.sandbox import LEASE_FILE +from agentdescent.sandbox_container import (CONTAINER_WORKDIR, ContainerProvider, + detect_engine) + +__all__ = ["CONTAINER_HOME", "PROVIDER_FILES", "LocalSandbox", "SessionSandbox", + "Workspace", "sandbox_engine", "toolchain_root"] + + +#: What the provider itself writes into a workspace, and which is therefore not the +#: session's work. `.agentdescent-mount` is the file `_verify_mount` looks for from the +#: inside to tell a working bind mount from an empty one; without this the first diff of +#: every sandboxed episode reported it as an edit the node never made. +PROVIDER_FILES = frozenset({".agentdescent-mount", LEASE_FILE}) + + +#: Where the CLI's own state goes inside the container. Not `/work`: `HOME` has to name +#: somewhere writable, and the CLI drops `.claude.json` beside it -- in the workspace +#: that is a file the diff would report as the session's work. +CONTAINER_HOME = "/clihome" + +#: Passed through to the container by name, and nothing else is. The values are read on +#: the host, one at a time, the way `SandboxSpec.env_allowlist` does it: a variable that +#: is not on this list cannot arrive by accident, which is the same reason the host's +#: session identity is dropped in `_session.session_env`. +CONTAINER_ENV: Sequence[str] = ( + "ANTHROPIC_BASE_URL", "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", + "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", + "http_proxy", "https_proxy", "no_proxy", + "MAX_THINKING_TOKENS", +) + +#: Where a TLS-inspecting proxy keeps the certificate everything inside has to trust. +_CA_CANDIDATES = ("/root/.ccr/ca-bundle.crt", "/etc/ssl/certs/ca-certificates.crt") + +#: Small, has a Python, and is a name rather than a build: an image the run has to build +#: first is an image that fails at the first episode on a machine with no build context. +DEFAULT_IMAGE = "python:3.12-slim" + + +def sandbox_engine(preferred: str = "") -> Optional[str]: + """The container engine this machine can actually use, or ``None``. + + :func:`agentdescent.sandbox_container.detect_engine` already asks both halves of the + question -- installed *and* answering -- which is the difference that matters here: + `dockerd` sitting on the PATH with no daemon behind it took one investigation's + first look and produced the wrong answer. + """ + return detect_engine(preferred or None) + + +def toolchain_root(binary: str = "claude") -> Optional[str]: + """The directory to mount so `binary` exists inside the container. + + A self-contained install puts the executable at ``/bin/``, and the rest + of ```` is what it needs; anything else, mount the directory the executable is + in. Returns ``None`` when the binary is somewhere shared like ``/usr/bin``, where + mounting the parent would mean mounting the host's system directories -- the thing + this module exists to avoid. + """ + found = shutil.which(binary) + if not found: + return None + real = os.path.realpath(found) + holder = os.path.dirname(real) + root = os.path.dirname(holder) if os.path.basename(holder) == "bin" else holder + if root in ("", "/", "/usr", "/usr/local", "/bin", "/sbin", "/opt"): + return None + return root + + +class Workspace: + """A directory to materialise into, and how to run a command against it. + + The same two questions whichever way the session runs: *where do I put the state and + read the answer back from*, and *what argv actually runs*. A local workspace answers + the second with the command itself; a container answers it with an ``exec`` into the + boundary, and the directory is the bind mount on the host side. + """ + + def __init__(self, path: str, *, prefix: Sequence[str] = (), + env: Optional[Mapping[str, str]] = None, + close=None, kill=None, binary: str = "", + binary_names: Sequence[str] = ()): + self.path = path + self._prefix = list(prefix) + self._env = dict(env or {}) + self._close = close + self._kill = kill + #: The session binary as it exists *under the mount*, and the spellings of it a + #: caller might use. Only those are rewritten: a command is not always the agent + #: -- the tests run `sh` through here -- and rewriting every argv[0] turns a + #: shell probe into a confusing "Please run /login". + self._binary = binary + self._names = {n for n in binary_names if n} + + def command(self, argv: Sequence[str], env: Mapping[str, str]) -> List[str]: + """The argv to run on *this* host for a session command.""" + argv = list(argv) + if not self._prefix: + return argv + if self._binary and argv and argv[0] in self._names: + argv[0] = self._binary # the mounted path, not the host's PATH + passed: List[str] = [] + for name in CONTAINER_ENV: + value = env.get(name) + if value is not None: + passed += ["--env", f"{name}={value}"] + for name, value in self._env.items(): + passed += ["--env", f"{name}={value}"] + return self._prefix[:-1] + passed + [self._prefix[-1]] + argv + + def kill(self) -> None: + """Stop whatever is still running inside. A no-op for a local workspace.""" + if self._kill is not None: + try: + self._kill() + except Exception: # noqa: BLE001 - killing is best effort by definition + pass + + def close(self) -> None: + if self._close is not None: + self._close() + else: + shutil.rmtree(self.path, ignore_errors=True) + + +class LocalSandbox: + """No boundary: a throwaway directory, which is where this port started. + + Kept as a class rather than a `None` check so the two paths have one shape, and so a + run that is not isolated says so in the same place a run that is says the opposite. + """ + + available = False + reason = "not asked for" + + def __init__(self, root: str = ""): + self._root = root or None + self.opened = 0 + + def open(self, prefix: str = "genesis-") -> Workspace: + self.opened += 1 + return Workspace(tempfile.mkdtemp(prefix=prefix, dir=self._root)) + + def summary(self) -> str: + return "off (a plain directory; a session can read this machine)" + + +class SessionSandbox: + """Hands out workspaces: containers when there is an engine, directories otherwise. + + One of these per run. `available` is decided once at construction, because a run + that cannot isolate should say so once rather than at every episode -- and because + the answer cannot change halfway through without the run being a different + experiment in the middle. + """ + + def __init__(self, *, engine: str = "", image: str = "", root: str = "", + binary: str = "claude", home: str = "", + memory_mb: int = 4096, cpu: float = 0.0): + self.engine = sandbox_engine(engine) + self.toolchain = toolchain_root(binary) + # The *resolved* path, because that is the one under the mount: `claude` is a + # symlink from a node install's `bin` here, and exec'ing the symlink's own path + # inside the container is `stat: no such file or directory`. + found = shutil.which(binary) + self.binary = os.path.realpath(found) if found else binary + #: Every spelling of the session binary a caller might hand us. + self.binary_names = tuple({binary, found or binary, self.binary}) + self._image = image or DEFAULT_IMAGE + self._root = root or None + self._home = home + self._memory_mb = memory_mb + self._cpu = cpu + self._ca = next((p for p in _CA_CANDIDATES if os.path.exists(p)), "") + #: Why the run is not isolated, when it is not. Empty when it is. + self.reason = "" + if self.engine is None: + self.reason = ("no container engine is answering (docker or podman); " + "sessions run in a plain directory and can read this machine") + elif self.toolchain is None: + self.reason = (f"the `{binary}` binary is not in a self-contained install, " + "so there is nothing to mount into a container") + self.provider = None + if not self.reason: + self.provider = _Provider(self, engine=self.engine, image=self._image) + #: Containers handed out, for the run's own report. + self.opened = 0 + + @property + def available(self) -> bool: + return self.provider is not None + + def spec(self) -> SandboxSpec: + # `network="inherit"` is the one place this weakens the engine's defaults, and + # it is not optional: the session exists to reach a model endpoint. Everything + # else stays as `run_command` writes it -- read-only root, no capabilities, no + # new privileges, a pids ceiling, and the host uid so the diff is readable. + return SandboxSpec(image=self._image, network="inherit", + workspace_root=self._root, + memory_mb=self._memory_mb or None, + cpu=self._cpu or None) + + def open(self, prefix: str = "genesis-") -> Workspace: + """A workspace for one session.""" + if self.provider is None: + return Workspace(tempfile.mkdtemp(prefix=prefix, dir=self._root)) + sandbox = self.provider.acquire(self.spec()) + self.opened += 1 + env = {"HOME": CONTAINER_HOME, "TMPDIR": "/tmp", + "CLAUDE_CONFIG_DIR": CONTAINER_HOME} + if self._ca: + # Four names for one file because four different stacks look for it: node + # for the CLI, and requests/openssl/pip for whatever the session installs. + env.update({"NODE_EXTRA_CA_CERTS": self._ca, "REQUESTS_CA_BUNDLE": self._ca, + "SSL_CERT_FILE": self._ca, "PIP_CERT": self._ca}) + return Workspace( + sandbox.root, + prefix=[self.engine, "exec", "-w", CONTAINER_WORKDIR, sandbox.container_id], + env=env, binary=self.binary, binary_names=self.binary_names, + kill=sandbox.kill, + close=lambda: self.provider.release(sandbox)) + + def summary(self) -> str: + if not self.available: + return f"off ({self.reason})" + return (f"{self.engine} {self._image}, {self.opened} container(s); " + f"only the workspace visible, network inherited") + + +class _Provider(ContainerProvider): + """The engine's provider, plus the mounts an *agent* session needs. + + Everything the boundary is made of stays where it is: this adds volumes to the + command the base class builds and changes nothing else. A session's own CLI state + directory is created per container so eight of them do not write one `.claude.json` + between them, and it lives under the run's state directory rather than inside the + workspace, where it would show up in the diff as work. + """ + + def __init__(self, sandbox: SessionSandbox, **kwargs): + super().__init__(**kwargs) + self._owner = sandbox + + def run_command(self, spec: SandboxSpec, ws: str, lease_id: str) -> List[str]: + command = super().run_command(spec, ws, lease_id) + mounts: List[str] = [] + if spec.network == "inherit": + # The base class reads `inherit` as "do not switch the network off", which + # leaves the engine's default bridge. That is not the host's network, and + # the difference is the whole thing here: the endpoint is reached through a + # proxy on the host's loopback, and on a bridge `127.0.0.1` is the container. + mounts += ["--network", "host"] + if self._owner.toolchain: + mounts += ["--volume", f"{self._owner.toolchain}:{self._owner.toolchain}:ro"] + if self._owner._ca: + mounts += ["--volume", f"{self._owner._ca}:{self._owner._ca}:ro"] + home = os.path.join(self._owner._home or ws, f"cli-{lease_id[:8]}") + os.makedirs(home, exist_ok=True) + mounts += ["--volume", f"{home}:{CONTAINER_HOME}:rw"] + # After `run` and its flags, before the image name and the idle command: the + # base class puts the image last but one, so inserting at the front of the + # flag block is the only placement that does not depend on its tail. + return command[:2] + mounts + command[2:] diff --git a/examples/genesis/_session.py b/examples/genesis/_session.py index 09a97a52..d1d58503 100644 --- a/examples/genesis/_session.py +++ b/examples/genesis/_session.py @@ -42,7 +42,6 @@ import json import os -import shutil import signal import subprocess import tempfile @@ -53,6 +52,8 @@ from .._common import cli_env +from ._sandbox import PROVIDER_FILES, LocalSandbox, Workspace + __all__ = ["AgentSession", "HOST_SESSION_VARS", "READ_ONLY_TOOLS", "READ_WRITE_TOOLS", "SCRATCH_DIR", "available_tools", "isolation_flags", "run_cli", "session_env", "session_home"] @@ -71,6 +72,11 @@ "CLAUDE_CODE_MESSAGING_TOKEN", "CLAUDE_CODE_DIAGNOSTICS_FILE", "CLAUDE_PID", + # Not identity, but the same kind of leak: the reasoning cap the *host* session is + # running under is not the one this run chose, and inherited it silently overrides + # `--thinking-tokens`. Dropped here so the variable is present only when a role + # puts it there. + "MAX_THINKING_TOKENS", ) _HOME_LOCK = threading.Lock() @@ -235,7 +241,7 @@ def _strays(workspace: str, state: Mapping[str, str], be. The scratch directory is not a stray -- it is where a session that has to answer rather than edit puts its answer -- and neither is anything in `read`. """ - skip = set(read) + skip = set(read) | set(PROVIDER_FILES) out: List[str] = [] for base, dirs, files in os.walk(workspace): dirs[:] = [d for d in dirs @@ -268,7 +274,11 @@ def __init__(self, *, tools: Sequence[str] = READ_WRITE_TOOLS, frozen: Sequence[str] = (), binary: str = "claude", model: str = "", max_turns: int = 40, timeout: float = 900.0, root: Optional[str] = None, thinking_tokens: int = 0, - allow_bash: bool = False): + allow_bash: bool = False, sandbox=None): + #: Where a session runs and what it can see from there -- see `._sandbox`. A + #: plain directory unless the run hands over a real one, because a default that + #: went looking for a container engine would make every unit test wait for one. + self.sandbox = sandbox if sandbox is not None else LocalSandbox(root or "") self._tools = tuple(tools) self._frozen = tuple(frozen) self._binary = binary @@ -305,26 +315,26 @@ def run(self, state: Mapping[str, str], prompt: str, *, reported as failed is still read: the turn limit in particular lands *after* the work, so the file is what decides, not the exit status. """ - workspace = tempfile.mkdtemp(prefix="genesis-sess-", dir=self._root) + space = self.sandbox.open("genesis-sess-") try: - materialize(dict(state), workspace) - self._write_settings(workspace) + materialize(dict(state), space.path) + self._write_settings(space.path) self.sessions += 1 - if not self._invoke(workspace, prompt): + if not self._invoke(space, prompt): self.failed += 1 out: Dict[str, str] = {} for rel in read: try: - with open(os.path.join(workspace, rel), encoding="utf-8") as fh: + with open(os.path.join(space.path, rel), encoding="utf-8") as fh: body = fh.read() except (OSError, UnicodeDecodeError): continue if body.strip(): out[rel] = body - self.changed = _strays(workspace, state, read) + self.changed = _strays(space.path, state, read) return out finally: - shutil.rmtree(workspace, ignore_errors=True) + space.close() def summary(self) -> str: return (f"sessions={self.sessions} failed={self.failed} " @@ -365,16 +375,20 @@ def _command(self, prompt: str, env: Optional[Mapping[str, str]] = None, command += ["--model", self._model] return command - def _invoke(self, workspace: str, prompt: str) -> bool: + def _invoke(self, space: Workspace, prompt: str) -> bool: env = session_env() if self._thinking_tokens: env["MAX_THINKING_TOKENS"] = str(self._thinking_tokens) try: - out, code, timed_out = run_cli(self._command(prompt, env), cwd=workspace, - env=env, timeout=self._timeout) + out, code, timed_out = run_cli(space.command(self._command(prompt, env), env), + cwd=space.path, env=env, + timeout=self._timeout) except Exception: # noqa: BLE001 - a dead session costs its turn return False if timed_out: + # Killing the process group kills the `exec`, not what it is exec'ing into: + # inside a container that is a different process tree on the same machine. + space.kill() # The wall, and it is worth telling apart: there is no report to read, # so `turns` does not count what this session did. The file it may have # written by then is still read back, which is why `run` reads either way. diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index abfcdf17..987e4bbf 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -102,6 +102,7 @@ from ._extract import ExtractPhase from ._judge import ParentJudge from ._review import CompletionJudge, ParentCodeReview, chain_reviews +from ._sandbox import LocalSandbox, SessionSandbox from ._session import session_home from ._suite import TEST_FAILURE from ._suite import cold_start, preflight @@ -305,6 +306,18 @@ def build_parser() -> argparse.ArgumentParser: "(`reasoning_effort`) rather than as a constant. " "Measured on one fly node against a coding-plan endpoint: " "uncapped 497s, capped at 2048 203-242s, same record") + parser.add_argument("--sandbox", default="auto", + choices=("auto", "container", "off"), + help=("run every agent session inside a container, using the " + "engine's own boundary (`agentdescent.sandbox_container`): " + "only the workspace visible, read-only root, no " + "capabilities, resource ceilings -- with the network " + "inherited, because a session has to reach its model. " + "`auto` uses one when an engine answers and says so when " + "none does; `container` refuses to run without one. " + "Without it the blind property is a line in the brief: " + "4 of 12 episodes in one run read a previous run's output " + "off /tmp, one of them the answer to its own failure")) parser.add_argument("--session-timeout", type=float, default=ClaudeCodeExecutor.TIMEOUT, metavar="SECONDS", help=(f"the wall on one agent session, in seconds (default " @@ -516,6 +529,20 @@ def main(argv=None) -> None: "invent") return + #: One boundary for the whole run, decided once: what a session can see is part of + #: what the run *is*, and a run that isolated some episodes and not others would be + #: two experiments with one set of numbers. + if args.sandbox == "off": + sandbox = LocalSandbox() + else: + sandbox = SessionSandbox(home=session_home()) + if not sandbox.available: + if args.sandbox == "container": + print(f"--sandbox container: {sandbox.reason}", file=sys.stderr) + return + print(f"--sandbox auto: {sandbox.reason}", file=sys.stderr) + sandbox = LocalSandbox() + def _session_kwargs(): # `--timeout` is the timeout on one *model call* -- `_common` says so in its # own help -- and a session is a loop of many calls, so its wall is its own @@ -523,7 +550,7 @@ def _session_kwargs(): # the executor, handed neither, sat on a default nobody in the run had chosen. return dict(model=(args.executor_model or args.model or ""), timeout=float(args.session_timeout or ClaudeCodeExecutor.TIMEOUT), - thinking_tokens=args.thinking_tokens) + thinking_tokens=args.thinking_tokens, sandbox=sandbox) #: Whether the manager, the reviewer and the extractor run as sessions. Decided #: once: a run that asked for them and has no CLI should say so once, not once per @@ -816,6 +843,7 @@ def _superseded(rendered, task, output, reward_): print(f"gate : accepted={judge.accepted} rejected={judge.rejected} " f"partial={judge.partial}") if sessions is not None or use_sessions: + print(f"sandbox : {sandbox.summary()}") # Where to go and read what an episode actually did, and the promise that it # is not in `~/.claude`: a run used to write its transcripts and its TodoWrite # state into the state of whatever session launched it. diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index b4643b1a..c810d834 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -11,6 +11,8 @@ import ast import inspect +import pathlib +import subprocess import json import os import posixpath @@ -29,6 +31,13 @@ from examples.genesis import _jqx as jqx from examples.genesis import _md as md from examples.genesis import _stackvm as stackvm + + +def _no_engine() -> bool: + """Is there a container engine answering? Asked once, at collection.""" + from examples.genesis._sandbox import sandbox_engine + + return sandbox_engine() is None from examples.genesis import genesis_recursive_worlds as genesis from examples.genesis._delegation import (Brief, Delegation, Edit, RecursiveDelegation, render_edits) @@ -2396,6 +2405,114 @@ def test_the_brief_keeps_the_session_inside_the_checkout(): assert "not this repository's state" in brief +# --------------------------------------------------------------------------- +# The boundary: what a session can see, rather than what its work becomes +# --------------------------------------------------------------------------- + +def test_the_sandbox_passes_named_variables_and_rewrites_only_the_agent(): + """Two things the exec has to get right, and both were got wrong first. + + The binary under the mount is the *resolved* one -- `claude` is a symlink out of a + node install here, and exec'ing the link's own path inside the container is + `stat: no such file or directory`. And only the agent's own argv[0] is rewritten: a + shell probe through the same workspace is not the agent, and rewriting it turns + `sh -c 'ls /'` into "Please run /login". + """ + from examples.genesis._sandbox import CONTAINER_HOME, Workspace + + space = Workspace("/host/ws", prefix=["docker", "exec", "-w", "/work", "cid"], + env={"HOME": CONTAINER_HOME}, binary="/opt/pkg/bin/claude", + binary_names=("claude", "/usr/bin/claude")) + got = space.command(["claude", "-p", "hi"], + {"ANTHROPIC_API_KEY": "k", "CLAUDE_CODE_SESSION_ID": "host"}) + assert got[-3:] == ["/opt/pkg/bin/claude", "-p", "hi"] + assert got[:4] == ["docker", "exec", "-w", "/work"] + assert got[got.index("--env") + 1] == "ANTHROPIC_API_KEY=k" + assert f"HOME={CONTAINER_HOME}" in got + # the host's session identity is not on the list, so it cannot arrive by accident + assert not any("CLAUDE_CODE_SESSION_ID" in part for part in got) + # a command that is not the agent goes in untouched + assert space.command(["sh", "-c", "ls /"], {})[-3:] == ["sh", "-c", "ls /"] + + +def test_a_shared_toolchain_directory_is_not_mounted(): + """`/usr/bin/claude` would mean mounting the host's system directories. + + The mount exists to put the agent inside a container that has no agent in it. A + binary that lives somewhere shared has no install directory of its own, and taking + its parent would hand the session exactly what the boundary is for. + """ + from examples.genesis import _sandbox + + calls = {} + + def fake_which(name): + return calls.get(name) + + old = _sandbox.shutil.which + _sandbox.shutil.which = fake_which + try: + calls["claude"] = "/opt/claude-code/bin/claude" + assert _sandbox.toolchain_root("claude") == "/opt/claude-code" + calls["claude"] = "/usr/bin/claude" + assert _sandbox.toolchain_root("claude") is None + calls["claude"] = None + assert _sandbox.toolchain_root("claude") is None + finally: + _sandbox.shutil.which = old + + +def test_without_an_engine_the_run_says_so_rather_than_pretending(tmp_path): + """A fallback that claimed to isolate would be worse than none. + + `LocalSandbox` is what the port did before this module existed, kept as a class so + both paths have one shape -- and so the run reports which one it got. + """ + from examples.genesis._sandbox import LocalSandbox + + local = LocalSandbox(str(tmp_path)) + assert not local.available + space = local.open("x-") + try: + assert space.path.startswith(str(tmp_path)) + assert space.command(["claude", "-p", "hi"], {}) == ["claude", "-p", "hi"] + finally: + space.close() + assert "can read this machine" in local.summary() + + +@pytest.mark.skipif(_no_engine(), reason="needs a container engine") +def test_a_sandboxed_session_cannot_read_the_machine_it_runs_on(tmp_path): + """The claim, checked against a real engine rather than against the flags. + + Four of twelve episodes in one run read a previous run's output off `/tmp`, and one + of them opened the answer to the acceptance failure it had been handed. This is the + same filesystem, asked the same questions, from inside. + """ + from examples.genesis._sandbox import SessionSandbox + + sandbox = SessionSandbox(home=str(tmp_path)) + if not sandbox.available: # engine went away between checks + pytest.skip(sandbox.reason) + space = sandbox.open("test-") + try: + (pathlib.Path(space.path) / "mine.txt").write_text("the work\n") + + def run(script): + out = subprocess.run(space.command(["sh", "-c", script], {}), + capture_output=True, text=True, timeout=90) + return (out.stdout or out.stderr).strip() + + assert "No such file" in run("ls /home/user/agentdescent") + assert run("find / -name 'algo-genesis.md' 2>/dev/null; echo END") == "END" + assert run("ls /tmp | wc -l") == "0" + assert "mine.txt" in run("ls /work") + assert "Read-only file system" in run("touch /etc/x 2>&1 | head -1") + assert run("grep CapEff /proc/self/status").endswith("0000000000000000") + finally: + space.close() + + def test_stackvm_is_deeper_than_minilang_which_is_why_it_exists(): """The second domain is not "harder code" -- it is somewhere for the recursion to go. `src/vm/ops` is a node whose parent is itself a child.""" From e7d0bfd9f674b5d62ff62607599733c3237db790 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 14:39:20 +0000 Subject: [PATCH 25/42] genesis: let a sandboxed run name the image its sessions get The default image is a name rather than a build, so a run never fails at its first episode for want of a build context. What it does not carry is a test runner -- and an episode is told to run the suite it is judged by, so every one of them pays for installing it: a probe session spent two of its turns on pip, one of them on the proxy's CERTIFICATE_VERIFY_FAILED. --sandbox-image lets a machine that has built one point at it. Two lines and a default that does not change. Genesis tests green (the driver's parser is exercised by test_the_session_wall_is_not_the_model_call_timeout); the full suite was green on the commit before this one and this changes one flag. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/genesis/genesis_recursive_worlds.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index 987e4bbf..07e5e6e5 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -102,6 +102,7 @@ from ._extract import ExtractPhase from ._judge import ParentJudge from ._review import CompletionJudge, ParentCodeReview, chain_reviews +from ._sandbox import DEFAULT_IMAGE as SANDBOX_IMAGE from ._sandbox import LocalSandbox, SessionSandbox from ._session import session_home from ._suite import TEST_FAILURE @@ -318,6 +319,13 @@ def build_parser() -> argparse.ArgumentParser: "Without it the blind property is a line in the brief: " "4 of 12 episodes in one run read a previous run's output " "off /tmp, one of them the answer to its own failure")) + parser.add_argument("--sandbox-image", default="", metavar="IMAGE", + help=("the image a sandboxed session runs in (default " + f"{SANDBOX_IMAGE}). The default is a name rather than a " + "build, so a run never fails at its first episode for " + "want of a build context -- but it carries no test " + "runner, and an episode is told to run the suite it is " + "judged by, so every one of them pays for installing it")) parser.add_argument("--session-timeout", type=float, default=ClaudeCodeExecutor.TIMEOUT, metavar="SECONDS", help=(f"the wall on one agent session, in seconds (default " @@ -535,7 +543,7 @@ def main(argv=None) -> None: if args.sandbox == "off": sandbox = LocalSandbox() else: - sandbox = SessionSandbox(home=session_home()) + sandbox = SessionSandbox(home=session_home(), image=args.sandbox_image) if not sandbox.available: if args.sandbox == "container": print(f"--sandbox container: {sandbox.reason}", file=sys.stderr) From 4cc3b7ba58914d03b7763c1acc42bcea9155273c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 14:50:45 +0000 Subject: [PATCH 26/42] genesis: the architect was the one role the sandbox could not reach The driver hands every session role `_session_kwargs()` -- model, wall, reasoning cap, and now the sandbox. Four of the five take `**kwargs` into AgentSession and picked it up; ArchitectSession spells its signature out, so adding the sandbox silently dropped it out of that contract. The run died at phase 1 with `unexpected keyword argument 'sandbox'` *after* printing its whole header, which is the worst shape for this failure: eighteen lines of a working run, then a traceback, and a background task that reports exit 0 a minute after it started. A test now builds all five roles from one kwarg set and asserts each of them carries it, which is the contract the driver has always assumed and never checked. Full suite green. The sandboxed run is past its first phase-1 node with six containers up. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/genesis/_architect_session.py | 5 +++-- tests/test_genesis_example.py | 29 ++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/examples/genesis/_architect_session.py b/examples/genesis/_architect_session.py index eba6fbc7..02f69d7d 100644 --- a/examples/genesis/_architect_session.py +++ b/examples/genesis/_architect_session.py @@ -94,13 +94,14 @@ class ArchitectSession: def __init__(self, *, frozen: Sequence[str] = (), binary: str = "claude", model: str = "", max_turns: int = 0, timeout: float = 900.0, - root: Optional[str] = None, thinking_tokens: int = 0): + root: Optional[str] = None, thinking_tokens: int = 0, + sandbox=None): # `:read_write` upstream, minus the shell -- see `_session.READ_WRITE_TOOLS` # for why the executor gets Bash and a design session does not. self.session = AgentSession( tools=READ_WRITE_TOOLS, frozen=frozen, binary=binary, model=model, max_turns=max_turns or self.TURNS, timeout=timeout, root=root, - thinking_tokens=thinking_tokens) + thinking_tokens=thinking_tokens, sandbox=sandbox) #: Files a session wrote that were not its own record, discarded. An architect #: that starts writing code is the failure mode this counts. The session runs #: in a throwaway copy, so a stray costs nothing but is worth seeing. diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index c810d834..c579409c 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -2513,6 +2513,35 @@ def run(script): space.close() +def test_every_session_role_takes_the_run_s_own_settings(): + """One kwarg set, five roles, and the driver hands it to all of them. + + It hands them `_session_kwargs()` -- model, wall, reasoning cap, sandbox -- and a + role whose signature is spelled out rather than `**kwargs` silently falls out of + that contract. Adding the sandbox did exactly that to the architect, and the run + died at phase 1 with `unexpected keyword argument 'sandbox'` *after* printing its + whole header, which reads like a working run for as long as it takes to scroll. + """ + from examples.genesis._architect_session import ArchitectSession + from examples.genesis._roles import ExtractSession, ManagerSession, ReviewSession + from examples.genesis._sandbox import LocalSandbox + + kwargs = dict(model="m", timeout=123.0, thinking_tokens=64, + sandbox=LocalSandbox()) + roles = [ArchitectSession(frozen=md.FROZEN, **kwargs), + ManagerSession(Delegation, frozen=md.FROZEN, **kwargs), + ReviewSession(contracts=md.CONTRACTS, frozen=md.FROZEN, **kwargs), + ExtractSession(frozen=md.FROZEN, **kwargs)] + for role in roles: + assert role.session._timeout == 123.0 + assert role.session._thinking_tokens == 64 + assert role.session.sandbox is kwargs["sandbox"] + + executor = ClaudeCodeExecutor(frozen=md.FROZEN, **kwargs) + assert executor._timeout == 123.0 and executor._thinking_tokens == 64 + assert executor.sandbox is kwargs["sandbox"] + + def test_stackvm_is_deeper_than_minilang_which_is_why_it_exists(): """The second domain is not "harder code" -- it is somewhere for the recursion to go. `src/vm/ops` is a node whose parent is itself a child.""" From 961663dd673dff709cf215e88e8d4898830e2b41 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 14:58:36 +0000 Subject: [PATCH 27/42] genesis: a killed run leaves its containers running; reap them at the next start A session container outlives its `release` when the process holding it dies. One run was four minutes into phase 2 with eight episodes in flight when the machine running it restarted, and afterwards all eight were still up and idling, holding memory and a workspace mount apiece. The engine already knows how to find them -- they carry its label and a start time, and `ContainerProvider.reap` removes the ones past the TTL, which by construction belong to nobody. Calling it once when the sandbox is built means the next run cleans up after the last one, and the count goes in the run's own report rather than being something you have to go and look for. Housekeeping never fails a run: a reap that raises is a reap that returns zero. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/genesis/_sandbox.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/examples/genesis/_sandbox.py b/examples/genesis/_sandbox.py index 7613e254..e9aaf13a 100644 --- a/examples/genesis/_sandbox.py +++ b/examples/genesis/_sandbox.py @@ -229,8 +229,21 @@ def __init__(self, *, engine: str = "", image: str = "", root: str = "", self.reason = (f"the `{binary}` binary is not in a self-contained install, " "so there is nothing to mount into a container") self.provider = None + #: Containers left behind by a run that was killed, removed at construction. + #: A session's container outlives its `release` when the process holding it + #: dies -- a session container restart took one run with eight episodes in + #: flight, and all eight were still up and idling afterwards. The engine + #: already knows how to find them: they carry its label and a start time, and + #: `reap` removes the ones past the TTL, which by construction are nobody's. + #: (The host workspace directories they were mounted from are a separate + #: lease, and `agentdescent.sandbox`'s own reaper is what clears those.) + self.reaped = 0 if not self.reason: self.provider = _Provider(self, engine=self.engine, image=self._image) + try: + self.reaped = self.provider.reap() + except Exception: # noqa: BLE001 - a run should not fail on housekeeping + self.reaped = 0 #: Containers handed out, for the run's own report. self.opened = 0 @@ -271,7 +284,8 @@ def open(self, prefix: str = "genesis-") -> Workspace: def summary(self) -> str: if not self.available: return f"off ({self.reason})" - return (f"{self.engine} {self._image}, {self.opened} container(s); " + reaped = f", {self.reaped} orphan(s) reaped" if self.reaped else "" + return (f"{self.engine} {self._image}, {self.opened} container(s){reaped}; " f"only the workspace visible, network inherited") From c210db26b62b84b3d47d85f77be38d3dc58a0af6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 19:56:10 +0000 Subject: [PATCH 28/42] genesis: what a session made is not what it wrote An episode is told to run the suite it is judged by. A suite run leaves a `.pytest_cache`, and `_read_tree` skipped `.git`, `.claude` and `__pycache__` but not that -- so the directory reached the accepted version and was materialised into every later workspace, where the next agent reads it as if it were the project. Caught four hours into a live run, in the transcripts. A manager session situated at `src/arena` opened `src/arena/.pytest_cache/`, spent seven turns reading `.gitignore`, `CACHEDIR.TAG`, `README.md` and `v/cache/lastfailed`, and then wrote its plan to `src/arena/.pytest_cache/.genesis/plan.md`. AgentSession reads back the path the caller named and nothing else, so that plan was never read: the episode cost a whole session and returned nothing. One workspace had nested the directory twice -- `src/brain/central_complex/.pytest_cache/.pytest_cache/`. It also shows up from outside as the thing that made the run look like it was slowing down: new code files per check went 18, 27, 14, 10, 4 while sessions per check went 21, 33, 33, 36, 54. ARTIFACT_DIRS now names what a session *makes* rather than writes, and both readers skip it -- the executor's diff and the role sessions' stray count. A virtual environment is in the list without having been seen yet: it is the same failure with three orders of magnitude more files. Full suite green. The live run started before this landed and does not get it; the next one does. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/genesis/_claude_code.py | 11 ++++++--- examples/genesis/_session.py | 33 ++++++++++++++++++++++--- tests/test_genesis_example.py | 42 ++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 7 deletions(-) diff --git a/examples/genesis/_claude_code.py b/examples/genesis/_claude_code.py index 063b40ca..fb3ac8b4 100644 --- a/examples/genesis/_claude_code.py +++ b/examples/genesis/_claude_code.py @@ -61,7 +61,8 @@ from ._delegation import Brief, Edit from ._sandbox import PROVIDER_FILES, LocalSandbox, Workspace -from ._session import available_tools, isolation_flags, run_cli, session_env +from ._session import (ARTIFACT_DIRS, ARTIFACT_FILES, available_tools, + isolation_flags, run_cli, session_env) from ._spatial import SITUATED_EDIT_PROTOCOL # noqa: F401 (documented sibling) from ._world import normalise, owns @@ -268,11 +269,15 @@ def _read_tree(workspace: str) -> Dict[str, str]: """The worktree as a state dict, skipping what is not the project.""" out: Dict[str, str] = {} for base, dirs, files in os.walk(workspace): - dirs[:] = [d for d in dirs if d not in (".git", ".claude", "__pycache__")] + # What the session *made* is not what it wrote: an episode is told to run the + # suite, a suite run leaves `.pytest_cache`, and without this that lands in the + # accepted version and is materialised into every later workspace -- where the + # next agent reads it as if it were the project. See `_session.ARTIFACT_DIRS`. + dirs[:] = [d for d in dirs if d not in ARTIFACT_DIRS] for name in files: full = os.path.join(base, name) rel = os.path.relpath(full, workspace).replace(os.sep, "/") - if rel in PROVIDER_FILES: + if rel in PROVIDER_FILES or name in ARTIFACT_FILES: continue # the sandbox's own bookkeeping, not the node's work try: with open(full, encoding="utf-8") as handle: diff --git a/examples/genesis/_session.py b/examples/genesis/_session.py index d1d58503..3af21d49 100644 --- a/examples/genesis/_session.py +++ b/examples/genesis/_session.py @@ -54,7 +54,8 @@ from ._sandbox import PROVIDER_FILES, LocalSandbox, Workspace -__all__ = ["AgentSession", "HOST_SESSION_VARS", "READ_ONLY_TOOLS", +__all__ = ["ARTIFACT_DIRS", "ARTIFACT_FILES", "AgentSession", + "HOST_SESSION_VARS", "READ_ONLY_TOOLS", "READ_WRITE_TOOLS", "SCRATCH_DIR", "available_tools", "isolation_flags", "run_cli", "session_env", "session_home"] @@ -171,6 +172,31 @@ def available_tools(tools: Sequence[str], flags: Sequence[str]) -> List[str]: #: skips it, and the workspace is deleted either way. SCRATCH_DIR = ".genesis" +#: Directories a session *makes* rather than writes, skipped when the worktree is read +#: back. `.git`, `.claude` and `__pycache__` were here from the start; `.pytest_cache` +#: was not, and an episode is told to run the suite it is judged by, so every one of +#: them makes one. +#: +#: Measured, in a run four hours in: `src/arena/.pytest_cache/` reached the accepted +#: state, was materialised into every later workspace, and a manager session at +#: `src/arena` then spent seven turns reading `CACHEDIR.TAG`, `README.md` and +#: `v/cache/lastfailed` before writing its plan to +#: `src/arena/.pytest_cache/.genesis/plan.md`. :meth:`AgentSession.run` reads back the +#: path the caller named and nothing else, so that plan was never read: the episode +#: cost a session and returned nothing. One of them had nested the directory twice. +#: +#: A virtual environment is the same failure with three orders of magnitude more files, +#: which is why `venv` is here without having been seen yet. +ARTIFACT_DIRS = frozenset({ + ".git", ".claude", "__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache", + ".hypothesis", ".tox", ".nox", "node_modules", ".venv", "venv", "htmlcov", + ".genesis", +}) + +#: The same idea for files that are output rather than work. +ARTIFACT_FILES = frozenset({".coverage", "coverage.xml", ".DS_Store"}) + + #: What upstream's `:read` agents get, mapped onto the CLI's tool names. The one job a #: read-only agent does write -- `CONTEXT.md` -- is granted per role rather than here. #: @@ -244,12 +270,11 @@ def _strays(workspace: str, state: Mapping[str, str], skip = set(read) | set(PROVIDER_FILES) out: List[str] = [] for base, dirs, files in os.walk(workspace): - dirs[:] = [d for d in dirs - if d not in (".git", ".claude", "__pycache__", SCRATCH_DIR)] + dirs[:] = [d for d in dirs if d not in ARTIFACT_DIRS] for name in files: full = os.path.join(base, name) rel = os.path.relpath(full, workspace).replace(os.sep, "/") - if rel in skip: + if rel in skip or name in ARTIFACT_FILES: continue try: with open(full, encoding="utf-8") as handle: diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index c579409c..3498f40e 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -2542,6 +2542,48 @@ def test_every_session_role_takes_the_run_s_own_settings(): assert executor.sandbox is kwargs["sandbox"] +#: A session that does what the brief tells it to: writes a module, writes a test, and +#: runs the suite -- which leaves a `.pytest_cache` behind. +TIDY = ''' +import json, os, sys +if "--version" in sys.argv: + print("0.0.0 (fake)"); raise SystemExit(0) +os.makedirs("src/core", exist_ok=True) +open("src/core/vectors.py", "w").write("def f():\\n return 1\\n") +os.makedirs("src/core/.pytest_cache/v/cache", exist_ok=True) +open("src/core/.pytest_cache/CACHEDIR.TAG", "w").write("Signature: 8a477f597d28d172\\n") +open("src/core/.pytest_cache/v/cache/lastfailed", "w").write("{}") +open(".coverage", "w").write("binary-ish") +print(json.dumps({"is_error": False, "num_turns": 6})) +''' + + +def test_what_a_session_made_is_not_what_it_wrote(tmp_path): + """Running the suite is the job; the cache it leaves is not the work. + + Without this the directory reaches the accepted version and is materialised into + every later workspace. Measured four hours into one run: a manager session at + `src/arena` spent seven turns reading `CACHEDIR.TAG`, `README.md` and + `v/cache/lastfailed`, then wrote its plan to + `src/arena/.pytest_cache/.genesis/plan.md` -- which the caller does not read back, + so the episode cost a session and returned nothing. One workspace had nested the + directory twice. + """ + executor = ClaudeCodeExecutor(frozen=md.FROZEN, binary=_fake_claude(tmp_path, TIDY)) + edits = executor(Brief(world=LocalWorld(version=1, path="src/core", + readonly=md.FROZEN), + objective="o", context="", state=dict(md.initial_files()), + task=md.build_tasks()[0], output="FAIL", reward=0.0, depth=2)) + assert [e.path for e in edits] == ["src/core/vectors.py"] + + # ...and the same for a role session, which reports what it touched + from examples.genesis._session import AgentSession + + session = AgentSession(binary=_fake_claude(tmp_path, TIDY)) + session.run({}, "do it", read=[".genesis/plan.md"]) + assert session.changed == ["src/core/vectors.py"] + + def test_stackvm_is_deeper_than_minilang_which_is_why_it_exists(): """The second domain is not "harder code" -- it is somewhere for the recursion to go. `src/vm/ops` is a node whose parent is itself a child.""" From f75d72edde8397b063242fee5c7d1cfae4e4a998 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 23:55:28 +0000 Subject: [PATCH 29/42] genesis: a tool's own directory is never a node ARTIFACT_DIRS keeps `.pytest_cache` out of the accepted version, which is where the problem starts. It is not where it ends. In the run that turned this up the directory got in anyway, and the managers delegated *into* it: 21 executor episodes were situated at a cache directory, nested as deep as `src/brain/central_complex/.pytest_cache/.pytest_cache/.pytest_cache`. Each was an episode, a container and a session spent reading `CACHEDIR.TAG`. So the delegation refuses it too. A node is a *source* directory; a dot directory is never one, and neither is anything else in ARTIFACT_DIRS. Counted as a mistaken node, which is exactly what it is -- a path read as a node when it is not one -- beside the existing refusals for a file and for a module the path would shadow. Two fences on one failure because they fail differently: the first stops the directory existing in the state at all, and the second holds even when something else puts one there. Full suite green. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/genesis/_delegation.py | 13 +++++++++++++ tests/test_genesis_example.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/examples/genesis/_delegation.py b/examples/genesis/_delegation.py index 19832491..14e7320d 100644 --- a/examples/genesis/_delegation.py +++ b/examples/genesis/_delegation.py @@ -41,6 +41,7 @@ from agentdescent.filetree import parse_tree from ._octopus import three_way +from ._session import ARTIFACT_DIRS from ._world import (CONTEXT_FILE, KNOWN_ISSUES, EpisodeRecord, LocalWorld, WorldLog, directly_at, looks_like_file, normalise, owns, @@ -582,6 +583,18 @@ def _is_node(self, world: LocalWorld, delegation: Delegation, # not yet, which is every file in a formation run until someone writes it. self.mistaken_nodes += 1 return False + if any(part.startswith(".") or part in ARTIFACT_DIRS + for part in path.split("/")): + # A node is a *source* directory. A tool's own directory is not one even + # when it looks like one from here, and `_session.ARTIFACT_DIRS` keeping + # them out of the accepted version is the first fence, not the only one: + # in one run `.pytest_cache` reached the state anyway and the managers + # delegated into it 21 times, nesting three deep -- + # `src/brain/central_complex/.pytest_cache/.pytest_cache/.pytest_cache`. + # Each of those was an episode spent on a cache. Counted as mistaken + # because that is exactly what it is: a directory read as a node. + self.mistaken_nodes += 1 + return False if shadowed_by_module(state, path): self.shadowed_nodes += 1 return False diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index 3498f40e..72621732 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -2584,6 +2584,36 @@ def test_what_a_session_made_is_not_what_it_wrote(tmp_path): assert session.changed == ["src/core/vectors.py"] +def test_a_tool_s_own_directory_is_never_a_node(): + """The second fence on the same failure, and it earned its place. + + `_session.ARTIFACT_DIRS` keeps `.pytest_cache` out of the accepted version, which + is where the problem starts. It is not where it ends: in one run the directory got + in anyway and the managers delegated *into* it 21 times, nesting three deep at + `src/brain/central_complex/.pytest_cache/.pytest_cache/.pytest_cache`. Every one of + those was an episode, a container and a session spent on a cache. + + A node is a source directory. A dot directory is never one, and neither is + `node_modules` -- so the delegation refuses both and counts them where it already + counts a path read as a node when it is not one. + """ + delegation = RecursiveDelegation(manager=lambda brief: [], executor=lambda brief: [], + log=WorldLog(), max_depth=3, + contracts=md.CONTRACTS) + world = LocalWorld(version=1, path="src", readonly=md.FROZEN) + state = dict(md.initial_files()) + + for bad in ("src/.pytest_cache", "src/.pytest_cache/v/cache", + "src/core/.mypy_cache", "src/node_modules", "src/.venv", + "src/core/.pytest_cache/.pytest_cache"): + assert not delegation._is_node(world, Delegation(bad, "o"), state), bad + assert delegation.mistaken_nodes == 6 + + # ...and a real directory beside them still opens + assert delegation._is_node(world, Delegation("src/core", "o"), state) + assert delegation.mistaken_nodes == 6 + + def test_stackvm_is_deeper_than_minilang_which_is_why_it_exists(): """The second domain is not "harder code" -- it is somewhere for the recursion to go. `src/vm/ops` is a node whose parent is itself a child.""" From cc3edac16d32845e23378aeae1d57a98b6d74055 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 16 Sep 2026 00:15:09 +0000 Subject: [PATCH 30/42] genesis: a bracketed note costs its count, never its child A run's phase 1 reported `architect designed 1 nodes, deepest 0` from a record that named six children in plain sight. The architect session was fine -- six turns, no errors, a 3,924-character record with a full routing table. The parser threw it away. The brief asks for `(12 files)` and the pattern read `\d+`, so `(~32 files)` -- "about thirty-two", a perfectly reasonable thing to write -- did not match. The count sat in an optional group that still had to match wherever it appeared, so failing it failed the *whole line*: the child did not lose its size, it vanished. Six lines, six vanished children, one node. The bracket now swallows anything and the number is dug out of it afterwards, so `(~32 files)`, `(about 32 files)`, `(32 files, maybe more)` and `(several files)` all keep their child, and only the last loses its count. Rare and expensive, which is the worst combination: across every run on this machine, 7,244 routing lines carried an exact count and 49 an approximate one -- 0.7%, and one of those 49 landed on a root architect and cost a whole run. A one-node tree has no delegation, no parent review and no spatial contract in it at all. Full suite green. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/genesis/_world.py | 25 ++++++++++++++++++++++--- tests/test_genesis_example.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/examples/genesis/_world.py b/examples/genesis/_world.py index 46363557..acc249b6 100644 --- a/examples/genesis/_world.py +++ b/examples/genesis/_world.py @@ -92,7 +92,7 @@ _ROUTE_LINE = re.compile( r"""^\s*[-*]\s* # a markdown list item `?\s*(?P\.?/?[A-Za-z0-9._\-/]+?)\s*/?`?\s* # the path, backticks optional - (?:\(\s*(?P\d+)\s*files?\s*\)\s*)? # optional "(12 files)" + (?:\((?P[^)]*)\)\s*)? # an optional note in brackets, any content (?:$|[-=]+>|\u2192|:) # end of line, '->', an arrow, or a colon """, re.VERBOSE) @@ -144,6 +144,25 @@ def parse_routing(body: str) -> List[str]: +def _count(note: Optional[str]) -> int: + """The file count out of a routing line's bracketed note, or 0. + + The note is read loosely on purpose. The brief asks for `(12 files)`, and an + architect wrote `(~32 files)` -- "about thirty-two", which is a perfectly good + thing to say and which an exact `\\d+` refused. Worse, the refusal did not fall + back to "no count": the optional group could not match, so the *whole line* failed + and the child vanished. One run designed a tree of exactly one node that way, from + a record that named six children in plain sight. + + So the bracket now swallows anything and the number is dug out afterwards: a note + that carries no number costs its count, never its child. + """ + if not note: + return 0 + digits = re.search(r"\d+", note) + return int(digits.group()) if digits else 0 + + def parse_route_sizes(body: str) -> Dict[str, int]: """``{path: how many files the architect said it holds}`` for one record. @@ -166,10 +185,10 @@ def parse_route_sizes(body: str) -> Dict[str, int]: if not inside: continue match = _ROUTE_LINE.match(line) - if match and match.group("files"): + if match and _count(match.group("files")): path = normalise(match.group("path")) if path: - out.setdefault(path, int(match.group("files"))) + out.setdefault(path, _count(match.group("files"))) return out def parse_routes(body: str) -> List[Tuple[str, str]]: diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index 72621732..d4408651 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -2614,6 +2614,40 @@ def test_a_tool_s_own_directory_is_never_a_node(): assert delegation.mistaken_nodes == 6 +def test_a_bracketed_note_costs_its_count_never_its_child(): + """`(~32 files)` designed a tree of one node. + + The brief asks for `(12 files)` and the pattern read `\\d+`, so "about + thirty-two" -- a perfectly reasonable thing for an architect to write -- did not + match. And because the count sat in an *optional group* that still had to match + where it appeared, failing it failed the whole line: the child did not lose its + size, it vanished. One run's phase 1 reported `architect designed 1 nodes, + deepest 0` from a record that named six children in plain sight, and a one-node + tree is a run with no delegation, no parent review and no spatial contract in it + at all. + """ + from examples.genesis._world import parse_route_sizes, parse_routes + + def table(note): + return f"## Routing Table\n- `./src/brain/` {note} -> the brain\n" + + for note in ("(32 files)", "(~32 files)", "(≈32 files)", "(about 32 files)", + "(32 files, maybe more)", "(several files)", ""): + assert parse_routes(table(note)) == [("src/brain", "the brain")], note + + # the count is read where there is one, and simply absent where there is not + assert parse_route_sizes(table("(~32 files)")) == {"src/brain": 32} + assert parse_route_sizes(table("(several files)")) == {} + assert parse_route_sizes(table("")) == {} + + # and the whole record parses the way the run needs it to + record = ("# src\n\n## Routing Table\n" + "- `./src/neural/` (~9 files) -> the spiking engine\n" + "- `./src/brain/` (~32 files) -> the brain\n" + "- `./src/body/` (~6 files) -> the body\n") + assert [p for p, _ in parse_routes(record)] == ["src/neural", "src/brain", "src/body"] + + def test_stackvm_is_deeper_than_minilang_which_is_why_it_exists(): """The second domain is not "harder code" -- it is somewhere for the recursion to go. `src/vm/ops` is a node whose parent is itself a child.""" From 70a0ecfe037d60e2e852dd4b9ce9cde2d702f022 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 16 Sep 2026 01:02:04 +0000 Subject: [PATCH 31/42] genesis: tell the failure evidence whose repair it is The blind failure ended "Write your own test that reproduces this, put it beside the code it covers, and make both pass". Every agent in a rollout is shown that same text, and at a node it reads as an order to fix the import here -- so eight different nodes each wrote their own _cli.py, 79 writes between them, src/brain/olfactory/_cli.py alone 21 times. Only the one at the repository root could ever have resolved the root import; the rest were local imitations of a file that has to exist elsewhere, and work the parent then had to undo. The evidence now says it is the repository's, not necessarily yours: judge it against what you own, write the test only if the fix belongs under your path, and otherwise name the path in the final message and leave it alone. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/genesis/_suite.py | 28 ++++++++++++++++++++++++++-- tests/test_genesis_example.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/examples/genesis/_suite.py b/examples/genesis/_suite.py index 55c2fb30..ace7637d 100644 --- a/examples/genesis/_suite.py +++ b/examples/genesis/_suite.py @@ -370,6 +370,25 @@ def review(parent, returned): #: #: So in blind mode the acceptance suite never enters the repository, the prompt is the #: requirement it came from, and the tests *in* the repository are the agents' own. +#: The evidence an episode is given, and what it is *for*. +#: +#: The failing assertion belongs to the whole repository, and every episode in the +#: rollout is handed the same one -- including a leaf five levels down that owns none +#: of the code it names. The old wording said only "write your own test that reproduces +#: this and make both pass", which reads as an instruction whatever node you are at, +#: and a node may only write under itself. So a leaf at `src/brain/olfactory`, told to +#: reproduce ``No module named '_cli'``, built `src/brain/olfactory/_cli.py`: a local +#: imitation of a root-level entry point, which cannot fix a root-level import. +#: +#: Measured across the runs on one machine: **eight** distinct `_cli.py` paths, one per +#: node, 79 writes between them, and only the one at the repository root could ever have +#: mattered. `src/brain/olfactory/_cli.py` alone was written 21 times. +#: +#: The fix is to say what the evidence is rather than only what to do with it. The +#: request channel the brief already describes -- "say so in your final message and the +#: agent responsible for that path will be asked" -- is the right answer when the fix +#: is somebody else's, and it was being drowned out by a sentence that sounded like a +#: task. BLIND_FAILURE = """An acceptance assertion is failing. You cannot read it -- the \ acceptance suite is not in this repository and never will be. What you get is the \ requirement it was written from, its name, and what it reported: @@ -378,8 +397,13 @@ def review(parent, returned): and it reports: {output} -Write your own test that reproduces this, put it beside the code it covers, and make \ -both pass. A test you write is the only test you can read.""" +This is the **repository's** evidence, not necessarily yours: every agent in this \ +rollout is shown the same failure, and the code that has to change may be nobody's but \ +the root's. Judge it against what you own. If the fix belongs under your path, write \ +your own test that reproduces it, put it beside the code it covers, and make both pass \ +-- a test you write is the only test you can read. If it belongs somewhere else, say \ +which path and why in your final message and leave it alone: a local imitation of a \ +file that has to exist elsewhere fixes nothing and is work the parent has to undo.""" def llm_manager(complete) -> Callable[[Brief], Sequence[Delegation]]: diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index d4408651..20d915a4 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -2648,6 +2648,36 @@ def table(note): assert [p for p, _ in parse_routes(record)] == ["src/neural", "src/brain", "src/body"] +def test_the_failure_evidence_says_whose_it_is(): + """Every episode is shown the same failing assertion; most of them do not own it. + + The old wording ended "write your own test that reproduces this and make both + pass", which reads as an instruction at whatever node you happen to be, and a node + may only write under itself. So a leaf at `src/brain/olfactory`, handed the + repository's `No module named '_cli'`, built `src/brain/olfactory/_cli.py` -- a + local imitation of a root-level entry point, which cannot fix a root-level import. + Across the runs on one machine that produced eight distinct `_cli.py` paths, one + per node, 79 writes between them, and only the root's could ever have mattered. + + The brief already has the right answer in it -- report the need upward and let the + responsible agent handle it -- and it was being drowned out. + """ + from examples.genesis._suite import BLIND_FAILURE + + text = BLIND_FAILURE.format(prompt="req 3", output="FAIL:ModuleNotFoundError") + assert "repository's** evidence, not necessarily yours" in text + assert "Judge it against what you own" in text + assert "say which path and why" in text + assert "a local imitation of a file that has to exist elsewhere fixes nothing" in text + # the conditional is what makes it a judgement rather than an order + assert "If the fix belongs under your path" in text + + # and it still reaches the episode through the executor's brief + brief = CLAUDE_CODE_BRIEF.format(path="src/brain", objective="o", + frozen="REQUIREMENTS.md", failure=text) + assert "not necessarily yours" in brief and "Write only under `src/brain`" in brief + + def test_stackvm_is_deeper_than_minilang_which_is_why_it_exists(): """The second domain is not "harder code" -- it is somewhere for the recursion to go. `src/vm/ops` is a node whose parent is itself a child.""" From 010899dfd65c53e503018392099fb1084e339601 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 16 Sep 2026 01:32:26 +0000 Subject: [PATCH 32/42] genesis: stop the file-count cap eating whole episodes The spatial contract caps a proposal at max_files_per_diff, and the cap rejects the diff rather than the files over the line -- so an episode one file past it contributes nothing. At 6, counting distinct paths written across 309 productive sessions, 11% of the episodes that did work lost all of it; the largest was carrying 23 files. Nothing said so: the two existing counters are about authority (wrote outside its subtree, mistook a file for a node) and neither moves for this, so a run that lost an eighth of its work and a run whose agents had nothing to say printed the same header. Two fixes. The drop is counted, as discarded_diffs=N (M files) in the world summary. And the cap stops pretending to be a trust region: the trust region is the node's subtree, enforced edit by edit, and twelve files under one node are not more dangerous than six. For a session executor it is now a runaway guard at 64 -- three times the largest legitimate episode, where only a loop that dumps a tree reaches it. 24 is where it stops binding at all. A single completion asked for whole files keeps 6, because there six really is one. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- docs/algo-genesis.md | 44 ++++++++++++++++++++ examples/genesis/_spatial.py | 18 +++++++- examples/genesis/_world.py | 36 +++++++++++++++- examples/genesis/genesis_recursive_worlds.py | 16 ++++++- tests/test_genesis_example.py | 44 ++++++++++++++++++++ 5 files changed, 154 insertions(+), 4 deletions(-) diff --git a/docs/algo-genesis.md b/docs/algo-genesis.md index 3cdf1df2..1f7a2147 100644 --- a/docs/algo-genesis.md +++ b/docs/algo-genesis.md @@ -755,6 +755,50 @@ them was still listening two hours later with its working directory already dele Sessions now lead their own process group and the group is killed on the way out — after the wall, and after a clean finish too. +### The cap that discards the episode rather than the surplus + +A third thing the transcripts showed, found by counting rather than reading. The spatial +contract caps a proposal at `max_files_per_diff`, and the cap rejects the **diff**, not +the files over the line — so an episode one file past it contributes nothing at all. +Counting distinct paths written per session across 309 productive sessions on this +machine: + +| files written in one episode | sessions | at a cap of 6 | +|---|---:|---| +| 1–6 | 274 | kept | +| 7–12 | 32 | **lost whole** | +| 13–23 | 3 | **lost whole** | + +**11% of the episodes that did work contributed none of it**, the largest of them +carrying 23 files, and nothing in the run said so: the two violation counters are about +*authority* — an agent wrote outside its subtree, or mistook a file for a node — and +neither moves for this. A run that lost an eighth of its work and a run whose agents had +nothing to say printed the same header. + +Both halves were wrong, and they are separate fixes. + +The drop is now **counted** — `discarded_diffs=N (M files)` in the world summary — +because a silent loss is the thing this port keeps finding. + +And the cap is no longer a trust region, because it was never doing that job here. The +trust region is the node's **subtree**, enforced edit by edit by the spatial contract: +twelve files under `src/brain/olfactory/` are not more dangerous than six, and the +parent's verdict and the frozen suite judge both the same way. A file count on top of +that is a third bound that only ever fires on legitimate work. So for a session executor +it is set where only pathology reaches it — a loop that dumps a tree — and nowhere near +a node's build: + +| cap | episodes lost whole, of 309 | +|---:|---:| +| 6 | 35 (11.3%) | +| 12 | 3 (1.0%) | +| **24** | **0** | +| 64 | 0 | + +24 is where it stops binding; **64** is the number, three times the largest legitimate +episode. A single completion asked for whole files keeps the tight 6 — that executor +proposes one or two files, and six there really is a runaway. + ### A session launched from a session is not that session The transcripts turned up a second thing, and it is the more expensive one. A run diff --git a/examples/genesis/_spatial.py b/examples/genesis/_spatial.py index f8e255ae..a7d29d64 100644 --- a/examples/genesis/_spatial.py +++ b/examples/genesis/_spatial.py @@ -183,6 +183,13 @@ class SpatialContract: #: evaluator. Enforced here for *proposals* and, where the frozen files are #: what score the candidate, by the runner's pristine overlay. frozen: Sequence[str] = () + #: The cap is on the *diff*, so exceeding it discards the whole proposal and not + #: the surplus -- an episode one file over contributes nothing. That makes it a + #: guard against a runaway rather than a trust region: the trust region here is + #: the node's own subtree, enforced above edit by edit, and it does not get + #: tighter because the work took more files. Size it above what one episode of + #: the run's executor legitimately produces; `genesis_recursive_worlds` measures + #: that per executor and says where the numbers come from. max_files_per_diff: int = 4 max_file_bytes: int = DEFAULT_MAX_FILE_BYTES #: Where dropped edits are counted. Optional so the strategy can be used on @@ -243,7 +250,16 @@ def to_diff(self, state, proposal, author, base_version, target) -> Optional[Dif continue # would be rejected as oversized anyway if state.get(path) != content: ops[path] = content - if not ops or len(ops) > self.max_files_per_diff: + if not ops: + return None + if len(ops) > self.max_files_per_diff: + # The two drops above lose one edit; this one loses the episode. A + # session that spent forty turns writing nine files, running the suite + # and getting it green contributes none of the nine, and used to do so + # without leaving a trace anywhere in the run. Counted now -- see + # `WorldLog.discarded_diffs` for what the silence was hiding. + if self.log is not None: + self.log.note_discarded_diff(len(ops)) return None fingerprint = stable_hash(tuple(sorted((k, v) for k, v in ops.items()))) return Diff(diff_id=f"{author}:{fingerprint & 0xFFFFFFFF:08x}:{base_version}", diff --git a/examples/genesis/_world.py b/examples/genesis/_world.py index acc249b6..7091471a 100644 --- a/examples/genesis/_world.py +++ b/examples/genesis/_world.py @@ -675,6 +675,8 @@ def __init__(self) -> None: self._rework: Dict[str, str] = {} self._contract_violations = 0 self._shape_violations = 0 + self._discarded_diffs = 0 + self._discarded_files = 0 self._counter = 0 # -- episodes ---------------------------------------------------------- @@ -739,6 +741,11 @@ def note_shape_violation(self, n: int = 1) -> None: with self._lock: self._shape_violations += n + def note_discarded_diff(self, files: int) -> None: + with self._lock: + self._discarded_diffs += 1 + self._discarded_files += files + @property def shape_violations(self) -> int: """Edits dropped for making the key space stop being a tree. @@ -750,6 +757,30 @@ def shape_violations(self) -> int: with self._lock: return self._shape_violations + @property + def discarded_diffs(self) -> int: + """Episodes whose work was thrown away whole for being too many files. + + The two violation counters above are about *authority*: an agent wrote + somewhere it may not. This one is about *size*, and it is the only drop + that costs the run an entire episode rather than one edit -- the cap is on + the diff, so a session that wrote nine good files contributes none of them. + + It exists because the drop used to be silent. Counting distinct file paths + written across 309 productive sessions on this machine: 11% wrote more than + six files and so lost everything, the largest 23 files. A run that lost an + eighth of its work and a run whose agents had nothing to say printed the + same header. See ``discarded_files`` for what those episodes were carrying. + """ + with self._lock: + return self._discarded_diffs + + @property + def discarded_files(self) -> int: + """How many files the discarded diffs were carrying, summed.""" + with self._lock: + return self._discarded_files + @property def contract_violations(self) -> int: """Edits dropped for escaping their author's subtree. @@ -769,4 +800,7 @@ def summary(self) -> str: f"rejected={verdicts.get('rejected', 0)} " f"rework={verdicts.get('rework', 0)} " f"contract_violations={self.contract_violations} " - f"shape_violations={self.shape_violations}") + f"shape_violations={self.shape_violations} " + f"discarded_diffs={self.discarded_diffs}" + + (f" ({self.discarded_files} files)" + if self._discarded_diffs else "")) diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index 07e5e6e5..56be9a99 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -635,8 +635,20 @@ def _session_kwargs(): + (f", missing {', '.join(gaps)}" if gaps else "")) log = WorldLog() - strategy = SpatialContract(initial_files=initial, frozen=spec.FROZEN, - log=log, max_files_per_diff=6) + # The cap rejects the *diff*, not the files over the line, so every number here + # is a number of episodes thrown away whole. For a session executor that makes it + # a runaway guard and nothing else: the trust region is already the node's own + # subtree, which the contract enforces edit by edit, and twelve files under + # `src/brain/olfactory/` are not more dangerous than six. So it is set where only + # pathology reaches it. Counting distinct paths written across 309 productive + # sessions on this machine, the largest legitimate episode carried 23 files, and + # a six-file cap lost 11% of them; at 24 nothing legitimate is lost at all and 64 + # leaves three times that headroom, while still catching a loop that dumps a tree. + # A single completion asked for whole files is a different thing -- it proposes + # one or two, and six is already a runaway there -- so it keeps the tight number. + strategy = SpatialContract(initial_files=initial, frozen=spec.FROZEN, log=log, + max_files_per_diff=( + 64 if args.executor == "claude-code" else 6)) # `manager.ex` states the parent's validation as three things: review the child's # results, run the tests, and reject anti-patterns it can see in the code. The # middle one is a number and was all this port had; the zero-field run is what diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index 20d915a4..d3359d36 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -337,6 +337,50 @@ def test_an_edit_outside_the_authors_subtree_is_dropped_and_counted(): assert log.contract_violations == 1 +def test_a_diff_over_the_file_cap_loses_the_whole_episode_and_says_so(): + """The cap discards the proposal, not the surplus -- so it has to be counted. + + The two violation counters are about authority and cost one edit each. This one + is about size and costs the episode: a session that wrote nine files, ran the + suite and got it green contributes none of the nine. Measured across 309 + productive sessions on this machine, 11% wrote more than six files. + """ + log = WorldLog() + strategy = _strategy(log=log, max_files_per_diff=2) + edits = [Edit("src", f"src/m{i}.py", "x") for i in range(3)] + assert strategy.to_diff(strategy.initial(), render_edits(edits, "r"), + "w0", 1, "world") is None + assert log.discarded_diffs == 1 and log.discarded_files == 3 + # Authority was never in question, so neither violation counter moves. + assert log.contract_violations == 0 and log.shape_violations == 0 + assert "discarded_diffs=1 (3 files)" in log.summary() + # And under the cap the same edits go through, which is what makes the count + # a measure of the cap rather than of the work. + assert strategy.to_diff(strategy.initial(), render_edits(edits[:2], "r"), + "w0", 1, "world") is not None + assert log.discarded_diffs == 1 + + +def test_the_session_executor_gets_a_cap_no_real_episode_reaches(): + """A runaway guard, not a trust region -- the subtree is the trust region. + + The cap throws away the diff rather than the surplus, so any number it binds at + is a number of episodes lost whole. Across 309 productive sessions on this + machine the largest legitimate episode carried 23 files; 24 is where the cap + stops binding at all. It has to sit above that, not at the size of a node build. + """ + caps = [] + for node in ast.walk(ast.parse(inspect.getsource(genesis.main))): + if not isinstance(node, ast.keyword) or node.arg != "max_files_per_diff": + continue + assert isinstance(node.value, ast.IfExp), ( + "the cap no longer depends on which executor is running") + caps = [n.value for n in (node.value.body, node.value.orelse)] + assert all(isinstance(c, int) for c in caps), caps + # 24 is the measured ceiling of legitimate work; below it the cap eats episodes. + assert max(caps) > 24 > min(caps), caps + + def test_a_frozen_path_is_refused_even_to_the_root(): strategy = _strategy() proposal = render_edits([Edit("", "spec/CONTEXT.md", "rewritten")], "r") From 5ae2da95e7c2bb6fa9e5d40c192d3b338f0d7caf Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 16 Sep 2026 02:39:08 +0000 Subject: [PATCH 33/42] genesis: report progress during the growth phase Between "Growing the world" and the final summary the run printed nothing at all, and for a formation run that is hours. Asking whether anything was being accepted meant reading CLI transcripts and looking inside live containers -- and a live worktree holds the base state PLUS whatever the session has written since, which is not the accepted state. Two readings out of three drawn that way were wrong. evolve() already takes an on_round hook; the run now passes one and prints a line per merger sweep with reward, artifact size, committed, rejected, rollouts and elapsed. RoundInfo.reasons goes on the end because committed=0 has two causes that need opposite fixes: the gate refused the work, or the work never reached the gate. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- examples/genesis/genesis_recursive_worlds.py | 23 +++++++++++++++++++ tests/test_genesis_example.py | 24 ++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index 56be9a99..5d9a94dc 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -762,8 +762,31 @@ def _superseded(rendered, task, output, reward_): print(f"\nGrowing the world ({args.workers} workers, " f"{'barrier-free' if args.asynchronous else 'synchronous DP'})...\n") + def _progress(info) -> None: + """One line per merger sweep, because a run that prints nothing is a run + you have to do archaeology on. + + Between "Growing the world" and the final summary this port printed + **nothing at all** -- for a formation run that is hours. Finding out + whether work was being accepted meant reading episode transcripts and + peering into live containers, and two readings out of three were wrong + because what a worktree holds is the base state *plus* whatever the + session has written so far, which is not the accepted state and never was. + + `RoundInfo.reasons` is the field that matters: "the gate says my proposals + do not help" and "they never reached the gate" both show as committed=0 + and need opposite fixes. + """ + reasons = " ".join(f"{k}={v}" for k, v in sorted(info.reasons.items())) + print(f" sweep {info.round:>3} reward={info.held_out_reward:.3f} " + f"files={info.n_items:<4} committed={info.committed:<3} " + f"rejected={info.rejected:<3} rollouts={info.rollouts:<4} " + f"{int(info.elapsed_s)//60}m" + (f" [{reasons}]" if reasons else ""), + flush=True) + result = evolve( tasks, spec.reward, run=run, propose=_superseded, strategy=strategy, + on_round=_progress, artifact_id="world", blast_radius=SKILL_BLAST_RADIUS, # Say the budget outright rather than letting `rounds` be reinterpreted: # `--episodes` is a count of ROOT episodes in both arms, and under the diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index d3359d36..ac19e8f2 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -337,6 +337,30 @@ def test_an_edit_outside_the_authors_subtree_is_dropped_and_counted(): assert log.contract_violations == 1 +def test_the_growth_phase_reports_progress_rather_than_going_silent(): + """`evolve` takes an on_round hook and the run has to use it. + + Without it the run printed one line at the start of the growth phase and the + next line hours later, at the end. Asking "is anything being accepted" then + meant reading CLI transcripts and looking inside live containers -- and what a + live worktree holds is the base state *plus* the session's own writes, so two + readings out of three were wrong. `RoundInfo.reasons` is the field that + separates "the gate refused my work" from "my work never reached the gate". + """ + tree = ast.parse(inspect.getsource(genesis.main)) + hook = [n for n in ast.walk(tree) + if isinstance(n, ast.keyword) and n.arg == "on_round"] + assert hook, "the growth phase no longer reports progress" + name = hook[0].value + assert isinstance(name, ast.Name), "on_round should be a named local, not a lambda" + body = [n for n in ast.walk(tree) + if isinstance(n, ast.FunctionDef) and n.name == name.id] + assert body, f"{name.id} is passed as on_round but not defined in main()" + printed = ast.dump(body[0]) + for field in ("committed", "rejected", "reasons", "held_out_reward"): + assert field in printed, f"the progress line never reports {field}" + + def test_a_diff_over_the_file_cap_loses_the_whole_episode_and_says_so(): """The cap discards the proposal, not the surplus -- so it has to be counted. From 7eb979f6264319a8b5e89cb08bbcd63599a20e2a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 16 Sep 2026 03:04:54 +0000 Subject: [PATCH 34/42] genesis: run siblings together, and stop workers duplicating each other A rollout is a walk of the whole Context Tree with one Claude Code session per node, and it was a for loop. On the fly tree -- 8 nodes, deepest 2 -- a session takes five to fifteen minutes, so a rollout took about an hour. py-spy on a live run, 75 minutes in, showed all four workers still inside their FIRST propose(), three levels deep in nested _episode frames. The accepted state held three files. That read as an acceptance problem and was not one: the parent gate accepts a tie, four of the five reviews that had run said ACCEPT, and the staleness policy keeps a card whose reward is merely unchanged. Nothing was being rejected. Nothing had finished. And --workers counts whole tree walks, so four workers were four independent descents of the same eight nodes -- two of them on src/brain/navigation at the same moment, doing the same node's work, of which one result could survive. Siblings now run concurrently and are folded in delegations order, so the proposal a rollout returns is unchanged. The spatial contract is what makes that safe: a child writes only under its own subtree, so two siblings cannot touch the same path. --node-workers says how wide one level may get, --max-sessions bounds what the endpoint actually sees, and the bound lives in run_cli rather than in the pools -- sessions are the scarce thing, and they never nest, so one semaphore there covers the whole tree with no way for a parent to deadlock on its own children. Every counter the port reports is now incremented under a lock. x += 1 is a read and a write with a bytecode boundary between them, and these numbers are the port's evidence. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- docs/algo-genesis.md | 47 +++++++++ examples/genesis/_claude_code.py | 24 +++-- examples/genesis/_delegation.py | 104 ++++++++++++++----- examples/genesis/_session.py | 60 ++++++++++- examples/genesis/genesis_recursive_worlds.py | 34 +++++- tests/test_genesis_example.py | 79 ++++++++++++++ 6 files changed, 309 insertions(+), 39 deletions(-) diff --git a/docs/algo-genesis.md b/docs/algo-genesis.md index 1f7a2147..ea78d519 100644 --- a/docs/algo-genesis.md +++ b/docs/algo-genesis.md @@ -799,6 +799,53 @@ a node's build: episode. A single completion asked for whole files keeps the tight 6 — that executor proposes one or two files, and six there really is a runaway. +### One rollout was an hour, and four workers spent it on the same tree + +A rollout here is `RecursiveDelegation.propose`: a walk of the whole Context Tree with +one Claude Code session per node. It was a `for` loop, so the walk was serial — and on +the `fly` tree (8 nodes, deepest 2) a session takes five to fifteen minutes, which makes +a rollout about an hour. `py-spy` on a live run, 75 minutes in: + +``` +Thread-1: _episode → _episode → _episode → _episode +Thread-2: _accountability_pass → _episode → _episode → _episode +Thread-3: _accountability_pass → _episode → _episode +``` + +All four workers still inside their **first** `propose()`. The accepted state held three +files. That looked like an acceptance problem and was not one: the parent gate accepts a +tie, the reviewer said ACCEPT in four of the five reviews that had run, and the staleness +policy keeps a card whose reward is merely unchanged. Nothing was being rejected. Nothing +had finished. + +Worse, `--workers` is the number of *whole tree walks* in flight, so four workers were +four independent descents of the same eight nodes — two of them sat on +`src/brain/navigation` at the same moment, doing the same node's work, of which only one +result could ever survive. + +So siblings run together now, and the knobs say which kind of parallelism you are asking +for: + +| flag | what it multiplies | +|---|---| +| `--workers` | whole tree walks at once (keep small; they duplicate each other) | +| `--node-workers` | siblings one manager runs at once (this is the useful one) | +| `--max-sessions` | the hard bound on what the endpoint sees; defaults to their product | + +The spatial contract is what makes the concurrency safe rather than a merge problem: a +child may write only under its own subtree, so two siblings cannot touch the same path, +and each gets its own throwaway worktree branched from the same base. What the parent +does *with* the results — the octopus fold, the conflict check, the requests it adopts — +is unchanged and still runs in `delegations` order, so the same results produce the same +proposal. The bound lives in `run_cli`, not in the thread pools, because sessions are the +scarce thing and a pool per level would multiply; sessions never nest (a manager's own +session runs before and after its children's, never during), so one semaphore there +bounds the whole tree with no way for a parent to deadlock on its own children. + +Every counter the port reports is now incremented under a lock. `x += 1` is a read and a +write with a bytecode boundary between them, and this port's claim is that its numbers +are measured. + ### A session launched from a session is not that session The transcripts turned up a second thing, and it is the more expensive one. A run diff --git a/examples/genesis/_claude_code.py b/examples/genesis/_claude_code.py index fb3ac8b4..68fd0f08 100644 --- a/examples/genesis/_claude_code.py +++ b/examples/genesis/_claude_code.py @@ -53,6 +53,7 @@ import os import shutil import subprocess +import threading from typing import Callable, Dict, List, Mapping, Optional, Sequence from agentdescent.filetree import match_any, materialize @@ -157,6 +158,11 @@ def __init__(self, *, frozen: Sequence[str] = (), binary: str = "claude", #: Sessions run, and what came back. `failed` is the total; `timeouts` is the #: part of it that was still working when the wall came, and `edits` is #: counted for those too -- a session that ran out of time wrote what it wrote. + #: Siblings run concurrently, so every counter below is incremented + #: from more than one thread. `x += 1` is a read and a write with a + #: bytecode boundary between them; without this the numbers this port + #: reports would quietly undercount exactly when the run is busiest. + self._tally = threading.Lock() self.sessions = 0 self.failed = 0 self.timeouts = 0 @@ -171,14 +177,18 @@ def __call__(self, brief: Brief) -> Sequence[Edit]: before = dict(brief.state) materialize(before, space.path) self._write_settings(space.path) - self.sessions += 1 + with self._tally: + self.sessions += 1 ok = self._run(space, owner, brief) if not ok: - self.failed += 1 + with self._tally: + self.failed += 1 after = _read_tree(space.path) edits = _diff(before, after, owner) - self.edits += sum(1 for e in edits if owns(owner, e.path)) - self.requests += sum(1 for e in edits if not owns(owner, e.path)) + with self._tally: + self.edits += sum(1 for e in edits if owns(owner, e.path)) + with self._tally: + self.requests += sum(1 for e in edits if not owns(owner, e.path)) return edits finally: space.close() @@ -250,11 +260,13 @@ def _run(self, space: Workspace, owner: str, brief: Brief) -> bool: # `turns` -- which is why the counter read 309 for a run whose transcripts # hold 2 607. The diff is taken either way: the caller reads the worktree # after this returns, and half-finished work is still work. - self.timeouts += 1 + with self._tally: + self.timeouts += 1 return False try: report = json.loads(out.decode("utf-8", "replace") or "{}") - self.turns += int(report.get("num_turns") or 0) + with self._tally: + self.turns += int(report.get("num_turns") or 0) return not report.get("is_error", code != 0) except Exception: # noqa: BLE001 - no JSON is not a reason to lose the diff return code == 0 diff --git a/examples/genesis/_delegation.py b/examples/genesis/_delegation.py index 14e7320d..4f645e33 100644 --- a/examples/genesis/_delegation.py +++ b/examples/genesis/_delegation.py @@ -35,7 +35,9 @@ from __future__ import annotations import json -from dataclasses import dataclass, replace +import threading +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field, replace from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple from agentdescent.filetree import parse_tree @@ -156,6 +158,12 @@ class RecursiveDelegation: log: WorldLog max_depth: int = 2 max_edits: int = 4 + #: How many siblings one manager may have in flight at once. 1 is the serial + #: walk this port started with; above 1 the children of a node run together and + #: are folded in `delegations` order afterwards, so the proposal is unchanged. + #: The real bound on load is `_session.set_session_limit`, not this -- this only + #: says how wide one level may get. + node_workers: int = 1 root_path: str = "" #: Human-supplied, read-only files every agent is shown in full, wherever it #: stands -- the specification, the constraints, the validation contract. The @@ -197,6 +205,13 @@ class RecursiveDelegation: #: Child nodes opened at a path their parent did not yet route to, and #: therefore written into the parent's routing table. routes_opened: int = 0 + #: Guards every counter below. They are incremented from sibling threads now, + #: and `self.x += 1` is a read and a write with a bytecode boundary between + #: them -- two siblings landing there together lose a count. This port's whole + #: claim is that its numbers are measured, so the numbers have to survive the + #: concurrency it introduced. + _tally: threading.Lock = field(default_factory=threading.Lock, repr=False, + compare=False) #: Delegations refused because the target was a file, not a node. mistaken_nodes: int = 0 #: Delegations refused because the directory would shadow a sibling module of @@ -235,6 +250,11 @@ class RecursiveDelegation: # -- the ProposalPolicy protocol --------------------------------------- + def _bump(self, name: str, n: int = 1) -> None: + """One counter, incremented under the tally lock. See `_tally`.""" + with self._tally: + setattr(self, name, getattr(self, name) + n) + def propose(self, ctx) -> Sequence[str]: state = _state_of(ctx.rendered) self.last_state = dict(state) @@ -267,7 +287,7 @@ def _rollout(self, ctx, root: LocalWorld, state: Mapping[str, str], # A need that reached the top of this episode's chain and still fell # outside its root's authority. Counted rather than dropped quietly: it # means the decomposition put work where nobody could do it. - self.unmet_requests += len(unmet) + self._bump("unmet_requests", len(unmet)) edits = self._bound(edits) if not edits: @@ -309,21 +329,21 @@ def _episode(self, world: LocalWorld, objective: str, state: Mapping[str, str], # repository root. Resolve it against the node before anything # else looks at it, and count the ones that needed it. path, relative = resolve_edit_path(state, world.path, raw.path) - self.resolved_relative += int(relative) + self._bump("resolved_relative", int(relative)) if raw.content is not None and shadows_package(state, path): # A file written beside a node directory of the same name makes # that node unreachable from every import -- see `shadows_package`. - self.shadowing_edits += 1 + self._bump("shadowing_edits", 1) continue produced.append(Edit(owner=world.path, path=path, content=raw.content, kind=raw.kind)) produced += self._refresh_record(world, state) for edit in produced: if edit.path.endswith(CONTEXT_FILE) and state.get(edit.path) != edit.content: - self.record_updates += 1 + self._bump("record_updates", 1) edits = [e for e in produced if owns(world.path, e.path)] requests = [e for e in produced if not owns(world.path, e.path)] - self.requests_raised += len(requests) + self._bump("requests_raised", len(requests)) record = self.log.record(EpisodeRecord( agent_id=self.log.next_id("executor"), role="executor", path=world.path, depth=depth, version=world.version, @@ -363,11 +383,43 @@ def _episode(self, world: LocalWorld, objective: str, state: Mapping[str, str], #: What this episode's review returned work over, carried into the manager's #: own turn below -- see `_accountability_pass`. findings: List[Tuple[str, str, str]] = [] - for delegation in delegations: - child = world.delegate(delegation.path) - returned, asked, child_record = self._episode( - child, delegation.objective, state, ctx, depth=depth + 1, - rollout=rollout, base=child_base) + # Siblings run **concurrently**, and are folded **in order**. + # + # What made this serial was not the contract; it was a `for` loop. The + # contract is the reason it does not have to be: a child may write only + # under its own subtree, so two siblings cannot touch the same path, and + # each gets its own throwaway worktree branched from the same `child_base`. + # What the parent then does with the results -- the octopus fold, the + # conflict check, the pending requests -- is left exactly as it was, in + # `delegations` order, so the merge stays deterministic and a rerun with the + # same results produces the same proposal. + # + # The cost of not doing this, measured on an 8-node tree: one rollout is a + # depth-first walk with one Claude Code session per node, at five to fifteen + # minutes each, so a rollout took about an hour -- and four workers each + # walked the *same* tree, two of them landing on `src/brain/navigation` at + # the same moment to do the same node's work, of which only one result could + # ever survive. After 75 minutes `py-spy` showed all four workers still + # inside their first `propose()`. Nothing was being rejected; nothing had + # finished. + # + # `_session.set_session_limit` is what keeps this from becoming a flood -- + # the bound is on sessions, not on this pool, because sessions are the + # scarce thing and a pool per level would multiply. + pairs = [(d, world.delegate(d.path)) for d in delegations] + if self.node_workers > 1 and len(pairs) > 1: + with ThreadPoolExecutor(max_workers=min(self.node_workers, len(pairs)), + thread_name_prefix="genesis-sibling") as pool: + futures = [pool.submit(self._episode, child, d.objective, state, ctx, + depth=depth + 1, rollout=rollout, + base=child_base) + for d, child in pairs] + outcomes = [f.result() for f in futures] + else: + outcomes = [self._episode(child, d.objective, state, ctx, depth=depth + 1, + rollout=rollout, base=child_base) + for d, child in pairs] + for (delegation, child), (returned, asked, child_record) in zip(pairs, outcomes): children.append(child_record) child_record.verdict, child_record.reason = self._judge( child, returned, asked, brief) @@ -379,7 +431,7 @@ def _episode(self, world: LocalWorld, objective: str, state: Mapping[str, str], mine = [replace(e, owner=world.path) for e in asked if owns(world.path, e.path)] if mine: - self.adopted_requests += len(mine) + self._bump("adopted_requests", len(mine)) self._fold(held, owner_of, mine, world, state) pending += [e for e in asked if not owns(world.path, e.path)] if child_record.verdict == "rejected": @@ -480,7 +532,7 @@ def _accountability_pass(self, world: LocalWorld, objective: str, else: amended[edit.path] = edit.content if findings: - self.accountability_findings += len(findings) + self._bump("accountability_findings", len(findings)) brief = Brief(world=world, objective=_with_findings(f"review and accountability: {objective}", world.path, findings), @@ -490,18 +542,18 @@ def _accountability_pass(self, world: LocalWorld, objective: str, out: List[Edit] = list(self._refresh_record(world, amended)) for raw in self.executor(brief) or (): path, relative = resolve_edit_path(amended, world.path, raw.path) - self.resolved_relative += int(relative) + self._bump("resolved_relative", int(relative)) if raw.content is not None and shadows_package(amended, path): # The accountability turn is where this actually happened: the manager # at `src/potentials/kernels` delegated `lennard_jones/` to a child and # then wrote `lennard_jones.py` beside it. - self.shadowing_edits += 1 + self._bump("shadowing_edits", 1) elif directly_at(world.path, path): out.append(Edit(owner=world.path, path=path, content=raw.content, kind=raw.kind)) else: - self.accountability_declined += 1 - self.accountability_edits += len(out) + self._bump("accountability_declined", 1) + self._bump("accountability_edits", len(out)) return out def _refresh_record(self, world: LocalWorld, @@ -519,7 +571,7 @@ def _refresh_record(self, world: LocalWorld, revised = self.refine(world.path, state) if not revised or revised == state.get(key): return [] - self.records_revised += 1 + self._bump("records_revised", 1) return [Edit(owner=world.path, path=key, content=revised, kind="context")] def _fold(self, held: Dict[str, Edit], owner_of: Dict[str, str], @@ -545,15 +597,15 @@ def _fold(self, held: Dict[str, Edit], owner_of: Dict[str, str], continue # the two agreed; nothing to merge if mine.content is None or edit.content is None: conflicts.append(edit.path) # a delete against an edit - self.sibling_conflicts += 1 + self._bump("sibling_conflicts", 1) continue fused = three_way(state.get(edit.path, ""), mine.content, edit.content) if fused is None: conflicts.append(edit.path) - self.sibling_conflicts += 1 + self._bump("sibling_conflicts", 1) continue held[edit.path] = replace(mine, content=fused) - self.sibling_merges += 1 + self._bump("sibling_merges", 1) return conflicts def _is_node(self, world: LocalWorld, delegation: Delegation, @@ -581,7 +633,7 @@ def _is_node(self, world: LocalWorld, delegation: Delegation, if path in state or looks_like_file(path): # `in state` catches a file that exists; the shape catches one that does # not yet, which is every file in a formation run until someone writes it. - self.mistaken_nodes += 1 + self._bump("mistaken_nodes", 1) return False if any(part.startswith(".") or part in ARTIFACT_DIRS for part in path.split("/")): @@ -593,10 +645,10 @@ def _is_node(self, world: LocalWorld, delegation: Delegation, # `src/brain/central_complex/.pytest_cache/.pytest_cache/.pytest_cache`. # Each of those was an episode spent on a cache. Counted as mistaken # because that is exactly what it is: a directory read as a node. - self.mistaken_nodes += 1 + self._bump("mistaken_nodes", 1) return False if shadowed_by_module(state, path): - self.shadowed_nodes += 1 + self._bump("shadowed_nodes", 1) return False return True @@ -643,7 +695,7 @@ def _routing_note(self, world: LocalWorld, state: Mapping[str, str], body, changed = updated, True if not changed: return None - self.routes_opened += len(opened) + self._bump("routes_opened", len(opened)) # Creating the record outranks the work; adding a line to one does not. return Edit(owner=world.path, path=key, content=body, kind="context" if key in state else "record") @@ -714,7 +766,7 @@ def _bound(self, edits: Sequence[Edit]) -> List[Edit]: ordered = [by_path[p] for p in sorted(by_path, key=lambda p: (order.get(by_path[p].kind, 2), p))] if len(ordered) > self.max_edits: - self.truncated += len(ordered) - self.max_edits + self._bump("truncated", len(ordered) - self.max_edits) ordered = ordered[:self.max_edits] return ordered diff --git a/examples/genesis/_session.py b/examples/genesis/_session.py index 3af21d49..5112126a 100644 --- a/examples/genesis/_session.py +++ b/examples/genesis/_session.py @@ -43,6 +43,7 @@ import json import os import signal +import contextlib import subprocess import tempfile import threading @@ -55,7 +56,7 @@ from ._sandbox import PROVIDER_FILES, LocalSandbox, Workspace __all__ = ["ARTIFACT_DIRS", "ARTIFACT_FILES", "AgentSession", - "HOST_SESSION_VARS", "READ_ONLY_TOOLS", + "HOST_SESSION_VARS", "READ_ONLY_TOOLS", "set_session_limit", "READ_WRITE_TOOLS", "SCRATCH_DIR", "available_tools", "isolation_flags", "run_cli", "session_env", "session_home"] @@ -216,6 +217,40 @@ def available_tools(tools: Sequence[str], flags: Sequence[str]) -> List[str]: READ_WRITE_TOOLS = ("Read", "Write", "Edit", "Glob", "Grep") +#: How many sessions may be in flight at once, across the whole run. Every role +#: goes through `run_cli`, and sessions never nest -- a manager's own session runs +#: before and after its children's, never during -- so one semaphore here bounds the +#: whole tree without any chance of a parent deadlocking on its own children. +#: +#: It exists because siblings run concurrently now. Without a bound, a tree of +#: branching factor *b* and depth *d* puts up to b^d sessions on one endpoint at +#: once, and the endpoint this port measured answers a 429 within 39 seconds but +#: takes minutes when it is simply given more than it will serve. +_SLOTS: Optional[threading.BoundedSemaphore] = None +_SLOTS_LOCK = threading.Lock() + + +def set_session_limit(limit: int) -> None: + """Bound concurrent sessions for this process. ``limit <= 0`` removes the bound.""" + global _SLOTS + with _SLOTS_LOCK: + _SLOTS = threading.BoundedSemaphore(limit) if limit > 0 else None + + +@contextlib.contextmanager +def _slot(): + with _SLOTS_LOCK: + sem = _SLOTS + if sem is None: + yield + return + sem.acquire() + try: + yield + finally: + sem.release() + + def run_cli(command: Sequence[str], *, cwd: str, env: Mapping[str, str], timeout: float) -> Tuple[bytes, int, bool]: """Run one session to completion or to the wall; ``(stdout, code, timed_out)``. @@ -232,6 +267,12 @@ def run_cli(command: Sequence[str], *, cwd: str, env: Mapping[str, str], different events with different fixes: a session that hit the wall was still working, and one that died at 26 seconds was not. """ + with _slot(): + return _run_cli(command, cwd=cwd, env=env, timeout=timeout) + + +def _run_cli(command: Sequence[str], *, cwd: str, env: Mapping[str, str], + timeout: float) -> Tuple[bytes, int, bool]: proc = subprocess.Popen(list(command), cwd=cwd, env=dict(env), stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=True) @@ -322,6 +363,11 @@ def __init__(self, *, tools: Sequence[str] = READ_WRITE_TOOLS, #: Sessions run, and what came back. `failed` is the total; `timeouts` is #: the half of it that was still working when the wall arrived, which is the #: only one of the two that `--session-timeout` can do anything about. + #: Siblings run concurrently, so every counter below is incremented + #: from more than one thread. `x += 1` is a read and a write with a + #: bytecode boundary between them; without this the numbers this port + #: reports would quietly undercount exactly when the run is busiest. + self._tally = threading.Lock() self.sessions = 0 self.failed = 0 self.timeouts = 0 @@ -344,9 +390,11 @@ def run(self, state: Mapping[str, str], prompt: str, *, try: materialize(dict(state), space.path) self._write_settings(space.path) - self.sessions += 1 + with self._tally: + self.sessions += 1 if not self._invoke(space, prompt): - self.failed += 1 + with self._tally: + self.failed += 1 out: Dict[str, str] = {} for rel in read: try: @@ -417,11 +465,13 @@ def _invoke(self, space: Workspace, prompt: str) -> bool: # The wall, and it is worth telling apart: there is no report to read, # so `turns` does not count what this session did. The file it may have # written by then is still read back, which is why `run` reads either way. - self.timeouts += 1 + with self._tally: + self.timeouts += 1 return False try: report = json.loads(out.decode("utf-8", "replace") or "{}") - self.turns += int(report.get("num_turns") or 0) + with self._tally: + self.turns += int(report.get("num_turns") or 0) return not report.get("is_error", code != 0) except Exception: # noqa: BLE001 - no JSON is not a reason to lose the work return code == 0 diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index 5d9a94dc..5b7b8a97 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -104,7 +104,7 @@ from ._review import CompletionJudge, ParentCodeReview, chain_reviews from ._sandbox import DEFAULT_IMAGE as SANDBOX_IMAGE from ._sandbox import LocalSandbox, SessionSandbox -from ._session import session_home +from ._session import session_home, set_session_limit from ._suite import TEST_FAILURE from ._suite import cold_start, preflight from ._octopus import OctopusConflict, git_available @@ -203,7 +203,25 @@ def build_parser() -> argparse.ArgumentParser: "nodes deep, stackvm four")) parser.add_argument("--episodes", type=int, default=24, help="total ROOT episodes; one root episode is one rollout") - parser.add_argument("--workers", type=int, default=4) + parser.add_argument("--workers", type=int, default=4, + help="rollouts in flight. Each one is a whole tree walk, so " + "raising this makes several workers descend the SAME " + "tree at once -- measured on an 8-node tree, two of them " + "landed on the same node at the same moment to do the " + "same work, of which one result could survive. Prefer " + "--node-workers for parallelism and keep this small") + parser.add_argument("--node-workers", type=int, default=4, metavar="N", + help="siblings one manager may run at once. The spatial " + "contract is what makes it safe -- a child writes only " + "under its own subtree, so siblings cannot collide -- and " + "the parent still folds their results in delegation " + "order, so the proposal is unchanged. 1 is the serial " + "walk: on an 8-node tree that was about an hour per " + "rollout, one session at a time") + parser.add_argument("--max-sessions", type=int, default=0, metavar="N", + help="hard bound on Claude Code sessions in flight across the " + "whole run, which is the number the endpoint actually " + "sees. 0 derives it from --workers x --node-workers") parser.add_argument("--depth", type=int, default=3, help="maximum recursive delegation depth (v,p) -> (v,q)") parser.add_argument("--max-turns", type=int, default=0, @@ -422,6 +440,17 @@ def main(argv=None) -> None: print(f"Merge : {merge}" + ("" if git_available() else " [git missing: every contested file falls back]")) print(f"Gate : {gate}") + # The three numbers that decide how long a run takes, together, because each is + # misleading alone. `--workers` is whole tree walks; `--node-workers` is how wide + # one level of a walk gets; their product is what the endpoint sees, and that is + # the number a rate limit is about. + node_workers = max(1, args.node_workers) + in_flight = args.max_sessions or (args.workers * node_workers) + set_session_limit(in_flight) + print(f"Parallel : {args.workers} rollout(s) x {node_workers} sibling(s) " + f"= up to {in_flight} session(s) on the endpoint at once" + + (" [serial siblings: one rollout is a depth-first walk, " + "one session at a time]" if node_workers == 1 else "")) print("Episode : " + ("one headless Claude Code session in a throwaway worktree, " f"up to {args.max_turns or ClaudeCodeExecutor.ROOT_TURNS} " f"turns at the root and " @@ -720,6 +749,7 @@ def _refine(path, state): executor=(sessions if sessions is not None else spec.llm_executor(complete) if complete else spec.offline_executor), log=log, max_depth=args.depth, max_edits=4, contracts=spec.CONTRACTS, + node_workers=max(1, args.node_workers), readonly=spec.FROZEN, # Three links, and the middle one is the only one that can ask "did you test # what you just wrote". A domain whose tests are the agents' own supplies it. diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index ac19e8f2..aef521a0 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -16,6 +16,7 @@ import json import os import posixpath +import threading import time import pytest @@ -337,6 +338,84 @@ def test_an_edit_outside_the_authors_subtree_is_dropped_and_counted(): assert log.contract_violations == 1 +def test_siblings_run_together_and_are_folded_in_order(): + """Concurrency where it costs nothing, determinism where it would cost the merge. + + A rollout is a walk of the whole tree with one Claude Code session per node. + Serial, on an 8-node tree that was about an hour per rollout -- `py-spy` showed + all four workers still inside their first `propose()` after 75 minutes, two of + them on the same node doing the same work. The spatial contract is why siblings + may overlap: a child writes only under its own subtree, so they cannot collide. + The fold must not overlap, so it stays in `delegations` order and the proposal + a rollout returns is the same either way. + """ + kids = ["src/a", "src/b", "src/c", "src/d"] + lock = threading.Lock() + + def _run(node_workers): + live, widest = [], [0] + + def executor(b): + with lock: + live.append(b.world.path) + widest[0] = max(widest[0], len(live)) + time.sleep(0.05) + with lock: + live.remove(b.world.path) + return [Edit(b.world.path, f"{b.world.path}/e.py", "x = 1\n")] + + policy = RecursiveDelegation( + manager=lambda b: ([Delegation(k, "build it") for k in kids] + if b.world.path == "src" else []), + executor=executor, log=WorldLog(), max_depth=3, root_path="src", + max_edits=16, accountability=False, node_workers=node_workers) + proposals = policy.propose(_proposal_ctx({"src/CONTEXT.md": _TABLE}, + Task(id="t", prompt="x"))) + paths = [e["path"] for e in parse_situated_edits(proposals[0])] + return widest[0], [p for p in paths if p.endswith("e.py")] + + widest_serial, folded_serial = _run(1) + assert widest_serial == 1, "serial mode ran siblings at the same time" + assert len(folded_serial) == len(kids), folded_serial + + widest_parallel, folded_parallel = _run(4) + assert widest_parallel > 1, "siblings still ran one at a time" + # Completion order is whatever the pool hands back; the fold is not. + assert folded_parallel == folded_serial + + +def test_the_run_bounds_what_the_endpoint_sees(): + """`--workers` x `--node-workers` is what reaches the API, so it is what the run + has to bound. Sessions never nest -- a manager's own session runs before and after + its children's, never during -- so one semaphore in `run_cli` bounds the whole tree + with no way for a parent to deadlock waiting on its own children.""" + tree = ast.parse(inspect.getsource(genesis.main)) + assert [n for n in ast.walk(tree) if isinstance(n, ast.Call) + and isinstance(n.func, ast.Name) and n.func.id == "set_session_limit"], \ + "the run never bounds concurrent sessions" + from examples.genesis import _session + try: + _session.set_session_limit(2) + held, widest, lock = [], [0], threading.Lock() + + def _one(): + with _session._slot(): + with lock: + held.append(1) + widest[0] = max(widest[0], len(held)) + time.sleep(0.05) + with lock: + held.pop() + threads = [threading.Thread(target=_one) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join() + assert widest[0] == 2, f"the bound let {widest[0]} through" + finally: + _session.set_session_limit(0) + + def test_the_growth_phase_reports_progress_rather_than_going_silent(): """`evolve` takes an on_round hook and the run has to use it. From d32bea34d0ae921c411f2ff0b0513a341bbf3e5c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 16 Sep 2026 07:10:43 +0000 Subject: [PATCH 35/42] genesis: a signed-in run declares its toolset instead of shipping all of them --bare removes the inherited tool schemas and a signed-in run cannot use it: it sets CLAUDE_CODE_SIMPLE=1, and that same switch makes the CLI refuse to read OAuth (measured: duration_api_ms 0 and an authentication error). The two cannot be separated, so such a run got the whole prompt -- and the whole prompt is mostly schemas. Asked to list its tools, one of these sessions names forty-two of them: Artifact, CronCreate, DesignSync, PushNotification, Workflow, ShowOnboardingRolePicker and the rest, none reachable from a throwaway worktree, all priced every turn. --allowedTools does not help; it auto-approves, and every other schema is sent regardless. What decides which schemas exist is which tools are defined, and --agents defines its own. Measured on one endpoint with the same one-line prompt: 24 550 tokens of context per turn with --allowedTools, 6 522 with an --agents entry declaring five tools, 8 042 for the port's own executor command (three more tools, Bash among them), against 4 868 under --bare. Two thirds of the tax, gone, with the sign-in intact. lean_agent_flags returns nothing when --bare is already in play, so a keyed run still takes the better path and the declaration is only what a sign-in gets instead. The remaining gap to --bare is the system prompt, which --agents cannot touch. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- docs/algo-genesis.md | 33 ++++++++++++++++++++ examples/genesis/_claude_code.py | 5 +++ examples/genesis/_session.py | 53 +++++++++++++++++++++++++++++++- tests/test_genesis_example.py | 37 ++++++++++++++++++++++ 4 files changed, 127 insertions(+), 1 deletion(-) diff --git a/docs/algo-genesis.md b/docs/algo-genesis.md index ea78d519..ee1293d1 100644 --- a/docs/algo-genesis.md +++ b/docs/algo-genesis.md @@ -755,6 +755,39 @@ them was still listening two hours later with its working directory already dele Sessions now lead their own process group and the group is killed on the way out — after the wall, and after a clean finish too. +### A signed-in run pays for thirty-seven tools it cannot reach + +`--bare` is what removes the inherited schemas, and it cannot be used by a run that is +**signed in** rather than keyed: it sets `CLAUDE_CODE_SIMPLE=1`, and that same switch +makes the CLI refuse to read OAuth. Measured, with a valid sign-in and no key: +`duration_api_ms: 0` and an authentication error. The two cannot be separated. + +So a signed-in run got the full prompt, and the full prompt is mostly tool schemas. +Asked to list what it has, such a session names **forty-two** tools — `Artifact`, +`ArtifactComments`, `ArtifactData`, `CronCreate`, `CronDelete`, `CronList`, +`DesignSync`, `PushNotification`, `ShowOnboardingRolePicker`, `Workflow`, `SendUserFile` +and the rest — when the run asked for four. `--allowedTools` is an auto-approve list: it +says what may be *called*, and every other schema is sent anyway. + +What decides which schemas exist is which tools are **defined**, and `--agents` defines +its own. Measured on one endpoint with the same one-line prompt: + +| | context per turn | | +|---|---:|---| +| signed in, `--allowedTools Read,Edit,Glob,Grep` | **24 550** | 1.00× | +| signed in, `--agents` declaring five tools | **6 522** | 0.27× | +| signed in, the port's own executor command | **8 042** | 0.33× | +| keyed, `--bare` | 4 868 | 0.20× | + +The executor's number is higher than the bare probe because it declares three more +tools — `Write`, `TodoWrite` and `Bash`, the last being the only way a session can run +the suite it is judged by. The 3 174 that `--bare` still saves is the system prompt +itself, which `--agents` cannot touch: eight sections, and small beside the schemas. + +`lean_agent_flags` returns nothing when `--bare` is already in play, so the two +mechanisms never both run — a key still takes the better path, and the declaration is +what a sign-in gets instead. + ### The cap that discards the episode rather than the surplus A third thing the transcripts showed, found by counting rather than reading. The spatial diff --git a/examples/genesis/_claude_code.py b/examples/genesis/_claude_code.py index 68fd0f08..3813bc9a 100644 --- a/examples/genesis/_claude_code.py +++ b/examples/genesis/_claude_code.py @@ -63,6 +63,7 @@ from ._delegation import Brief, Edit from ._sandbox import PROVIDER_FILES, LocalSandbox, Workspace from ._session import (ARTIFACT_DIRS, ARTIFACT_FILES, available_tools, + lean_agent_flags, isolation_flags, run_cli, session_env) from ._spatial import SITUATED_EDIT_PROTOCOL # noqa: F401 (documented sibling) from ._world import normalise, owns @@ -226,6 +227,10 @@ def _command(self, prompt: str, turns: int, "--max-turns", str(turns), "--allowedTools", ",".join(tools), "--disallowedTools", "WebFetch,WebSearch,Task"] + flags + # Declaring the toolset is what keeps the other thirty-seven schemas out of a + # signed-in run; `--allowedTools` above only auto-approves. No-ops under + # `--bare`. See `_session.lean_agent_flags`. + command += lean_agent_flags(tools, flags) if self._model: command += ["--model", self._model] return command diff --git a/examples/genesis/_session.py b/examples/genesis/_session.py index 5112126a..6082e93c 100644 --- a/examples/genesis/_session.py +++ b/examples/genesis/_session.py @@ -58,7 +58,8 @@ __all__ = ["ARTIFACT_DIRS", "ARTIFACT_FILES", "AgentSession", "HOST_SESSION_VARS", "READ_ONLY_TOOLS", "set_session_limit", "READ_WRITE_TOOLS", "SCRATCH_DIR", "available_tools", - "isolation_flags", "run_cli", "session_env", "session_home"] + "isolation_flags", "lean_agent_flags", "run_cli", "session_env", + "session_home"] #: How the host tells a CLI it spawns who it is. Inherited, every session in a run @@ -142,6 +143,53 @@ def isolation_flags(env: Mapping[str, str]) -> List[str]: return ["--bare", "--strict-mcp-config"] +#: The name the lean agent is registered under. Arbitrary, but it has to match +#: between `--agents` and `--agent`. +LEAN_AGENT = "genesis" + + +def lean_agent_flags(tools: Sequence[str], flags: Sequence[str]) -> List[str]: + """`--agents`/`--agent` declaring only the tools a role uses, or nothing. + + `--allowedTools` is an auto-approve list: it says what may be *called*, and every + other schema is sent anyway. What decides which schemas exist is which tools are + **defined**, and a custom agent defines its own. Measured on one endpoint with the + same one-line prompt, signed in rather than keyed: + + | | context per turn | + |---|---:| + | `--allowedTools Read,Edit,Glob,Grep` | **24 550** | + | a `--agents` entry declaring five tools | **6 522** | + | `--bare` (needs a key) | 4 868 | + + Forty-two tools arrive by default — `Artifact`, `CronCreate`, `DesignSync`, + `PushNotification`, `Workflow`, `ShowOnboardingRolePicker` and the rest — and a + session writing code in a throwaway worktree can use none of them. Asked to list + what it has, an agent declared this way names only the tools in `tools`. + + It is the fallback for a run **signed in rather than keyed**: `--bare` is the + better answer and is used whenever there is a key, but it sets + `CLAUDE_CODE_SIMPLE=1`, and that same switch is what makes the CLI refuse to read + OAuth — measured, `duration_api_ms: 0` and an authentication error. The two cannot + be separated, so a signed-in run takes this instead: 6 522 rather than 4 868, and + not 24 550. + + Returns nothing when `--bare` is already in `flags`, because there the schemas are + gone anyway and a second mechanism would only be another thing to keep true. + """ + if "--bare" in flags or not tools: + return [] + spec = {LEAN_AGENT: { + "description": "implements one node of a recursive software world", + # Deliberately thin: the brief is the instruction, and an agent prompt that + # said anything substantive would be a second, quieter brief competing with it. + "prompt": "Follow the instruction you are given.", + "tools": list(tools), + }} + return ["--agents", json.dumps(spec, separators=(",", ":")), + "--agent", LEAN_AGENT] + + def available_tools(tools: Sequence[str], flags: Sequence[str]) -> List[str]: """The requested tools, minus the one bare mode does not carry. @@ -444,6 +492,9 @@ def _command(self, prompt: str, env: Optional[Mapping[str, str]] = None, "--max-turns", str(self._max_turns), "--allowedTools", ",".join(tools), "--disallowedTools", ",".join(denied)] + flags + # The schemas follow what is *defined*, not what is allowed. No-op under + # `--bare`; this is what a signed-in run gets instead. See `lean_agent_flags`. + command += lean_agent_flags(tools, flags) if self._model: command += ["--model", self._model] return command diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index aef521a0..9dcaff92 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -338,6 +338,43 @@ def test_an_edit_outside_the_authors_subtree_is_dropped_and_counted(): assert log.contract_violations == 1 +def test_a_signed_in_run_declares_its_toolset_instead_of_shipping_forty_two(): + """`--allowedTools` auto-approves; what decides the schemas is what is *defined*. + + Measured on one endpoint with the same one-line prompt, signed in rather than + keyed: 24 550 tokens of context per turn with `--allowedTools Read,Edit,Glob,Grep`, + 6 522 with a `--agents` entry declaring five tools, 4 868 under `--bare`. Forty-two + tools arrive by default -- Artifact, CronCreate, DesignSync, Workflow and the rest + -- and none of them is reachable from a worktree. `--bare` is the better answer and + is used whenever the run has a key; it sets CLAUDE_CODE_SIMPLE=1, and that same + switch makes the CLI refuse to read OAuth, so a signed-in run takes this instead. + """ + from examples.genesis._session import LEAN_AGENT, lean_agent_flags + wanted = ["Read", "Edit", "Glob", "Grep", "Bash"] + + # Keyed: `--bare` already removed the schemas, so this adds nothing. + assert lean_agent_flags(wanted, ["--bare", "--strict-mcp-config"]) == [] + + flags = lean_agent_flags(wanted, []) + assert flags[0] == "--agents" and flags[2:] == ["--agent", LEAN_AGENT] + spec = json.loads(flags[1]) + assert list(spec) == [LEAN_AGENT] + # Exactly what the role asked for -- a tool left out here is a tool the session + # does not have, which is the point, and one added is a schema nobody needs. + assert spec[LEAN_AGENT]["tools"] == wanted + assert lean_agent_flags([], []) == [], "no tools is not an empty agent" + + +def test_every_session_role_declares_its_toolset(): + """Both command builders, because the saving is per session and there are five.""" + from examples.genesis._claude_code import ClaudeCodeExecutor + from examples.genesis._session import AgentSession + for cls in (ClaudeCodeExecutor, AgentSession): + src = inspect.getsource(cls._command) + assert "lean_agent_flags" in src, ( + f"{cls.__name__}._command ships every schema to a signed-in run") + + def test_siblings_run_together_and_are_folded_in_order(): """Concurrency where it costs nothing, determinism where it would cost the merge. From 9c4311ecfd677b085407e83b157bfe188b2899ed Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 16 Sep 2026 07:30:35 +0000 Subject: [PATCH 36/42] genesis: say up front that a signed-in run cannot use the container sandbox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A key is a string in the environment and crosses into a container with it, which is why a keyed run is isolated and authenticated at once. A sign-in is not a string: the CLI reaches the endpoint through the host's session ingress, which the container does not have and which this module does not put there. So every containerised session answered "Not logged in · Please run /login", and the run found out one episode at a time -- sessions=4 failed=4 edits=0, architect designed 0 nodes, a whole run spent on a condition that was knowable before the first episode started. SessionSandbox now reports it beside the two reasons it already gave, a missing container engine and an unmountable toolchain, and degrades to a plain directory with the reason printed. The remedy it names is an API key, which is also the arrangement with the smallest prompt, since a key is what --bare needs. Running signed-in without the sandbox is the other option and is worth reading carefully: the session is then on the host as the host's user and reaches everything the sign-in reaches anyway, so what that trade gives up is the isolation, not the exposure. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- docs/algo-genesis.md | 34 ++++++++++++++++++++++++++++++++++ examples/genesis/_sandbox.py | 28 ++++++++++++++++++++++++++++ tests/test_genesis_example.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+) diff --git a/docs/algo-genesis.md b/docs/algo-genesis.md index ee1293d1..7b1660e3 100644 --- a/docs/algo-genesis.md +++ b/docs/algo-genesis.md @@ -755,6 +755,40 @@ them was still listening two hours later with its working directory already dele Sessions now lead their own process group and the group is killed on the way out — after the wall, and after a clean finish too. +### A signed-in run cannot use the container sandbox at all + +Worse than the prompt size, and found the same way. A key is a string in the +environment, and it crosses into a container with it — that is how `CONTAINER_ENV` +works and why a keyed run is isolated and authenticated at once. A **sign-in** is not a +string: the CLI reaches the endpoint through the host's session ingress, which the +container does not have. + +So every containerised session answered `Not logged in · Please run /login`. The run's +own summary, for four episodes of four: + +``` +Phase 1 : architect designed 0 nodes, deepest 0, 1 replies unusable +claude code : sessions=4 failed=4 timeout=0 turns=4 edits=0 requests=0 +architect : sessions=1 failed=1 timeout=0 turns=1 strays=0 +``` + +A whole run for a condition that was knowable before the first episode started. The +sandbox now says so up front, beside the two reasons it already gave — no container +engine, and a toolchain that cannot be mounted — and degrades to a plain directory with +the reason printed rather than failing an episode at a time. + +That leaves three arrangements, and the first is the one to prefer: + +| | container sandbox | context per turn | where the credential is | +|---|---|---:|---| +| **an API key** | yes | **4 868** (`--bare`) | an environment variable, inside the container | +| signed in, `--sandbox off` | no | 8 042 | on the host, which the session shares anyway | +| signed in, container | — | — | cannot authenticate | + +The middle row is worth reading carefully, because it looks safer than it is: a session +running without the sandbox is on the host as the host's user, so it reaches everything +the sign-in reaches regardless. The isolation is what was lost, not the exposure. + ### A signed-in run pays for thirty-seven tools it cannot reach `--bare` is what removes the inherited schemas, and it cannot be used by a run that is diff --git a/examples/genesis/_sandbox.py b/examples/genesis/_sandbox.py index e9aaf13a..92ca62b7 100644 --- a/examples/genesis/_sandbox.py +++ b/examples/genesis/_sandbox.py @@ -84,6 +84,22 @@ DEFAULT_IMAGE = "python:3.12-slim" +def signed_in_only(env: Optional[Mapping[str, str]] = None) -> bool: + """True when the CLI authenticates as a signed-in user rather than from a key. + + A key is a string in the environment and crosses into a container with it. A + sign-in is not: the CLI reaches the endpoint through the host's session ingress, + which the container does not have and which this module does not put there. So the + two arrangements differ in whether an isolated session can authenticate at all, and + a run needs to know which one it is *before* it spends episodes finding out. + """ + out = os.environ if env is None else env + if out.get("ANTHROPIC_API_KEY") or out.get("ANTHROPIC_AUTH_TOKEN"): + return False + return any(out.get(name) for name in + ("CLAUDE_CODE_REMOTE", "CLAUDE_CODE_PROVIDER_MANAGED_BY_HOST")) + + def sandbox_engine(preferred: str = "") -> Optional[str]: """The container engine this machine can actually use, or ``None``. @@ -228,6 +244,18 @@ def __init__(self, *, engine: str = "", image: str = "", root: str = "", elif self.toolchain is None: self.reason = (f"the `{binary}` binary is not in a self-contained install, " "so there is nothing to mount into a container") + elif signed_in_only(): + # The third way a container cannot work, and the one that used to be + # silent. A signed-in CLI does not authenticate from an environment key: + # it goes through the host's session ingress, which `CONTAINER_ENV` does + # not carry and is not this module's to carry. Inside a container the CLI + # therefore answers `Not logged in`, and the run finds out one episode at + # a time -- measured, `sessions=4 failed=4`, a whole run for a condition + # that was knowable before the first one started. + self.reason = ("this CLI is signed in rather than keyed, and a signed-in " + "session cannot authenticate from inside a container; " + "sessions run in a plain directory and can read this " + "machine. Give the run an API key for an isolated one") self.provider = None #: Containers left behind by a run that was killed, removed at construction. #: A session's container outlives its `release` when the process holding it diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index 9dcaff92..0a65ae70 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -18,6 +18,7 @@ import posixpath import threading import time +from unittest import mock import pytest @@ -338,6 +339,33 @@ def test_an_edit_outside_the_authors_subtree_is_dropped_and_counted(): assert log.contract_violations == 1 +def test_a_signed_in_run_is_told_a_container_cannot_authenticate(): + """The third way the container sandbox cannot engage, and it used to be silent. + + A key is a string in the environment and crosses into a container with it. A + sign-in is not: the CLI reaches the endpoint through the host's session ingress, + which the container does not have. Inside one the CLI answers `Not logged in · + Please run /login`, and a run finds out one episode at a time -- measured, + `sessions=4 failed=4 edits=0`, a whole run spent on a condition that was knowable + before the first episode started. The sandbox says so up front now, the way it + already does for a missing engine and an unmountable toolchain. + """ + from examples.genesis._sandbox import SessionSandbox, signed_in_only + + assert not signed_in_only({"ANTHROPIC_API_KEY": "k", "CLAUDE_CODE_REMOTE": "true"}) + assert not signed_in_only({"ANTHROPIC_AUTH_TOKEN": "k", + "CLAUDE_CODE_PROVIDER_MANAGED_BY_HOST": "1"}) + assert signed_in_only({"CLAUDE_CODE_REMOTE": "true"}) + assert signed_in_only({"CLAUDE_CODE_PROVIDER_MANAGED_BY_HOST": "1"}) + assert not signed_in_only({}), "no markers at all is not a sign-in" + + # The reason is only reached when the engine and the toolchain are both fine, so + # the constructor is exercised for the shape of the check rather than its verdict + # on this machine: keyed, the sign-in branch must not be the one that fires. + with mock.patch.dict(os.environ, {"ANTHROPIC_API_KEY": "k"}): + assert "signed in" not in SessionSandbox().reason + + def test_a_signed_in_run_declares_its_toolset_instead_of_shipping_forty_two(): """`--allowedTools` auto-approves; what decides the schemas is what is *defined*. From 2db162dc40efb5fca5380db4ebe091863baeae98 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 16 Sep 2026 09:39:52 +0000 Subject: [PATCH 37/42] genesis: one proposal cap, because the tighter one always ruled A rollout descends the entire Context Tree -- the root delegates, every leaf writes, the parents fold, and one proposal carries all of it. Two bounds stood between that and the accepted version. SpatialContract.max_files_per_diff is the one raised to 64 for a session executor; RecursiveDelegation.max_edits sits upstream of it in _bound and was 4, a trust region sized for a single completion proposing a file or two. Only the tighter one ever applied, so raising the other changed nothing. Measured on an 8-node fly tree: the sessions wrote 153 implementation files, every sweep committed exactly 4, and after three rollouts the accepted state held nine records and six Python files. src/training wrote 26 of them, src/brain/circuits 24, src/arena 22, src/frontend 21 -- and the version grew by four a round. Both now read one named value. What _bound does within the bound is unchanged and was not a defect: a node-creating record is trimmed last, because nothing else in the accepted version says the node exists and the source file it would be dropped for is re-proposable next round; work next; routine upkeep first. A test pins that order, and another pins the two caps to one name so they cannot drift apart again. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- docs/algo-genesis.md | 30 ++++++++++ examples/genesis/genesis_recursive_worlds.py | 16 ++++- tests/test_genesis_example.py | 62 +++++++++++++++++++- 3 files changed, 102 insertions(+), 6 deletions(-) diff --git a/docs/algo-genesis.md b/docs/algo-genesis.md index 7b1660e3..bb6a8ba8 100644 --- a/docs/algo-genesis.md +++ b/docs/algo-genesis.md @@ -822,6 +822,36 @@ itself, which `--agents` cannot touch: eight sections, and small beside the sche mechanisms never both run — a key still takes the better path, and the declaration is what a sign-in gets instead. +### One rollout is the whole repository, and a cap of four was cutting it to four + +`RecursiveDelegation.propose` descends the **entire** Context Tree: the root delegates, +every leaf writes, the parents fold what came back, and one proposal carries all of it. +So a rollout is not "pick a node and implement it" — it is a complete draft of the +repository, judged and merged as one accepted event, which is what `w = (v, p)` means. + +Two bounds stood between that draft and the accepted version, and only the tighter one +ever applied. `SpatialContract.max_files_per_diff` is the one raised to 64 above; +`RecursiveDelegation.max_edits`, in `_bound`, sits **upstream** of it and was **4** — a +trust region sized for a single completion proposing a file or two. Measured on an +8-node `fly` tree, host sign-in, Haiku: + +| | | +|---|---:| +| implementation files the sessions wrote | **153** | +| files each sweep committed | **4** | +| Python files in the accepted state after three rollouts | **6** | +| `CONTEXT.md` records in it | 9 | + +`src/training` wrote 26 files, `src/brain/circuits` 24, `src/arena` 22, `src/frontend` +21, `src/backend` 17 — and the version grew by four a round. Raising the other cap had +changed nothing, because this one always cut first. + +They are one number now. What `_bound` does *within* the bound was not a defect and is +worth stating, since it looks like one: a node-creating `record` is trimmed **last** +(nothing else in the accepted version says the node exists, and the source file it +would be dropped for is re-proposable next round), `work` next, routine `context` +upkeep first. + ### The cap that discards the episode rather than the surplus A third thing the transcripts showed, found by counting rather than reading. The spatial diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index 5b7b8a97..d2c34282 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -675,9 +675,9 @@ def _session_kwargs(): # leaves three times that headroom, while still catching a loop that dumps a tree. # A single completion asked for whole files is a different thing -- it proposes # one or two, and six is already a runaway there -- so it keeps the tight number. + proposal_cap = 64 if args.executor == "claude-code" else 6 strategy = SpatialContract(initial_files=initial, frozen=spec.FROZEN, log=log, - max_files_per_diff=( - 64 if args.executor == "claude-code" else 6)) + max_files_per_diff=proposal_cap) # `manager.ex` states the parent's validation as three things: review the child's # results, run the tests, and reject anti-patterns it can see in the code. The # middle one is a number and was all this port had; the zero-field run is what @@ -748,7 +748,17 @@ def _refine(path, state): else spec.offline_manager), executor=(sessions if sessions is not None else spec.llm_executor(complete) if complete else spec.offline_executor), - log=log, max_depth=args.depth, max_edits=4, contracts=spec.CONTRACTS, + log=log, max_depth=args.depth, contracts=spec.CONTRACTS, + # One rollout is a walk of the WHOLE tree: every leaf writes, the parents + # fold, and what comes back is one proposal carrying all of it. Bounding + # that at 4 was a trust region sized for a single completion proposing a + # file or two, and it silently ruled over the strategy's own cap -- so the + # cap raised to 64 there never bound anything. Measured on an 8-node fly + # tree: the sessions wrote 153 implementation files, each sweep committed + # exactly 4, and the accepted state after three rollouts held nine records + # and six Python files. The two bounds are one number for that reason; a + # tighter one here is the only one that ever applies. + max_edits=proposal_cap, node_workers=max(1, args.node_workers), readonly=spec.FROZEN, # Three links, and the middle one is the only one that can ask "did you test diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index 0a65ae70..17ea1fba 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -505,6 +505,59 @@ def test_the_growth_phase_reports_progress_rather_than_going_silent(): assert field in printed, f"the progress line never reports {field}" +def test_the_proposal_cap_is_one_number_not_two(): + """A rollout walks the WHOLE tree, and two caps were bounding what came back. + + `SpatialContract.max_files_per_diff` is the one this port raised to 64 for a + session executor. `RecursiveDelegation.max_edits` sits upstream of it, in + `_bound`, and was 4 -- a trust region sized for a single completion proposing a + file or two, applied to a proposal carrying every leaf's work. The tighter one + is the only one that ever applies, so raising the other changed nothing. + + Measured on an 8-node `fly` tree: the sessions wrote 153 implementation files, + every sweep committed exactly 4, and after three rollouts the accepted state held + nine records and six Python files. + """ + tree = ast.parse(inspect.getsource(genesis.main)) + names = set() + for node in ast.walk(tree): + if not isinstance(node, ast.keyword): + continue + if node.arg in ("max_files_per_diff", "max_edits"): + assert isinstance(node.value, ast.Name), ( + f"{node.arg} is not the shared cap; a literal here can drift below " + "the other and silently rule") + names.add(node.value.id) + assert names, "neither cap is set from a named value" + assert len(names) == 1, f"the two caps read different names: {sorted(names)}" + + +def test_the_whole_tree_survives_a_rollout_when_the_cap_allows_it(): + """`_bound` keeps a node-creating record, then work, and sheds upkeep first.""" + policy = RecursiveDelegation(manager=lambda b: [], executor=lambda b: [], + log=WorldLog(), max_edits=4) + edits = [Edit("src/a", "src/a/impl.py", "x", kind="work"), + Edit("src/a", "src/a/test_impl.py", "x", kind="work"), + Edit("src/b", "src/b/impl.py", "x", kind="work"), + Edit("src/b", "src/b/test_impl.py", "x", kind="work"), + Edit("src/new", "src/new/CONTEXT.md", "#", kind="record"), + Edit("src/a", "src/a/CONTEXT.md", "#", kind="context")] + kept = {e.path: e.kind for e in policy._bound(edits)} + assert policy.truncated == 2 + # The record that brings a node into existence is the one thing a later round + # cannot re-propose, so it is trimmed last; routine upkeep goes first. + assert "src/new/CONTEXT.md" in kept + assert "src/a/CONTEXT.md" not in kept + assert sum(1 for k in kept.values() if k == "work") == 3 + + # Room for all of it, and all of it comes back -- the shape a whole-tree + # rollout needs, and what the run's own cap now allows. + roomy = RecursiveDelegation(manager=lambda b: [], executor=lambda b: [], + log=WorldLog(), max_edits=64) + assert len(roomy._bound(edits)) == len(edits) + assert roomy.truncated == 0 + + def test_a_diff_over_the_file_cap_loses_the_whole_episode_and_says_so(): """The cap discards the proposal, not the surplus -- so it has to be counted. @@ -539,11 +592,14 @@ def test_the_session_executor_gets_a_cap_no_real_episode_reaches(): """ caps = [] for node in ast.walk(ast.parse(inspect.getsource(genesis.main))): - if not isinstance(node, ast.keyword) or node.arg != "max_files_per_diff": + # The cap is one named value now, shared with `max_edits` -- see + # `test_the_proposal_cap_is_one_number_not_two` for why two drifted apart. + if not isinstance(node, ast.Assign) or not isinstance(node.value, ast.IfExp): + continue + if not any(isinstance(t, ast.Name) and "cap" in t.id for t in node.targets): continue - assert isinstance(node.value, ast.IfExp), ( - "the cap no longer depends on which executor is running") caps = [n.value for n in (node.value.body, node.value.orelse)] + assert caps, "the cap no longer depends on which executor is running" assert all(isinstance(c, int) for c in caps), caps # 24 is the measured ceiling of legitimate work; below it the cap eats episodes. assert max(caps) > 24 > min(caps), caps From 7df9774bb3e3d98d9c8e219dc7e3031cd87cba02 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 17 Sep 2026 08:42:34 +0000 Subject: [PATCH 38/42] genesis: pass the engine the same proposal cap the port enforces A proposal crosses five bounds between the session that writes a file and the state that keeps it. Two of them were this port's and were raised together. Two are the engine's -- `trust_region_ops` and `trust_region_chars` -- and nothing here had ever passed an `agg_config`, so the tightest cap on the path was a default of six ops, sized for a rule table and applied to a proposal carrying an eight-node repository. Measured: one rollout, thirteen sessions, 680 turns, forty minutes, merged `committed=0 rejected=1 [oversized=1]` with `discarded_diffs=0` and `truncated_edits=0` -- the port's own bounds passed it through cleanly and the engine rejected it behind them. Nine CONTEXT.md records landed and not one line of implementation. The largest single file any session wrote was 12,813 chars, so only the op count was ever binding. `engine_bounds(strategy)` derives the engine's two from the strategy that carries the cap, so there is one gate and it is the one that counts what it drops. `batch_trigger` and `max_wait_rounds` are carried across because `evolve()` builds `AggregatorConfig(batch_trigger=2, max_wait_rounds=1)` when passed nothing, not the dataclass defaults, and a config passed in replaces that object whole -- a run widening its trust region would otherwise have doubled its batch trigger in silence. Three tests, each verified to fail against the old behaviour: the caps all read one number, a diff at the full cap clears the engine's real trust-region predicate, and the batching numbers are read back out of a live `evolve()` rather than repeated as literals. `agentdescent/` is unchanged. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016CjFJhQ2Wxq9Vr3Fu38KrW --- docs/algo-genesis.md | 57 ++++++++++++- examples/genesis/genesis_recursive_worlds.py | 48 +++++++++++ tests/test_genesis_example.py | 85 ++++++++++++++++++-- 3 files changed, 182 insertions(+), 8 deletions(-) diff --git a/docs/algo-genesis.md b/docs/algo-genesis.md index bb6a8ba8..dfcd4219 100644 --- a/docs/algo-genesis.md +++ b/docs/algo-genesis.md @@ -846,7 +846,8 @@ trust region sized for a single completion proposing a file or two. Measured on 21, `src/backend` 17 — and the version grew by four a round. Raising the other cap had changed nothing, because this one always cut first. -They are one number now. What `_bound` does *within* the bound was not a defect and is +They are one number now — and so is a third one behind them, which took another run to +find; see *Five gates* below. What `_bound` does *within* the bound was not a defect and is worth stating, since it looks like one: a node-creating `record` is trimmed **last** (nothing else in the accepted version says the node exists, and the source file it would be dropped for is re-proposable next round), `work` next, routine `context` @@ -896,6 +897,60 @@ a node's build: episode. A single completion asked for whole files keeps the tight 6 — that executor proposes one or two files, and six there really is a runaway. +### Five gates, and the habit of raising one at a time + +Both fixes above were right and neither was enough, because both were found the same +way: follow the path a proposal takes until something rejects it, raise that, re-run. +Three runs in a row died that way. The last of them — one rollout, thirteen sessions, +680 turns, forty minutes, with the two caps above already raised — merged this: + +``` +sweep 0 reward=0.000 files=9 committed=0 rejected=1 rollouts=1 40m [oversized=1] +world : accepted=5 rejected=4 rework=4 depth=4 discarded_diffs=0 truncated_edits=0 +wrote 9 files (nine CONTEXT.md records, not one line of implementation) +``` + +`discarded_diffs=0` and `truncated_edits=0`: the port's own two bounds passed the +proposal through cleanly. It was rejected behind them, by a **third** cap, in the engine. + +A proposal crosses five bounds between the session that writes a file and the state that +keeps it. Listing them all was the fix; each of them had been the answer once: + +| | where | field | was | now | +|---|---|---|---:|---:| +| 1 | `RecursiveDelegation._bound` | `max_edits` | 4 | the cap | +| 2 | `SpatialContract.to_diff` | `max_files_per_diff` | 6 | the cap | +| 3 | `SpatialContract.to_diff` | `max_file_bytes` | 28 000 | unchanged | +| 4 | `Aggregator._apply_trust_region` | `trust_region_ops` | **6** | the cap | +| 5 | `Aggregator._apply_trust_region` | `trust_region_chars` | 32 000 | ≥ #3 | + +4 and 5 are the **engine's** defaults, and this port had never passed an `agg_config` +at all — so they were invisible from the example, sized for the rule tables the engine +was built on, and applied unchanged to a proposal carrying an eight-node repository. +Six ops. The measurement that settles it: across that rollout the largest single file +any session wrote was **12 813 chars**, well inside both character bounds, so only the +*op count* was ever binding. + +They are one number now, `engine_bounds(strategy)`, and the shape is deliberate: the +port's cap decides, because it is the one that counts what it drops +(`discarded_diffs`) and the one whose value is argued for from measurement. The engine's +two are placed where they cannot bind first — ops at the same cap, chars no tighter than +the per-file bound that already filtered every op, so a file over #3 is dropped on its +own instead of returning as a whole diff lost to #5. + +One trap on the way, worth writing down because it is silent in the other direction: +`evolve()` does not fall back to `AggregatorConfig()` when it is passed nothing. It +builds `AggregatorConfig(batch_trigger=2, max_wait_rounds=1)` — and a config passed in +replaces that object whole. A run that only wanted a wider trust region would have +doubled its batch trigger and tripled its wait on the way past. Both arms were measured +and both arrive at (2, 1); the port carries them across, and a test reads them back out +of a real `evolve()` rather than repeating the literals, so the day the engine picks +different ones the port is told rather than drifting. + +The habit is the finding, not the constant. Following one path to the first rejection +finds *a* cap; it cannot tell you it was the only one. Enumerating every gate on the +path costs one reading of the merge routine and would have saved three runs. + ### One rollout was an hour, and four workers spent it on the same tree A rollout here is `RecursiveDelegation.propose`: a walk of the whole Context Tree with diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index d2c34282..aabaeaaf 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -79,6 +79,7 @@ import posixpath from agentdescent import Policies, evolve +from agentdescent.aggregator import AggregatorConfig from agentdescent.evolution import EvolvingArtifact from agentdescent.agents import Usage from agentdescent.filetree import load_tree, match_any @@ -395,6 +396,47 @@ def build_parser() -> argparse.ArgumentParser: return parser +def engine_bounds(strategy: SpatialContract) -> AggregatorConfig: + """The engine's copy of this port's proposal bounds, so only one of them rules. + + A proposal passes five bounds between the session that wrote the files and the + state that keeps them, and raising one at a time is how three runs in a row were + thrown away. The whole chain, in order: + + 1. ``RecursiveDelegation._bound`` -> ``max_edits`` (the cap) + 2. ``SpatialContract.to_diff`` -> ``max_files_per_diff`` (the cap) + 3. ``SpatialContract.to_diff`` -> ``max_file_bytes`` (28 000/file) + 4. ``Aggregator._apply_trust_region`` -> ``trust_region_ops`` (engine, 6) + 5. ``Aggregator._apply_trust_region`` -> ``trust_region_chars`` (engine, 32 000) + + 1-3 are this port's and were raised together. 4 and 5 are the engine's defaults, + and nothing here ever passed an ``agg_config``, so a whole-tree proposal that + cleared the first three was rejected by the fourth -- measured: one rollout, + thirteen sessions, 680 turns, ``committed=0 rejected=1 [oversized=1]``, nine + records written and not one line of implementation. + + What is wanted is one gate, not five. The port's cap is the one that decides, + because it is the one that counts what it drops (:attr:`WorldLog.discarded_diffs`) + and the one whose number is argued for where it is set. So the engine's two are + placed where they cannot bind first: ops at the same cap, chars no tighter than + the per-file bound that already filtered every op. A file over ``max_file_bytes`` + is dropped on its own by 3; it must not come back as a whole diff lost to 5. + + ``batch_trigger`` and ``max_wait_rounds`` are carried across because ``evolve()`` + builds ``AggregatorConfig(batch_trigger=2, max_wait_rounds=1)`` when it is passed + nothing -- not the dataclass's own defaults -- and a config passed in replaces + that whole object. Measured in both arms, synchronous and barrier-free. A run + that only wanted a wider trust region would otherwise have doubled its batch + trigger on the way past. + """ + return AggregatorConfig( + batch_trigger=2, max_wait_rounds=1, + trust_region_ops=strategy.max_files_per_diff, + trust_region_chars=max(AggregatorConfig.trust_region_chars, + strategy.max_file_bytes), + ) + + def main(argv=None) -> None: args = build_parser().parse_args(argv) # Barrier-free by default, and nothing discarded for being late. Upstream has no @@ -678,6 +720,11 @@ def _session_kwargs(): proposal_cap = 64 if args.executor == "claude-code" else 6 strategy = SpatialContract(initial_files=initial, frozen=spec.FROZEN, log=log, max_files_per_diff=proposal_cap) + agg_config = engine_bounds(strategy) + print(f"Proposal : up to {proposal_cap} file(s) and " + f"{strategy.max_file_bytes} chars per file, per episode; the engine's " + f"trust region is set to match, so this is the only gate that can " + f"reject one for its size") # `manager.ex` states the parent's validation as three things: review the child's # results, run the tests, and reject anti-patterns it can see in the code. The # middle one is a number and was all this port had; the zero-field run is what @@ -828,6 +875,7 @@ def _progress(info) -> None: tasks, spec.reward, run=run, propose=_superseded, strategy=strategy, on_round=_progress, artifact_id="world", blast_radius=SKILL_BLAST_RADIUS, + agg_config=agg_config, # Say the budget outright rather than letting `rounds` be reinterpreted: # `--episodes` is a count of ROOT episodes in both arms, and under the # barrier-free default that is exactly what a worker rollout is. diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index 17ea1fba..630ea442 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -505,18 +505,21 @@ def test_the_growth_phase_reports_progress_rather_than_going_silent(): assert field in printed, f"the progress line never reports {field}" -def test_the_proposal_cap_is_one_number_not_two(): - """A rollout walks the WHOLE tree, and two caps were bounding what came back. +def test_every_cap_on_a_proposals_path_is_the_same_number(): + """A rollout walks the WHOLE tree, and five caps stood between it and the state. `SpatialContract.max_files_per_diff` is the one this port raised to 64 for a session executor. `RecursiveDelegation.max_edits` sits upstream of it, in `_bound`, and was 4 -- a trust region sized for a single completion proposing a - file or two, applied to a proposal carrying every leaf's work. The tighter one - is the only one that ever applies, so raising the other changed nothing. + file or two, applied to a proposal carrying every leaf's work. Behind both sits + the engine's own `trust_region_ops`, which defaults to 6 and which this port + never passed at all. The tightest one is the only one that ever applies, so + raising any single one of them changed nothing. - Measured on an 8-node `fly` tree: the sessions wrote 153 implementation files, - every sweep committed exactly 4, and after three rollouts the accepted state held - nine records and six Python files. + Measured on an 8-node `fly` tree: with the first two raised and the third still + at its default, one rollout ran thirteen sessions and 680 turns and merged + `committed=0 rejected=1 [oversized=1]` -- nine `CONTEXT.md` records and not one + line of implementation. """ tree = ast.parse(inspect.getsource(genesis.main)) names = set() @@ -531,6 +534,74 @@ def test_the_proposal_cap_is_one_number_not_two(): assert names, "neither cap is set from a named value" assert len(names) == 1, f"the two caps read different names: {sorted(names)}" + # ...and the engine's two are derived from the strategy that carries that cap, + # rather than left at defaults nobody passes. + cfg = genesis.engine_bounds(_strategy(max_files_per_diff=64)) + assert cfg.trust_region_ops == 64, ( + "the engine's op cap does not follow the port's -- a proposal the port " + "allowed is rejected `oversized` behind it") + assert cfg.trust_region_chars >= _strategy().max_file_bytes, ( + "a file the port lets through whole is over the engine's per-op cap, so " + "one oversized file loses the entire episode instead of just itself") + + +def test_a_diff_at_the_full_cap_clears_the_engines_trust_region(): + """The end-to-end version of the above: the port's cap is the only gate. + + This is the test fly26 did not have. It builds a proposal exactly at the cap, + puts it through the real `to_diff`, and asks the engine's real trust-region + predicate about the result -- which is the pair that disagreed. + """ + from agentdescent.aggregator import Aggregator + + cap = 12 + strategy = _strategy(max_files_per_diff=cap) + edits = [Edit("src", f"src/mod{n}.py", "x" * 4_000) for n in range(cap)] + diff = strategy.to_diff(strategy.initial(), render_edits(edits, "r"), "w0", 1, "world") + assert diff is not None and diff.size() == cap, "the port dropped it first" + + region = Aggregator._trust_region( + type("A", (), {"config": genesis.engine_bounds(strategy)})()) + assert diff.size() <= region.ops, ( + f"{diff.size()} ops against a region of {region.ops}: the engine rejects " + "a proposal the port was built to allow") + assert all(len(v) <= region.chars for v in diff.ops.values()) + + +def test_the_run_keeps_the_batching_evolve_would_have_chosen(): + """Passing `agg_config=` replaces the config `evolve()` builds, in full. + + That config is not `AggregatorConfig()` -- it is `AggregatorConfig(batch_trigger=2, + max_wait_rounds=1)`, built inside `evolve()` and unreachable from here. So a run + that passes its own config to widen the trust region silently doubles its batch + trigger and triples its wait unless it carries those two across. This reads the + numbers out of a real `evolve()` rather than repeating the literals, so the day + the engine picks different ones the port is told instead of drifting. + """ + import agentdescent.aggregator as aggregator + + seen = [] + original = aggregator.Aggregator.__init__ + + def spy(self, ledger, verifier, audit, config, *args, **kwargs): + seen.append(config) + return original(self, ledger, verifier, audit, config, *args, **kwargs) + + tasks = [Task(id=f"t{n}", prompt="p") for n in range(10)] + with mock.patch.object(aggregator.Aggregator, "__init__", spy): + evolve(tasks, lambda state, task: 0.0, run=lambda state, task: "a", + propose=lambda rendered, task, output, reward: None, + rounds=1, n_workers=1, seed=0) + assert seen, "no aggregator was built, so nothing was measured" + + chosen = seen[0] + ours = genesis.engine_bounds(_strategy()) + for field in ("batch_trigger", "max_wait_rounds"): + assert getattr(ours, field) == getattr(chosen, field), ( + f"{field} is {getattr(ours, field)} here and {getattr(chosen, field)} " + "in a run that passes no config -- passing one changed more than the " + "trust region") + def test_the_whole_tree_survives_a_rollout_when_the_cap_allows_it(): """`_bound` keeps a node-creating record, then work, and sheds upkeep first.""" From d4d8c6b3b4b83a279dcabee631dcdc7901e70ff9 Mon Sep 17 00:00:00 2001 From: Danyang Chen <15378933+Birfy@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:40:38 +0800 Subject: [PATCH 39/42] fix(genesis/sandbox): forward allowlisted credentials by name so no key lands on the docker exec argv Addresses item 4 of the 2026-10-13 review on #188: `Workspace.command` passed CONTAINER_ENV values as `--env NAME=VALUE`, which puts API keys on the argv where any process listing shows them. Allowlisted names are now passed as `--env NAME` and the engine client reads the value from the env `run_cli` runs it under (both callers already pass the same env). Fixed container paths (HOME, TMPDIR, CA bundle) stay by value. Adds a test that no secret value -- allowlisted or not -- appears in command(...) and that non-allowlisted names do not cross at all. --- examples/genesis/_sandbox.py | 19 +++++++++++++++---- tests/test_genesis_example.py | 32 +++++++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/examples/genesis/_sandbox.py b/examples/genesis/_sandbox.py index 92ca62b7..3ede6211 100644 --- a/examples/genesis/_sandbox.py +++ b/examples/genesis/_sandbox.py @@ -157,17 +157,28 @@ def __init__(self, path: str, *, prefix: Sequence[str] = (), self._names = {n for n in binary_names if n} def command(self, argv: Sequence[str], env: Mapping[str, str]) -> List[str]: - """The argv to run on *this* host for a session command.""" + """The argv to run on *this* host for a session command. + + ``env`` is the environment the argv will run under; the allowlisted + :data:`CONTAINER_ENV` names in it are forwarded into the container by name, + so the returned argv carries no credential values.""" argv = list(argv) if not self._prefix: return argv if self._binary and argv and argv[0] in self._names: argv[0] = self._binary # the mounted path, not the host's PATH passed: List[str] = [] + # By *name* only: `exec --env NAME` reads the value from the environment + # the engine client runs under, so a key never lands on the argv, where + # `ps` and every process listing on the host would show it. That makes + # `env` a contract: it must be the environment the returned argv is run + # with -- which both callers already pass as `run_cli(..., env=env)`. for name in CONTAINER_ENV: - value = env.get(name) - if value is not None: - passed += ["--env", f"{name}={value}"] + if env.get(name) is not None: + passed += ["--env", name] + # The workspace's own variables are fixed container paths (`HOME`, + # `TMPDIR`, the CA bundle), not credentials, and do not exist on the host + # side under those values -- so they are the one thing passed by value. for name, value in self._env.items(): passed += ["--env", f"{name}={value}"] return self._prefix[:-1] + passed + [self._prefix[-1]] + argv diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index 630ea442..bff6a574 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -2766,7 +2766,8 @@ def test_the_sandbox_passes_named_variables_and_rewrites_only_the_agent(): {"ANTHROPIC_API_KEY": "k", "CLAUDE_CODE_SESSION_ID": "host"}) assert got[-3:] == ["/opt/pkg/bin/claude", "-p", "hi"] assert got[:4] == ["docker", "exec", "-w", "/work"] - assert got[got.index("--env") + 1] == "ANTHROPIC_API_KEY=k" + # forwarded by name: `exec --env NAME` reads the value from the client's env + assert got[got.index("--env") + 1] == "ANTHROPIC_API_KEY" assert f"HOME={CONTAINER_HOME}" in got # the host's session identity is not on the list, so it cannot arrive by accident assert not any("CLAUDE_CODE_SESSION_ID" in part for part in got) @@ -2774,6 +2775,35 @@ def test_the_sandbox_passes_named_variables_and_rewrites_only_the_agent(): assert space.command(["sh", "-c", "ls /"], {})[-3:] == ["sh", "-c", "ls /"] +def test_no_secret_value_reaches_the_container_argv(): + """A key on the `docker exec` argv is visible to `ps` on the whole host. + + Allowlisted credentials cross by *name* (the engine client reads the value from + the environment `run_cli` runs it under); anything not on the allowlist does not + cross at all. Either way no secret value may appear in `command(...)`. + """ + from examples.genesis._sandbox import CONTAINER_ENV, CONTAINER_HOME, Workspace + + space = Workspace("/host/ws", prefix=["docker", "exec", "-w", "/work", "cid"], + env={"HOME": CONTAINER_HOME}, binary="/opt/pkg/bin/claude", + binary_names=("claude",)) + secrets = {"ANTHROPIC_API_KEY": "sk-ant-SECRET-1", + "ANTHROPIC_AUTH_TOKEN": "tok-SECRET-2", + "HTTPS_PROXY": "http://user:SECRET-3@proxy:8080", + # not on the allowlist: must not cross in any form + "OPENAI_API_KEY": "sk-SECRET-4", "GITHUB_TOKEN": "ghp_SECRET-5"} + got = space.command(["claude", "-p", "hi"], secrets) + joined = "\0".join(got) + for value in secrets.values(): + assert value not in joined + forwarded = [got[i + 1] for i, part in enumerate(got) if part == "--env"] + for name in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "HTTPS_PROXY"): + assert name in CONTAINER_ENV and name in forwarded + assert not any(n.startswith(("OPENAI_API_KEY", "GITHUB_TOKEN")) for n in forwarded) + # the fixed container paths are still passed by value + assert f"HOME={CONTAINER_HOME}" in forwarded + + def test_a_shared_toolchain_directory_is_not_mounted(): """`/usr/bin/claude` would mean mounting the host's system directories. From b129d3e4103ba720b433ff96cf7db9a57c44d262 Mon Sep 17 00:00:00 2001 From: Danyang Chen Date: Wed, 14 Oct 2026 08:08:58 +0800 Subject: [PATCH 40/42] test(_common): cover claude-cli provider choice and missing-binary error Address review item 3 on the Genesis draft: an existing port parser still builds with --provider claude-cli, and a missing CLI raises RuntimeError ("not on PATH") instead of an opaque FileNotFoundError. --- examples/_common.py | 7 +++++++ tests/test_example_entrypoints.py | 32 +++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/examples/_common.py b/examples/_common.py index 92642ed7..d462e875 100644 --- a/examples/_common.py +++ b/examples/_common.py @@ -513,6 +513,13 @@ def complete(prompt: str) -> str: try: done = subprocess.run(argv, input=prompt, capture_output=True, text=True, timeout=timeout, env=cli_env()) + except FileNotFoundError as exc: + if usage is not None: + usage.record(failed=True, seconds=time.time() - started) + raise RuntimeError( + f"{binary!r} is not on PATH; install the Claude Code CLI or " + f"pass --provider claude / openai instead" + ) from exc except subprocess.TimeoutExpired: if usage is not None: usage.record(failed=True, seconds=time.time() - started) diff --git a/tests/test_example_entrypoints.py b/tests/test_example_entrypoints.py index a146c6c9..6123998e 100644 --- a/tests/test_example_entrypoints.py +++ b/tests/test_example_entrypoints.py @@ -337,6 +337,38 @@ def factory_stub(**kwargs): assert seen == {factory: {"model": "test-model", "usage": None, "max_tokens": 99}} +def test_an_existing_port_parser_still_builds_with_claude_cli_choice(): + """``claude-cli`` is opt-in on the shared provider list; ports that never + asked for ``max_tokens_default`` must keep parsing, and ``--provider + claude-cli`` must be accepted rather than rejected as an unknown choice. + """ + from examples.skillopt import skillopt_skill_training as port + + parser = port.build_parser() + # Default path unchanged: still ``claude``, still no shared ``--timeout``. + defaults = parser.parse_args([]) + assert defaults.provider == "claude" + assert not hasattr(defaults, "timeout") + chosen = parser.parse_args(["--provider", "claude-cli"]) + assert chosen.provider == "claude-cli" + + +def test_completion_for_claude_cli_without_binary_raises_cleanly(monkeypatch): + """Missing CLI must not surface as an opaque ``FileNotFoundError``.""" + import subprocess as sp + + def boom(*_a, **_k): + raise FileNotFoundError(2, "No such file or directory", "claude") + + monkeypatch.setattr(sp, "run", boom) + args = argparse.Namespace(provider="claude-cli", model="haiku", + no_thinking=False, max_tokens=None, timeout=None, + _concurrency=None) + complete = common.completion_for(args) + with pytest.raises(RuntimeError, match="not on PATH"): + complete("ping") + + @pytest.mark.parametrize( "module", [port.module for port in PORTS] + [port_template], From 5222ecc9a318ca66070b7e4d74ec531d87f33b57 Mon Sep 17 00:00:00 2001 From: Birfy <15378933+Birfy@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:03:48 +0800 Subject: [PATCH 41/42] genesis: define 'test file' for own_review, enforce offline tests, fly deviations, changelog fragment - own_review/own_test_failures: explicit TEST_FILE_GLOBS (test_*.py, *_test.py) and NON_IMPLEMENTATION_GLOBS/GENERATED_DIRS; tests for sibling-node tests and __init__/generated-only returns (review item 5) - docs/algo-genesis.md: fly.py non-frozen and hidden-suite realignment vs upstream (item 6) - tests/test_genesis_example.py: autouse offline guard, slow/engine markers (item 7); signed-in-only secret test (item 4) - changelog.d/188-genesis-blind-mode.md --- changelog.d/188-genesis-blind-mode.md | 10 ++ docs/algo-genesis.md | 31 +++++ examples/genesis/_suite.py | 62 +++++++-- examples/genesis/genesis_recursive_worlds.py | 5 +- pyproject.toml | 4 + tests/test_genesis_example.py | 139 +++++++++++++++++++ 6 files changed, 236 insertions(+), 15 deletions(-) create mode 100644 changelog.d/188-genesis-blind-mode.md diff --git a/changelog.d/188-genesis-blind-mode.md b/changelog.d/188-genesis-blind-mode.md new file mode 100644 index 00000000..3d6b2476 --- /dev/null +++ b/changelog.d/188-genesis-blind-mode.md @@ -0,0 +1,10 @@ +Added +Genesis (`examples/genesis`): a blind mode in which the acceptance suite lives +outside the repository and the agents see the requirement and the assertion's +name rather than its source, `own_review` / `own_test_failures` so a child's own +tests are run and implementation code with no test under its node is sent back +(a test file is `test_*.py` or `*_test.py`; package markers, `conftest.py` and +generated modules owe none), the `fly` domain (only `REQUIREMENTS.md` is frozen), +a containerised session sandbox that forwards credentials by name rather than by +value, and the `claude-cli` provider with `--max-tokens` / `--timeout` in +`examples/_common.py`. `agentdescent/` itself is untouched (#188). diff --git a/docs/algo-genesis.md b/docs/algo-genesis.md index b37fbd0f..eeec5d8e 100644 --- a/docs/algo-genesis.md +++ b/docs/algo-genesis.md @@ -241,6 +241,7 @@ be: | `spec/CONTEXT.md` | the goal, in prose and formulas. Unavoidable: something has to say what to build. | | `tests/**` | the scoring. Upstream's equivalents are c-testsuite, LLVM and Csmith — also human-supplied, also frozen. | | `md.py` / `jqx.py` | a frozen entry point, so the result is a program rather than a package nobody can invoke. A choice, and a defensible one. | +| `fly.py` | **not supplied, not frozen.** The `fly` domain freezes only `REQUIREMENTS.md`; the entry point's calling convention is written into the requirement (its section 4) and the agents write `fly.py` themselves. See the recorded deviations. | | `CONTEXT.md` **per node, with routing tables** | …this one is not defensible. | Six of md's fifteen files are `CONTEXT.md` records that name the nodes and what @@ -1392,6 +1393,36 @@ so would a reader of this page without this paragraph. the upstream role in full: there is no fan-out to `subagent_investigator`, because fanning out read-only sub-agents is what a manager with read tools of its own is for, and that is the port's session work. +* **The `fly` domain does not freeze its entry point (`fly.py`), and that is a realignment + toward upstream, not a deviation from it.** `md` and `jqx` freeze a human-written driver + (`md.py`, `jqx.py`) so the result is a program rather than a package nobody can invoke; + the table above calls that "a choice, and a defensible one" — it is a choice of this + port's, not something upstream has: an upstream Mode B run is handed a one-sentence + objective, and nothing in the repository is frozen for it. `fly` therefore freezes only + `REQUIREMENTS.md` (`FROZEN = ("REQUIREMENTS.md",)`, also the only entry in `CONTRACTS`). + Whoever uses the simulator needs a calling convention, and for a fly simulator nothing + implies one the way "a C compiler" implies `cc -o foo foo.c`, so the convention is + stated as a *requirement* (section 4 of `REQUIREMENTS.md`: the four names `fly.py` + imports) and `fly.py` itself is written by the agents. The cost is real: a run can now + fail to produce an entry point at all, which a frozen driver made impossible, so + `fly` runs are not directly comparable to `md` / `jqx` numbers. +* **The `fly` acceptance suite is blind and lives outside the repository — a realignment + with how upstream is evaluated, and a deviation from this port's other domains.** + In `md`, `jqx`, `minilang` and `stackvm` the human writes the suite, it sits in the + frozen `tests/**`, and the failing test's *source* is pasted into the agent's prompt. In + upstream the agents write their own tests and run them ("tests are the definition of + done", `agents/manager.ex`); the external benchmarks (c-testsuite, LLVM, Csmith) are + applied by the experimenter afterwards and never enter the inner loop. `fly` follows + that: the `hidden` suite drives `evolve()` but is merged into a scratch copy only while + one evaluation runs, the prompt carries the requirement heading and the assertion's + *name* (no threshold, no inputs, no source), and a second `audit` set is sealed and + measured afterwards. The repository's own tests are the agents' and are enforced by + `own_review` (a child's failing own tests, or implementation code with no test under its + node, sends the work back — *test file* means a `.py` basename matching `test_*.py` or + `*_test.py`; see `TEST_FILE_GLOBS`). What this does **not** reproduce is upstream's human + merge, nor its benchmarks: the hidden suite is still this port's own, written by the + port's author, so the realignment is of *where the tests sit relative to the agents*, not + of what they measure. ## Run it diff --git a/examples/genesis/_suite.py b/examples/genesis/_suite.py index af401935..7aeb63b4 100644 --- a/examples/genesis/_suite.py +++ b/examples/genesis/_suite.py @@ -22,6 +22,7 @@ from __future__ import annotations +import fnmatch import json import os import shutil @@ -41,8 +42,39 @@ from ._spatial import SITUATED_EDIT_PROTOCOL, parse_situated_edits from ._world import CONTEXT_FILE, ROUTING_HEADING, SKILLS_DIR, normalise -__all__ = ["CRASHED", "PYTHON_MODULE_SKILL", "Suite", "TestSuite", "llm_executor", - "llm_manager", "reward", "reward_test", "run_cases", "run_test"] +__all__ = ["CRASHED", "PYTHON_MODULE_SKILL", "Suite", "TEST_FILE_GLOBS", "TestSuite", + "is_implementation_file", "is_test_file", "llm_executor", "llm_manager", + "reward", "reward_test", "run_cases", "run_test"] + +#: What counts as *a test file* in `own_review` / `own_test_failures`. Matched with +#: `fnmatch` against the file's **basename** only (the directory is the node's business, +#: not the name's), and only for ``.py`` files: the two spellings pytest itself collects +#: by default. A file that matches none of these is not a test, whatever it contains. +TEST_FILE_GLOBS: Tuple[str, ...] = ("test_*.py", "*_test.py") + +#: Python files that are code but are not *implementation* -- there is nothing in them to +#: test, so writing only these does not oblige a node to write a test. Package markers +#: and pytest's own `conftest.py` (basename globs), and machine-generated modules +#: (basename globs, plus anything under a `__pycache__` / `build` / `generated` directory). +NON_IMPLEMENTATION_GLOBS: Tuple[str, ...] = ("__init__.py", "conftest.py", + "*_pb2.py", "*_pb2_grpc.py") +GENERATED_DIRS: Tuple[str, ...] = ("__pycache__", "build", "generated") + + +def is_test_file(path: str) -> bool: + """Is ``path`` a test file? ``test_*.py`` or ``*_test.py``, by basename.""" + name = path.rsplit("/", 1)[-1] + return any(fnmatch.fnmatchcase(name, glob) for glob in TEST_FILE_GLOBS) + + +def is_implementation_file(path: str) -> bool: + """Python the agents wrote that is neither a test, a package marker, nor generated.""" + if not path.endswith(".py") or is_test_file(path): + return False + *directories, name = path.split("/") + if any(fnmatch.fnmatchcase(name, glob) for glob in NON_IMPLEMENTATION_GLOBS): + return False + return not any(part in GENERATED_DIRS for part in directories) #: One human-written skill, shared by the formation domains because the advice #: is about writing a Python module rather than about either language. Shipped so @@ -990,9 +1022,7 @@ def _requirement(self, stem: str, func: str) -> str: def _implementation_files(state, under: str = "") -> List[str]: """Python the agents wrote that is neither a test nor a package marker.""" return [path for path in state - if path.endswith(".py") - and not path.rsplit("/", 1)[-1].startswith("test_") - and "__init__" not in path + if is_implementation_file(path) and (not under or path == under or path.startswith(under + "/"))] def own_test_failures(self, state, under: str = "", @@ -1019,8 +1049,7 @@ def own_test_failures(self, state, under: str = "", such as a parent reviewing a child that legitimately returned no code yet. """ owned = {path: body for path, body in state.items() - if path.endswith(".py") - and path.rsplit("/", 1)[-1].startswith("test_") + if path.endswith(".py") and is_test_file(path) and (not under or path == under or path.startswith(under + "/"))} if not owned: if require_tests and self._implementation_files(state, under): @@ -1046,6 +1075,18 @@ def own_review(self, *, require_tests: bool = True): finished either: upstream every node is accountable for its own subtree and tests are the definition of done, so a node with untested code is a node whose parent has nothing to run. + + The terms are explicit, not "looks like a test": + + * a **test file** is a ``.py`` file whose basename matches one of + :data:`TEST_FILE_GLOBS` (``test_*.py`` or ``*_test.py``); + * an **implementation file** is any other ``.py`` file the child wrote, except + package markers / ``conftest.py`` / generated modules + (:data:`NON_IMPLEMENTATION_GLOBS`, :data:`GENERATED_DIRS`) -- a child that + returns only those owes no test; + * the test has to be **anywhere under the parent's node** (``parent.world.path``), + so a test a sibling node already wrote counts, and one outside the node does + not. Deleting a file never creates an obligation. """ def review(parent, returned): @@ -1061,11 +1102,8 @@ def review(parent, returned): return ("rework", "your own tests fail: " + "; ".join(failures[:3])) if require_tests: wrote = [e.path for e in returned - if e.content is not None and e.path.endswith(".py") - and not e.path.rsplit("/", 1)[-1].startswith("test_") - and "__init__" not in e.path] - tested = any(p.endswith(".py") - and p.rsplit("/", 1)[-1].startswith("test_") + if e.content is not None and is_implementation_file(e.path)] + tested = any(is_test_file(p) and (not path or p == path or p.startswith(path + "/")) for p in candidate) if wrote and not tested: diff --git a/examples/genesis/genesis_recursive_worlds.py b/examples/genesis/genesis_recursive_worlds.py index ce838ae8..0708770a 100644 --- a/examples/genesis/genesis_recursive_worlds.py +++ b/examples/genesis/genesis_recursive_worlds.py @@ -115,7 +115,7 @@ from ._session import session_home, set_session_limit from ._skills import SkillExtractor from ._suite import TEST_FAILURE -from ._suite import cold_start, preflight +from ._suite import cold_start, is_test_file, preflight from ._octopus import OctopusConflict, git_available from ._spatial import SpatialContract from ._worktree import Rollout, WorktreeLedger, git_worktrees_available @@ -967,8 +967,7 @@ def _progress(info) -> None: spec, "REQUIRES_MODEL", False) and not hasattr(spec, "reference_tree"): # A blind domain's repository suite is the agents' own, so there is no # denominator from `tasks` to report it against -- count what is there. - written = sum(1 for p in state if p.endswith(".py") - and p.rsplit("/", 1)[-1].startswith("test_")) + written = sum(1 for p in state if is_test_file(p)) print(f"their own suite : {written} test file(s), " + ("all passing" if not failures else f"{len(failures)} failing:\n " + "\n ".join(failures[:6]))) diff --git a/pyproject.toml b/pyproject.toml index bf1f9dba..3e603d77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,10 @@ agentdescent = ["integrations/*.md", "integrations/*.json"] [tool.pytest.ini_options] testpaths = ["tests"] addopts = "-q" +markers = [ + "slow: takes several seconds (deselect with -m 'not slow'); the default run still includes it", + "engine: needs a real container engine (docker/podman) and is skipped without one", +] # Put both the repo root and tests directory on sys.path. The root resolves # package imports; tests keeps legacy top-level test helpers importable after # making `tests` an explicit package for subprocess-based refs. diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index bff6a574..6dffdcff 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -35,6 +35,39 @@ from examples.genesis import _stackvm as stackvm +@pytest.fixture(autouse=True) +def _offline(request, monkeypatch): + """Everything in this module runs without a network, a container engine or `claude`. + + Enforced rather than assumed: a test that opens an internet socket, or spawns + `docker` / `podman` / `claude`, fails here instead of passing on a developer machine + that happens to have them. The one test marked ``engine`` talks to a real engine on + purpose and is skipped where there is none. + """ + import socket + + real_connect = socket.socket.connect + + def connect(self, address): + if self.family in (socket.AF_INET, socket.AF_INET6): + raise AssertionError(f"test_genesis_example must stay offline: {address!r}") + return real_connect(self, address) + + monkeypatch.setattr(socket.socket, "connect", connect) + if request.node.get_closest_marker("engine") is None: + real_init = subprocess.Popen.__init__ + + def guarded(self, args, *a, **kw): + argv = args if isinstance(args, (list, tuple)) else [args] + # a *bare* name is a lookup on PATH, i.e. the real binary; the tests' fakes + # are always run by path + if argv and str(argv[0]) in ("docker", "podman", "claude"): + raise AssertionError(f"test_genesis_example must not run {argv[0]!r}") + return real_init(self, args, *a, **kw) + + monkeypatch.setattr(subprocess.Popen, "__init__", guarded) + + def _no_engine() -> bool: """Is there a container engine answering? Asked once, at collection.""" from examples.genesis._sandbox import sandbox_engine @@ -1274,6 +1307,7 @@ def test_a_cold_started_world_writes_the_decomposition_it_was_not_given(): assert log.contract_violations == 0 +@pytest.mark.slow def test_the_md_suite_rejects_every_deliberately_wrong_implementation(): """A suite is only as good as what it refuses, and nothing else here checked that. @@ -1886,6 +1920,77 @@ def __init__(self, state, path): assert "your own tests fail" in verdict[1] +class _ReviewParent: + def __init__(self, state, path): + self.state, self.world = state, LocalWorld(version=0, path=path) + + +_PASSING = "def test_x():\n assert 1\n" + + +@pytest.mark.parametrize("path,expected", [ + ("src/a/test_kc.py", True), ("test_kc.py", True), ("src/a/kc_test.py", True), + ("src/a/kc.py", False), ("src/a/test_kc.txt", False), ("src/a/testkc.py", False), + ("src/test_dir/kc.py", False), ("src/a/latest.py", False), +]) +def test_a_test_file_is_defined_by_glob_on_the_basename(path, expected): + """`test_*.py` / `*_test.py`, `.py` only, and the *name* -- not the directory.""" + from examples.genesis._suite import is_test_file + assert is_test_file(path) is expected + + +@pytest.mark.parametrize("path,expected", [ + ("src/a/kc.py", True), ("src/a/my__init__helper.py", True), + ("src/a/__init__.py", False), ("src/a/conftest.py", False), + ("src/a/kc_pb2.py", False), ("src/a/generated/kc.py", False), + ("src/a/__pycache__/kc.py", False), ("src/a/test_kc.py", False), + ("src/a/notes.md", False), +]) +def test_an_implementation_file_excludes_markers_generated_and_tests(path, expected): + from examples.genesis._suite import is_implementation_file + assert is_implementation_file(path) is expected + + +def test_a_test_in_a_sibling_node_satisfies_the_rule_but_only_under_the_parent(): + """The rule is "a test anywhere under the parent's node", not "next to the file".""" + review = _blind_suite().own_review() + wrote = [Edit("src/a", "src/a/kc.py", "X = 1\n")] + # a sibling node already wrote a test, and it is under the parent's path + sibling = {CONTEXT_FILE: "# root\n", "src/a/b/test_other.py": _PASSING} + assert review(_ReviewParent(sibling, "src/a"), wrote) is None + # a test outside the parent's subtree does not count + outside = {CONTEXT_FILE: "# root\n", "src/z/test_other.py": _PASSING} + verdict = review(_ReviewParent(outside, "src/a"), wrote) + assert verdict is not None and verdict[0] == "rework" + # and a root-level parent (path "") sees every test in the repository + assert review(_ReviewParent(outside, ""), wrote) is None + + +def test_a_node_that_wrote_only_markers_or_generated_files_owes_no_test(): + review = _blind_suite().own_review() + state = {CONTEXT_FILE: "# root\n"} + for returned in ( + [Edit("src/a", "src/a/__init__.py", "")], + [Edit("src/a", "src/a/conftest.py", "import pytest\n")], + [Edit("src/a", "src/a/kc_pb2.py", "X = 1\n")], + [Edit("src/a", "src/a/generated/kc.py", "X = 1\n")], + [Edit("src/a", "src/a/README.md", "notes\n")], + [Edit("src/a", "src/a/old.py", None)], # a deletion creates no obligation + ): + assert review(_ReviewParent(state, "src/a"), returned) is None, returned + # one real implementation file next to the markers still does + mixed = [Edit("src/a", "src/a/__init__.py", ""), Edit("src/a", "src/a/kc.py", "X = 1\n")] + verdict = review(_ReviewParent(state, "src/a"), mixed) + assert verdict is not None and "src/a/kc.py" in verdict[1] + + +def test_a_star_test_py_file_counts_as_the_nodes_test(): + review = _blind_suite().own_review() + returned = [Edit("src/a", "src/a/kc.py", "X = 1\n"), + Edit("src/a", "src/a/kc_test.py", _PASSING)] + assert review(_ReviewParent({CONTEXT_FILE: "# root\n"}, "src/a"), returned) is None + + def test_the_run_reports_how_many_model_calls_were_in_flight_at_once(): """`usage.seconds / wallclock` looks like it answers this and does not. @@ -2461,6 +2566,7 @@ def test_a_session_that_dies_costs_its_episode_and_nothing_else(tmp_path): ''' +@pytest.mark.slow def test_a_session_that_runs_out_of_wall_keeps_what_it_wrote(tmp_path): """The wall is an interruption, not a verdict. @@ -2498,6 +2604,7 @@ def test_a_session_that_runs_out_of_wall_keeps_what_it_wrote(tmp_path): ''' +@pytest.mark.slow def test_a_session_does_not_outlive_the_episode_that_started_it(tmp_path): """The group is killed, not the process. @@ -2804,6 +2911,36 @@ def test_no_secret_value_reaches_the_container_argv(): assert f"HOME={CONTAINER_HOME}" in forwarded +def test_a_non_allowlisted_secret_never_appears_when_the_cli_is_signed_in_only(): + """Signed in rather than keyed: `ANTHROPIC_API_KEY` is not in play, and must stay out. + + `signed_in_only()` is the case where the host holds no key at all, so a key that + shows up anyway (a stray export, a `.env` loaded late) is exactly the one that must + not reach the argv, by value or by name, and neither may anything off the allowlist. + """ + from examples.genesis._sandbox import CONTAINER_HOME, Workspace, signed_in_only + + signed_in = {"CLAUDE_CODE_REMOTE": "true"} + assert signed_in_only(signed_in) + space = Workspace("/host/ws", prefix=["docker", "exec", "-w", "/work", "cid"], + env={"HOME": CONTAINER_HOME}, binary="/opt/pkg/bin/claude", + binary_names=("claude",)) + host = dict(signed_in, OPENAI_API_KEY="sk-SECRET-A", GITHUB_TOKEN="ghp_SECRET-B", + AWS_SECRET_ACCESS_KEY="SECRET-C", CLAUDE_CODE_SESSION_ID="host-sess") + got = space.command(["claude", "-p", "hi"], host) + joined = "\0".join(got) + for leaked in ("SECRET-A", "SECRET-B", "SECRET-C", "host-sess", "OPENAI_API_KEY", + "GITHUB_TOKEN", "AWS_SECRET_ACCESS_KEY", "CLAUDE_CODE_REMOTE", + "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN"): + assert leaked not in joined, leaked + # and a key that *is* present crosses by name only, never by value + keyed = dict(host, ANTHROPIC_API_KEY="sk-ant-SECRET-D") + assert not signed_in_only(keyed) + got = space.command(["claude", "-p", "hi"], keyed) + assert "sk-ant-SECRET-D" not in "\0".join(got) + assert got[got.index("--env") + 1] == "ANTHROPIC_API_KEY" + + def test_a_shared_toolchain_directory_is_not_mounted(): """`/usr/bin/claude` would mean mounting the host's system directories. @@ -2850,6 +2987,8 @@ def test_without_an_engine_the_run_says_so_rather_than_pretending(tmp_path): assert "can read this machine" in local.summary() +@pytest.mark.engine +@pytest.mark.slow @pytest.mark.skipif(_no_engine(), reason="needs a container engine") def test_a_sandboxed_session_cannot_read_the_machine_it_runs_on(tmp_path): """The claim, checked against a real engine rather than against the flags. From 958935b9e21ba311bdba4ed8acc318c1fd841aba Mon Sep 17 00:00:00 2001 From: Birfy <15378933+Birfy@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:17:15 +0800 Subject: [PATCH 42/42] test(genesis): keep the signed-in sandbox test off the real container engine CI has docker; SessionSandbox() probes it with `docker info`, which the offline guard rightly refused. Patch the engine/toolchain/provider and also cover the signed-in branch itself. --- tests/test_genesis_example.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/test_genesis_example.py b/tests/test_genesis_example.py index 6dffdcff..f36ce1ac 100644 --- a/tests/test_genesis_example.py +++ b/tests/test_genesis_example.py @@ -395,8 +395,22 @@ def test_a_signed_in_run_is_told_a_container_cannot_authenticate(): # The reason is only reached when the engine and the toolchain are both fine, so # the constructor is exercised for the shape of the check rather than its verdict # on this machine: keyed, the sign-in branch must not be the one that fires. - with mock.patch.dict(os.environ, {"ANTHROPIC_API_KEY": "k"}): - assert "signed in" not in SessionSandbox().reason + # No real engine, no real binary: the engine probe is `docker info` on a machine + # that has docker, and this test must not depend on whether it does. + from examples.genesis import _sandbox + + with mock.patch.object(_sandbox, "sandbox_engine", return_value="docker"), \ + mock.patch.object(_sandbox, "toolchain_root", return_value="/opt/claude"), \ + mock.patch.object(_sandbox, "_Provider") as provider: + provider.return_value.reap.return_value = 0 + with mock.patch.dict(os.environ, {"ANTHROPIC_API_KEY": "k"}): + assert "signed in" not in SessionSandbox().reason + keyless = {k: v for k, v in os.environ.items() + if k not in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN")} + keyless["CLAUDE_CODE_REMOTE"] = "true" + with mock.patch.dict(os.environ, keyless, clear=True): + sandbox = SessionSandbox() + assert "signed in" in sandbox.reason and not sandbox.available def test_a_signed_in_run_declares_its_toolset_instead_of_shipping_forty_two():