From e17b8359ca87c039f3b3d653018507270bd1af10 Mon Sep 17 00:00:00 2001 From: bitdeep <328399512+bitdeep@users.noreply.github.com> Date: Sat, 12 Sep 2026 19:46:06 +0000 Subject: [PATCH] feat(neural): connect MN9 spikes to the flybody rostrum Run the full FlyWire sugar-response graph and one native body step on a shared 0.1 ms clock. Map the published MN9 pair through an explicit firing-rate adapter to the allowlisted rostrum servo, with native wake handling and no pose writes. Add baseline and motor-block controls, pause/cancel handling, head focus and measured neural-to-body traces. Keep the existing resource budgets and document the biological limits, causal validation and browser evidence in English. --- README.md | 37 +++-- docs/browser-environment.md | 35 +++-- docs/motor-link.md | 138 ++++++++++++++++++ docs/neural-reference.md | 20 ++- package.json | 2 +- scripts/dev_environment.py | 2 +- scripts/motor_bridge.py | 235 +++++++++++++++++++++++++++++++ scripts/serve_environment.py | 86 ++++++++--- scripts/test_environment.py | 7 +- scripts/validate_motor_bridge.py | 153 ++++++++++++++++++++ web/environment.js | 23 ++- web/fly.js | 1 + web/index.html | 48 ++++++- web/motor.js | 100 +++++++++++++ web/neural.css | 3 + web/neural.js | 27 ++-- 16 files changed, 841 insertions(+), 76 deletions(-) create mode 100644 docs/motor-link.md create mode 100644 scripts/motor_bridge.py create mode 100644 scripts/validate_motor_bridge.py create mode 100644 web/motor.js diff --git a/README.md b/README.md index fc2381e..d4f8912 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,10 @@ Explore the **NETSPHERE** in a local browser, inspect one anatomical flybody animal, and stimulate a **127,400-neuron FlyWire reference model** while watching measured -neural activity. The browser currently keeps neural signals and body actuators -disconnected. [Open the development environment](#interactive-environment-in-the-browser) -or read the [neural model and its limits](docs/neural-reference.md). +neural activity. An experimental **MN9-to-proboscis link** now drives one body +actuator from simulated motor-neuron spikes, with explicit baseline and blocked-link +controls. [Open the development environment](#interactive-environment-in-the-browser) +or read the [motor link and its limits](docs/motor-link.md). The animal is the anatomically detailed [flybody](https://github.com/TuragaLab/flybody) model of *Drosophila melanogaster* (Google DeepMind and HHMI Janelia, *Nature* 2025). In the separate recorded-flight pipeline, its pretrained controller runs on CUDA, MuJoCo Warp integrates the body and wing aerodynamics at 20 kHz, and a geometric navigator steers the fly through the collidable interior. Every frame of those recordings comes from the integrated physical state. @@ -24,8 +25,9 @@ The full 60-second take, its metrics and the validation files are attached to th - It **is** whole-body physics: joints, wings with ellipsoid fluid forces, and the official DMPO flight policy (wingbeat pattern generator plus a residual MLP) driving the actuators. - It **is** a real 3D world: walls, pillars, ducts, cables and walkways with collision in the same MuJoCo model that integrates the fly. The camera moves through that space. - The **recorded flight** uses the official MLP policy. The browser development lab - also runs stimulus-response trials on the published FlyWire 630 connectome; - those neural signals are **not yet connected to the body's muscles**. + runs the published FlyWire 630 connectome and an experimental sugar-response + motor link. Its firing-rate-to-servo adapter is engineered; world sensing, + neural walking and neural flight remain unimplemented. - The navigator is **not** learned vision. It reads the known world geometry and the measured position at 100 Hz and picks turns and climbs with clearance for wings and body. It only changes the reference command; it never writes the animal's pose or velocity. ## Results @@ -78,20 +80,23 @@ verify_take.py independent checks: duration, decoded frames, clearance, ha ### Interactive environment in the browser -The local browser observatory contains **one passive flybody animal**, selectable +The local browser observatory contains **one physical flybody animal**, selectable with a close-up orbit camera, plus three city viewpoints and a gravity/contact -experiment. Its **neural controller is disconnected and actuator drive disabled**: -it settles physically, with no autonomous walking or flight policy. -The **Neural activity** panel runs a separate FlyWire 630 reference assay with -127,400 neurons and all 14,687,178 stored directed connections. It shows calculated -spikes, selected real connections and the workflow up to the disconnected muscles. -Stimulate antennal neurons, compare a baseline or block sensory output. +experiment. It settles passively between on-demand trials, with no autonomous +walking or flight policy. Open **Neural activity → Stimulate sugar neurons** to +run the entire FlyWire 630 graph (127,400 neurons, 14,687,178 stored connections) +and drive the rostrum from its two MN9 motor neurons. **View proboscis** focuses +the observer on the head. Neural and physical time advance together in this +500 ms experiment. Compare no input or a blocked motor link; the UI retains +measured spikes, drive and joint motion after completion. +The expandable antennal reference assay retains its separate clock and readouts. Three.js renders measured body poses on demand; native MuJoCo sleep reduces idle work. The backend is capped at 2 CPUs and 1 GiB. The browser requests high-performance GPU graphics and targets 30 FPS while moving, with bounded resolution; backend physics and neural computation remain on the CPU. [Anatomy preparation, controls and limitations](docs/browser-environment.md) · -[Neural preparation, numerical validation and limitations](docs/neural-reference.md). +[Neural preparation and numerical validation](docs/neural-reference.md) · +[Motor protocol, engineered adapter and causal checks](docs/motor-link.md). ```bash pnpm-docker install --frozen-lockfile # provisioned Socket-protected Docker launcher @@ -173,12 +178,14 @@ scripts/ verify_take.py independent acceptance checks on states, geometry and video verify_avoidance.py compare baseline, shifted-obstacle and avoidance-off runs city_world.py procedural collidable megastructure - serve_environment.py local browser service, one passive fly and live gravity test + serve_environment.py local browser service, one physical fly and live gravity test dev_environment.py container-owned backend watcher; one dev stack on port 8089 neural_reference.py incremental LIF dynamics over the full FlyWire 630 graph neural_lab.py bounded on-demand trials and measured activity telemetry + motor_bridge.py shared-clock sugar → FlyWire → MN9 → native rostrum servo + validate_motor_bridge.py causal motor controls, actuator isolation and clock checks fetch_neural_reference.py, prepare_neural_reference.py, validate_neural_reference.py - passive_fly.py cached anatomy attachment, disabled actuator drive + passive_fly.py cached anatomy attachment and passive initialization prepare_browser_fly.py bounded anatomy preparation, full-resolution mass properties test_environment.py physical contact, sleep/wake, state isolation and telemetry checks city_navigation.py receding-horizon geometric navigator diff --git a/docs/browser-environment.md b/docs/browser-environment.md index 4f71f6f..485f305 100644 --- a/docs/browser-environment.md +++ b/docs/browser-environment.md @@ -8,10 +8,16 @@ vertically, **Shift** to move faster and the mouse wheel to move forward/backwar The three city view buttons return to known observation points. Touch screens have movement buttons and support dragging to look. -The animal is a **passive physical body with its neural controller disconnected**. -All actuator forces are disabled; anatomical springs, joints, gravity and contact -remain active. It settles under physics; it does not walk or fly autonomously. -The **Neural activity** panel opens a separate connectome reference assay: +The animal settles passively and has **one experimental neural motor link**. +Open **Neural activity → Stimulate sugar neurons** to route calculated MN9 spikes +through an engineered adapter to the native rostrum servo. **View proboscis** +focuses the observer on the head. Compare **Baseline · no input** and **Block motor +link**; **Stop** removes actuator authority. New trials wait for physical rest, +and pause/resume affects both clocks. All other actuators remain disabled. +Anatomical springs, joints, gravity and contact stay active. It does not walk or +fly autonomously. [Motor protocol and causal validation](motor-link.md). + +The expandable **Antennal reference assay** retains a separate connectome assay: antennal stimulus → FlyWire LIF dynamics → neural readouts → disconnected muscles. Its structure view shows a selection of actual connections with schematic positions; the entire published graph is retained in the simulation. Counts and voltages come @@ -94,7 +100,7 @@ activating the `dev` profile cannot start a flight recording in the background. 103,807 triangles (source: 272,550), with maximum original-vertex displacement 15.2 µm, and a 2.56 MiB binary packet before HTTP compression. The 67 physical segments have a total mass of approximately 0.985 mg. -- Native MuJoCo sleep is allowed for this disconnected stage, at the default +- Native MuJoCo sleep is allowed between motor trials, at the default 0.001 cm/s tolerance. The attachment frame's rotational sleep length is corrected from its distance to the world origin to a conservative anatomical collision radius (0.286 cm). This changes sleep eligibility, not forces or integration. @@ -105,7 +111,9 @@ activating the `dev` profile cannot start a flight recording in the background. This accuracy/idle-cost tradeoff is specific to the passive viewer, not a connectome or controlled-locomotion validation. See [MuJoCo sleeping semantics](https://mujoco.readthedocs.io/en/stable/programming/simulation.html#sleeping-islands). - Connecting actuators later requires explicit wake handling. + The motor bridge uses MuJoCo's documented negative-zero applied-force wake + signal, adding no nonzero force. The fly tree cannot sleep while driven; + completing or stopping the trial restores passive sleep eligibility. - One backend simulation thread advances independently of browser tabs. Server events target changed body poses at up to 30 Hz and idle status at 1 Hz. Unchanged poses and trails are omitted per viewer; reconnect starts with a full @@ -115,6 +123,8 @@ activating the `dev` profile cannot start a flight recording in the background. 6 ms of thread CPU, checked between native stepping blocks, so costly body settling cannot hold up every state read for a full 250-step batch. Only executed steps are subtracted from the bounded accumulator. + A coupled motor trial advances one neural and one physical step together, + inside that same physics-thread budget. It adds no second simulation worker. While settling, physical time may advance slower than wall time; the footer reports the measured ratio. Resting bodies resume near 1×. Pause and 0.25× speed affect the @@ -152,13 +162,15 @@ activating the `dev` profile cannot start a flight recording in the background. The server only serves an allowlist of frontend assets and read endpoints: `/api/world`, `/api/state`, `/api/events`, `/health`, plus `/api/dev-version` in -development mode. `POST /api/command` accepts bounded drop, pause, speed and -`neural_trial` commands with same-origin JSON. Neural trial mode must be -`stimulus`, `baseline` or `blocked`; one trial runs at a time. It cannot read +development mode. `POST /api/command` accepts bounded drop, pause, speed, +`neural_trial`, `motor_trial` and `motor_stop` commands with same-origin JSON. +Trial mode must be `stimulus`, `baseline` or `blocked`; only one neural or motor +trial runs at a time. Motor commands cannot select arbitrary neurons, actuator +names, gains, torques or durations. It cannot read arbitrary project files or edit the scene. At most eight event streams are open at once. The container has a read-only filesystem and limits of 2 CPUs, 1 GiB RAM and 64 PIDs. These limits cover the backend; the browser is a separate process. -The neural worker has an additional duty budget of 0.15 core, runs only on request +The separate antennal worker has a duty budget of 0.15 core, runs only on request and publishes changes at up to 10 Hz. It adds no continuous scene render loop. This local server is not intended to be exposed directly to the internet. @@ -178,6 +190,9 @@ They check the unchanged anatomical contracts, disabled actuation, fly contact, native sleep and force-triggered wake, and compare sleep with 200 ms of awake dynamics (segment drift must remain below the measured visual reduction displacement, currently 15.2 µm). Delta telemetry is also checked. +The [motor validation](motor-link.md#validation) separately tests the full graph +against no-input and blocked-link controls, exact clock alignment, sleeping-body +wake, the actuator allowlist, cancellation and failure handling. Browser checks cover visible rendering, view changes, mouse/keyboard motion, pause/resume, slow motion, the gravity experiment, reconnect and mobile layout. The measured passive-viewer sample below predates the neural panel. The later diff --git a/docs/motor-link.md b/docs/motor-link.md new file mode 100644 index 0000000..0ba41d7 --- /dev/null +++ b/docs/motor-link.md @@ -0,0 +1,138 @@ +# Experimental MN9-to-proboscis link + +The browser can now turn **simulated FlyWire motor-neuron spikes into a physical +flybody joint response**. This first link uses sugar-responsive sensory neurons +and the MN9 motor pair. It drives only the rostrum, part of the proboscis. + +Open **http://localhost:8089 → Neural activity → Stimulate sugar neurons**. +**View proboscis** focuses the observer on the head; drag to choose an angle. +The panel retains measured MN9 spikes, adapter drive and joint angle after the +movement. Compare **Baseline · no input** and **Block motor link**. +New trials wait for the body to settle naturally. **Stop** cancels the trial; +the footer pauses or slows both physical and neural time together. + +```mermaid +flowchart LR + S["21 sugar GRNs
Direct input"] --> C["Full FlyWire 630 graph
127,400 LIF neurons"] + C --> M["MN9 left and right
Calculated spikes"] + M --> A["Engineered adapter
Filtered rate → servo target"] + A --> B["Native flybody rostrum
MuJoCo joint dynamics"] +``` + +This is an on-demand motor experiment. Input is delivered directly to identified +neurons; no sugar object, taste receptor mechanics, world sensing or physical +feedback into the brain is implemented. There is no autonomous walking/flight +controller or complete ventral nerve cord. The antennal assay remains separately +available and has no motor authority. + +## Biological source and engineering boundary + +The pinned [Shiu/Spiller notebook](https://github.com/philshiu/Drosophila_brain_model/blob/91bdd1e7dcf193f3e7ca5a8933497fcef63b7960/figures.ipynb) +identifies 21 right labellar sugar GRNs (`neu_sugar`) and two MN9 neurons +(`ids_mn9`). The runtime verifies that notebook's SHA-256 against the graph +manifest and extracts literal IDs without executing cells. The motor IDs are +`720575940660219265` (left) and `720575940645521262` (right). +The [neural reference guide](neural-reference.md) describes the entire retained +graph, LIF equations, numerical agreement and source licenses. + +MN9 innervates the rostrum protractor muscle, providing a documented motor +association for this first link. Proboscis movement involves additional muscles +and joints; connecting one servo does not reconstruct that complete system. +See [McKellar et al., *Controlling motor neurons of every muscle for fly proboscis reaching*, eLife 2020](https://elifesciences.org/articles/54978). + +The **adapter is our engineering approximation**, not a measured neuromuscular +transfer function or a learned policy. The bilateral MN9 pair is averaged into +one existing flybody rostrum hinge. Each spike contributes to a 30 ms exponential +rate estimate; a mean 100 Hz maps to full drive: + +```text +rate[t] = rate[t-1] × exp(-0.1 / 30) + MN9_spikes[t] × 1000 / (2 × 30) +drive[t] = clamp(rate[t] / 100, 0, 1) +rostrum_target[t] = 0.183 + drive[t] × (-1.24 - 0.183) radians +``` + +The existing position servo, gain, force limit, joint limits, springs, inertia, +contacts and friction calculate the response. No runtime pose or velocity writes +imitate movement. Actuation stays entirely off before the first MN9 spike. +The waveform and gain are not calibrated against an animal's measured kinematics. + +## Protocol and isolation + +Each trial resets neural state and uses seed `20260912`, while preserving the +body's integrated state and global physical clock. It lasts **500 ms**: 300 ms +of 200 Hz input per sugar neuron, then 200 ms without input. The published IDs +are reused; this specific finite-duration protocol is a local assay, not a claim +to reproduce every published feeding experiment. + +| Trial | Neural input | Motor link | +|---|---|---| +| Sugar response | Fixed sugar events | MN9 drives the rostrum adapter | +| Baseline | None | No spikes and no actuator drive | +| Block motor link | Identical sugar events | Normal brain activity; actuation disabled | + +One neural step precedes one native physical step, both **0.1 ms**, on the same +worker. A mismatch stops the trial. Pause freezes both clocks; paused time does +not consume the 90-second active wall-time allowance. Other guards are 10 CPU +seconds and 100,000 spikes, checked every 25 steps. Stop, completion and exceptions +disable actuation and release neural dynamic state. + +Actuator groups allow only `fly_001/rostrum`; every other actuator is disabled. +A sleeping body is woken using MuJoCo's documented **negative-zero applied-force +signal**, which adds no nonzero external force. Sleep is disallowed while the +servo is driven and restored afterwards. See the [MuJoCo sleeping documentation](https://mujoco.readthedocs.io/en/stable/programming/simulation.html#sleeping-islands). +The body then settles under its passive dynamics. + +The graph is shared read-only with the antennal lab. Trials are mutually exclusive, +use the existing 0.9-core physics duty budget and keep the dev ceiling of +2 CPUs / 1 GiB. Telemetry is bounded to 100 measured samples, emitted only when +changed; the browser adds no continuous idle animation. No new dependencies, +GPU compute allocation or persistent container is required. + +## Validation + +Run the causal checks in the existing disposable tooling image: + +```bash +LOCAL_UID="$(id -u)" LOCAL_GID="$(id -g)" \ + docker compose --profile neural-tools run --rm --no-deps \ + -e FLY_NEURAL=1 neural-tools \ + python scripts/validate_motor_bridge.py --output /out/motor-validation.json +``` + +The comparison clones the same settled **test** state for each intervention using +MuJoCo's complete data copy. The live service never resets the fly for a trial. +Checks cover unchanged state at start and before the first motor spike, shared +clocks, native wake, excluded actuators, concurrent trial rejection, cancellation +while paused, delta telemetry and failure on clock mismatch. + +Measured on 2026-09-12: + +| Trial | MN9 left / right spikes | Downstream spikes | Peak rostrum movement | Other actuator force | +|---|---:|---:|---:|---:| +| No input | 0 / 0 | 0 | 0° | 0 | +| Motor link blocked | 30 / 17 | 3,839 | 0° | 0 | +| Sugar response | 30 / 17 | 3,839 | 40.03° | 0 | + +The first motor force occurs at the first MN9 spike, **25.5 ms of simulated +time**, not wall time. Both stimulated trials have 5,076 total spikes, including +1,237 input spikes. Neural and physical clocks each advance 500 ms; all checks +complete without physics warnings. These establish a causal software-to-actuator +path, not biological equivalence, real-time performance or autonomous behavior. + +The ignored `out/neural-reference/motor-validation.json` records code/cache +hashes, protocol, counts, force, clock alignment, CPU/wall time and peak RSS. +The isolated validation process peaked near 215 MiB RSS; the stimulated 500 ms +run used about 2.3 CPU seconds. After the browser trials, a short resting-service +sample showed 149.8 MiB and 8.43% of one CPU core. These are observations of +specific workloads, not performance guarantees. + +Browser checks exercised all three modes, head focus, pause/resume with both +clocks frozen, cancellation with drive off, and the retained antennal assay +(2,906 total / 706 downstream spikes). The baseline and motor-blocked trials +added no new 3D frames to the settled scene. Desktop and 390×844 mobile panels +were inspected with no horizontal overflow. All 295 captured requests returned +HTTP 200, with no JavaScript exceptions. Resizing the software-rendered viewport +caused a graphics-context reset that recovered. This does not establish hardware +FPS. The isolated browser was closed after validation. + +The [browser guide](browser-environment.md) covers rendering limits and operation. diff --git a/docs/neural-reference.md b/docs/neural-reference.md index 2b7a63a..10b4993 100644 --- a/docs/neural-reference.md +++ b/docs/neural-reference.md @@ -7,14 +7,17 @@ counts preserved. It uses the data and model parameters from [Shiu and Spiller's reference repository at commit 91bdd1e](https://github.com/philshiu/Drosophila_brain_model/tree/91bdd1e7dcf193f3e7ca5a8933497fcef63b7960). This is not the newer MaleCNS dataset. Specimens and releases remain distinct. -The body remains passive. There is no motor mapping, walking pilot or flight -policy in the browser. The neural assay has its own clock and starts from rest -for each trial; it is not a continuously embodied animal. World-to-sensory mapping, -ventral nerve cord/muscle coupling and their validation remain future work. +The antennal assay described here has its own clock, starts from neural rest +and does not drive the body. A separate [sugar-response motor experiment](motor-link.md) +now couples the same full graph's MN9 neurons to the flybody rostrum through an +explicit engineered adapter, with a shared neural/physical clock. +World-to-sensory mapping, broader muscle control, walking and flight remain +future work; this is not a continuously embodied animal. ## Protocol and visible signals -Open **Neural activity** in the Fly 001 card. Each trial lasts 150 ms of neural +Open **Neural activity → Antennal reference assay** in the Fly 001 card. +Each antennal trial lasts 150 ms of neural time: 100 ms of input followed by 50 ms of recovery. The 146 published JON antennal neurons come from the CE, F and D groups in the reference notebook. Input is delivered directly to those neurons, not through simulated antenna @@ -40,7 +43,8 @@ on each connection. The neural panel updates from the existing event stream without invalidating the 3D scene. It redraws its diagram/timeline only after a changed sample, while open -and visible. Muscle disconnection stays explicit in the workflow. +and visible. Muscle disconnection stays explicit in the antennal workflow; +the sugar motor link has its own labeled workflow and measured body trace. ## Model and numerical comparison @@ -111,7 +115,9 @@ The viewer and neural worker share **one** development container, limited to 2 CPUs, 1 GiB and 64 PIDs, with no GPU compute allocation. Browser rendering separately requests its high-performance GPU. The neural worker has a 0.15-core duty budget in addition to the physics worker's 0.9-core budget. -One on-demand trial can run at a time; limits are 90 seconds elapsed, 10 CPU +Only one antennal or motor trial can run at a time. The motor experiment runs +inside the existing physics worker's budget, sharing the immutable graph. +Antennal limits are 90 seconds elapsed, 10 CPU seconds and 100,000 spikes, checked every 25 steps. A limit stops the trial explicitly with incomplete-result status instead of silently truncating it. Small block overshoot is possible; Docker provides the outer CPU/memory limits. diff --git a/package.json b/package.json index 3818094..a6057f8 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "type": "module", "packageManager": "pnpm@11.24.0", "scripts": { - "check": "node --check web/environment.js && node --check web/fly.js && node --check web/neural.js && node --check web/dev-reload.js" + "check": "node --check web/environment.js && node --check web/fly.js && node --check web/neural.js && node --check web/motor.js && node --check web/dev-reload.js" }, "dependencies": { "three": "0.180.0" diff --git a/scripts/dev_environment.py b/scripts/dev_environment.py index ff3363b..9f054d8 100644 --- a/scripts/dev_environment.py +++ b/scripts/dev_environment.py @@ -10,7 +10,7 @@ WATCHED = [ "serve_environment.py", "passive_fly.py", "city_world.py", "prepare_browser_fly.py", "fetch_neural_reference.py", - "neural_reference.py", "neural_lab.py", + "neural_reference.py", "neural_lab.py", "motor_bridge.py", ] diff --git a/scripts/motor_bridge.py b/scripts/motor_bridge.py new file mode 100644 index 0000000..0933ba1 --- /dev/null +++ b/scripts/motor_bridge.py @@ -0,0 +1,235 @@ +"""One explicit engineered MN9-to-rostrum link over the verified FlyWire graph.""" +import ast +import json +from pathlib import Path +import time + +import mujoco +import numpy as np + +from fetch_neural_reference import sha256 +from neural_reference import LIF, PARAMETERS, poisson_events +from passive_fly import FLY_ID + +TICKS = 5000 +STIMULUS_TICKS = 3000 +RATE_HZ = 200 +SEED = 20260912 +FILTER_MS = 30. +FULL_DRIVE_HZ = 100. +WALL_LIMIT = 90. +CPU_LIMIT = 10. +SPIKE_LIMIT = 100000 +MODES = ("stimulus", "baseline", "blocked") +SOURCE = Path(__file__).resolve().parents[1]/"data/neural-reference/figures.ipynb" + + +def feeding_identifiers(graph, source=SOURCE): + """Read literal published IDs; never execute notebook cells.""" + expected = graph.manifest["source_files"]["figures.ipynb"]["sha256"] + if sha256(source) != expected: + raise ValueError("Feeding protocol source checksum mismatch") + found = {} + for cell in json.loads(source.read_text())["cells"]: + if cell["cell_type"] != "code": + continue + tree = ast.parse("".join(cell["source"])) + for node in tree.body: + if not isinstance(node, ast.Assign): + continue + for target in node.targets: + if isinstance(target, ast.Name) and target.id in ("neu_sugar", "ids_mn9"): + values = ast.literal_eval(node.value) + if (not isinstance(values, list) or not all(type(x) is int for x in values) + or target.id in found and found[target.id] != values): + raise ValueError("Ambiguous feeding neuron identifiers") + found[target.id] = values + if len(found.get("neu_sugar", [])) != 21 or len(found.get("ids_mn9", [])) != 2: + raise ValueError("Published feeding neuron groups are missing") + # IDs exceed JavaScript's safe integer range. Expose their strings, not numbers. + lookup = {int(value): i for i, value in enumerate(graph.ids)} + return {key: [{"id": str(value), "index": lookup[value]} for value in values] + for key, values in found.items()} + + +class MotorBridge: + """Called only under the environment lock, on its single physics thread. + + No pose writes. One neural step and one native physical step share 0.1 ms. + All actuators except the allowlisted rostrum servo remain disabled by group. + """ + def __init__(self, graph, model, data, fly_tree): + self.graph, self.model, self.data = graph, model, data + self.fly_tree = fly_tree + self.neurons = None + self.events = None + self.state = {"revision": 0, "status": "unavailable", "drive_enabled": False} + self.descriptor = {"available": False} + self.actuator = None + try: + if graph is None: + raise ValueError("Prepare and validate the neural reference first.") + ids = feeding_identifiers(graph) + if abs(model.opt.timestep-PARAMETERS.dt_ms/1000) > 1e-12: + raise ValueError("Motor link requires matching 0.1 ms clocks") + self.inputs = np.array([item["index"] for item in ids["neu_sugar"]]) + self.readouts = [{**item, "label": label} for item, label in + zip(ids["ids_mn9"], ("MN9 left", "MN9 right"))] + self.motor_indices = np.array([item["index"] for item in self.readouts]) + self.sensory = np.zeros(graph.size, dtype=bool) + self.sensory[self.inputs] = True + joint = model.joint(f"{FLY_ID}/rostrum") + self.qpos, self.dof = int(joint.qposadr[0]), int(joint.dofadr[0]) + self.actuator = model.actuator(f"{FLY_ID}/rostrum").id + self.minimum, self.maximum = model.actuator_ctrlrange[self.actuator] + # Groups are runtime allowlists, not changes to anatomy/servo parameters. + model.actuator_group[:] = 0 + model.actuator_group[self.actuator] = 1 + model.opt.disableactuator = 1 + self.disable() + self.decay = float(np.exp(-PARAMETERS.dt_ms/FILTER_MS)) + self.descriptor = { + "available": True, "input_count": len(self.inputs), "inputs": ids["neu_sugar"], + "readouts": self.readouts, "rate_hz": RATE_HZ, "seed": SEED, + "total_ms": TICKS*PARAMETERS.dt_ms, + "stimulus_ms": STIMULUS_TICKS*PARAMETERS.dt_ms, + "filter_ms": FILTER_MS, "full_drive_hz": FULL_DRIVE_HZ, + "actuator": f"{FLY_ID}/rostrum", "control_range_rad": [self.minimum, self.maximum], + "adapter": "Engineered firing-rate to position-servo target", + "source_sha256": graph.manifest["source_files"]["figures.ipynb"]["sha256"], + } + self.state = {"revision": 1, "status": "idle", "trial": 0, + "drive_enabled": False, "history": [], "readouts": []} + except (OSError, ValueError, KeyError, SyntaxError) as exc: + self.disable() + self.descriptor["error"] = f"Motor link unavailable: {exc}" + self.state["error"] = self.descriptor["error"] + + @property + def active(self): + return self.neurons is not None + + def disable(self): + self.model.opt.disableflags |= int(mujoco.mjtDisableBit.mjDSBL_ACTUATION) + if self.actuator is not None: + self.data.ctrl[self.actuator] = 0 + self.data.qfrc_applied[self.dof] = 0. + self.model.tree_sleep_policy[self.fly_tree] = mujoco.mjtSleepPolicy.mjSLEEP_ALLOWED + + def start(self, mode): + if not isinstance(mode, str) or mode not in MODES: + raise ValueError("Motor mode must be stimulus, baseline or blocked") + if not self.descriptor["available"]: + raise ValueError(self.state["error"]) + if self.active: + raise ValueError("A motor trial is already running") + self.disable() + self.mode = mode + self.neurons = LIF(self.graph, self.inputs) + self.events = poisson_events(self.inputs, TICKS, + rate_hz=0 if mode == "baseline" else RATE_HZ, + seed=SEED, stimulus_ticks=STIMULUS_TICKS) + self.started, self.physical_start = time.monotonic(), float(self.data.time) + self.cpu_seconds = self.pause_seconds = 0. + self.paused_at = None + self.next_publish = 0. + self.rate = self.activation = self.total = self.input_total = self.motor_total = 0 + self.initial_angle = float(self.data.qpos[self.qpos]) + self.peak_excursion = self.peak_force = 0. + self.history = [] + self.state = {"revision": self.state["revision"]+1, "status": "running", + "trial": self.state["trial"]+1, "mode": mode, "drive_enabled": False, + "history": [], "readouts": [], "simulated_ms": 0, "physical_ms": 0, + "total_spikes": 0, "downstream_spikes": 0, + "peak_excursion_deg": 0, "peak_force": 0} + + def set_paused(self, value): + if not self.active: + return + now = time.monotonic() + if value and self.paused_at is None: + self.paused_at = now + elif not value and self.paused_at is not None: + self.pause_seconds += now-self.paused_at + self.paused_at = None + self.state = {**self.state, "revision": self.state["revision"]+1, "paused": value} + + def _publish(self, status="running", error=None): + neurons = self.neurons + now = time.monotonic() + self.state = { + **self.state, "revision": self.state["revision"]+1, "status": status, + "drive_enabled": status == "running" and self.mode == "stimulus" and self.rate > 0, + "simulated_ms": neurons.tick*PARAMETERS.dt_ms, + "physical_ms": (self.data.time-self.physical_start)*1000, + "wall_seconds": now-self.started, "cpu_seconds": self.cpu_seconds, + "total_spikes": self.total, "input_spikes": self.input_total, + "downstream_spikes": self.total-self.input_total, + "readouts": [{**item, "spikes": int(neurons.spike_counts[item["index"]]), + "voltage_mv": float(neurons.v[item["index"]])} + for item in self.readouts], + "activation": self.activation if status == "running" else 0., + "angle_deg": float(np.degrees(self.data.qpos[self.qpos])), + "peak_excursion_deg": self.peak_excursion, "peak_force": self.peak_force, + "history": list(self.history), + } + if error: + self.state["error"] = error + + def finish(self, status="cancelled", error=None): + self.disable() + if self.active: + self._publish(status, error) + # Keep only the small measured history. Release trial state at idle. + self.neurons = self.events = None + + def step(self): + if not self.active: + raise RuntimeError("No active motor trial") + cpu_started = time.thread_time() + try: + spikes = self.neurons.step(self.events[self.neurons.tick]) + self.total += len(spikes) + self.input_total += int(np.count_nonzero(self.sensory[spikes])) + motor_total = sum(int(self.neurons.spike_counts[i]) for i in self.motor_indices) + motor_spikes, self.motor_total = motor_total-self.motor_total, motor_total + # Exponential spike-rate estimate, averaged across the bilateral pair. + self.rate = self.rate*self.decay + motor_spikes*1000/(2*FILTER_MS) + self.activation = min(1., self.rate/FULL_DRIVE_HZ) + if self.mode == "stimulus" and self.rate > 0: + self.model.opt.disableflags &= ~int(mujoco.mjtDisableBit.mjDSBL_ACTUATION) + self.model.tree_sleep_policy[self.fly_tree] = mujoco.mjtSleepPolicy.mjSLEEP_NEVER + # MuJoCo's documented bytewise wake signal: negative zero wakes + # a sleeping actuated tree without injecting a nonzero force. + self.data.qfrc_applied[self.dof] = -0. + self.data.ctrl[self.actuator] = self.maximum + self.activation*(self.minimum-self.maximum) + mujoco.mj_step(self.model, self.data) + elapsed = (self.data.time-self.physical_start)*1000 + if abs(elapsed-self.neurons.tick*PARAMETERS.dt_ms) > 1e-5: + raise RuntimeError("Neural and physical clocks diverged") + angle = float(np.degrees(self.data.qpos[self.qpos])) + excursion = abs(angle-np.degrees(self.initial_angle)) + force = float(self.data.actuator_force[self.actuator]) + self.peak_excursion = max(self.peak_excursion, excursion) + self.peak_force = max(self.peak_force, abs(force)) + if self.neurons.tick % 50 == 0: + self.history.append({ + "ms": self.neurons.tick*PARAMETERS.dt_ms, "angle_deg": angle, + "drive": self.activation if self.mode == "stimulus" else 0., + "mn9_spikes": int(self.neurons.spike_counts[self.motor_indices].sum()), + }) + self.cpu_seconds += time.thread_time()-cpu_started + if self.neurons.tick % 25 == 0: + if (self.total > SPIKE_LIMIT or self.cpu_seconds > CPU_LIMIT + or time.monotonic()-self.started-self.pause_seconds > WALL_LIMIT): + raise RuntimeError("Motor trial reached its resource limit; results are incomplete.") + if not np.isfinite(self.data.qpos).all() or self.data.warning.number.any(): + raise RuntimeError("Motor trial stopped after a physics warning") + if self.neurons.tick == TICKS: + self.finish("complete") + elif time.monotonic() >= self.next_publish: + self._publish() + self.next_publish = time.monotonic()+.1 + except Exception as exc: + self.finish("error", str(exc)) + raise diff --git a/scripts/serve_environment.py b/scripts/serve_environment.py index 4a2e5bb..3baba57 100644 --- a/scripts/serve_environment.py +++ b/scripts/serve_environment.py @@ -1,4 +1,4 @@ -"""Local browser observatory over the existing City MJCF, without a fly controller.""" +"""Local City observatory with one physical fly and a bounded neural motor assay.""" import argparse import base64 import gzip @@ -50,8 +50,7 @@ def __init__(self): solref=[.002, 1], priority=1, contype=1, conaffinity=1) xml = root.to_xml_string() self.model = mujoco.MjModel.from_xml_string(xml, root.get_assets()) - # Actuator definitions remain present but no actuator generates force. - # Sleeping is explicitly allowed only for this disconnected, passive stage. + # Start passive. A motor trial may enable only its allowlisted actuator. self.model.opt.enableflags |= int(mujoco.mjtEnableBit.mjENBL_SLEEP) self.model.opt.disableflags |= int(mujoco.mjtDisableBit.mjDSBL_ACTUATION) self.model.tree_sleep_policy[:] = mujoco.mjtSleepPolicy.mjSLEEP_ALLOWED @@ -78,9 +77,15 @@ def __init__(self): self.lock = threading.Lock() self.stop = threading.Event() self.neural = None + self.motor = None if os.environ.get("FLY_NEURAL") == "1": from neural_lab import NeuralLab + from motor_bridge import MotorBridge self.neural = NeuralLab(self.stop) + self.motor = MotorBridge(self.neural.graph, self.model, self.data, self.fly_tree) + if self.motor.descriptor["available"]: + self.fly_description["neural_controller"] = "experimental MN9-to-rostrum link" + self.fly_description["actuator_drive"] = "rostrum only during motor trials" self.paused = False self.speed = 1.0 self.probe_active = False @@ -154,8 +159,11 @@ def _export(self, xml): "source": "scripts/city_world.py", "model_sha256": hashlib.sha256(xml.encode()).hexdigest(), "engine": f"MuJoCo {mujoco.__version__}", "timestep": float(model.opt.timestep), "gravity": model.opt.gravity.tolist(), "geoms": geoms, "materials": materials, - "neural_controller": None, "fly_body": self.fly_description, + "neural_controller": ("experimental MN9-to-rostrum link" + if self.motor and self.motor.descriptor["available"] else None), + "fly_body": self.fly_description, "neural": self.neural.descriptor if self.neural else None, + "motor": self.motor.descriptor if self.motor else None, "probe": {"radius": PROBE_RADIUS, "start": PROBE_START}, "views": [ {"id": "gallery", "title": "South gallery", "position": [-50, -35, 16], @@ -174,16 +182,32 @@ def command(self, command): if not isinstance(command, dict): raise ValueError("Expected a command object") action = command.get("action") - if action == "neural_trial" and set(command) == {"action", "mode"}: - if self.neural is None: - raise ValueError("The neural lab is not enabled in this environment") - self.neural.start(command["mode"]) - return with self.lock: - if action == "pause" and set(command) == {"action", "paused"}: + if self.stop.is_set(): + raise ValueError("The environment is stopping") + if action in ("neural_trial", "motor_trial") and set(command) == {"action", "mode"}: + if self.neural is None: + raise ValueError("The neural lab is not enabled in this environment") + if self.motor.active or self.neural.snapshot()["status"] == "running": + raise ValueError("A neural trial is already running") + if action == "motor_trial": + if self.paused or self.error: + raise ValueError("Resume the physical simulation before starting a motor trial") + if self.data.tree_asleep[self.fly_tree] < 0: + raise ValueError("Let the fly settle before starting a motor trial") + self.motor.start(command["mode"]) + self.next_pose_time = 0. + else: + self.neural.start(command["mode"]) + elif action == "motor_stop" and set(command) == {"action"}: + if self.motor: + self.motor.finish() + elif action == "pause" and set(command) == {"action", "paused"}: if not isinstance(command["paused"], bool): raise ValueError("paused must be boolean") self.paused = command["paused"] + if self.motor: + self.motor.set_paused(self.paused) elif action == "speed" and set(command) == {"action", "value"}: if type(command["value"]) not in (int, float) or command["value"] not in (.25, 1): raise ValueError("speed must be 0.25 or 1") @@ -200,6 +224,8 @@ def command(self, command): self.probe_generation += 1 self.probe_trail = [PROBE_START.copy()] self.paused = False + if self.motor: + self.motor.set_paused(False) else: raise ValueError("Unknown command") @@ -225,7 +251,7 @@ def _sample_contacts(self): self.fly_contacts_seen = True self.fly_contact_count = count - def snapshot(self, pose_revision=None, trail_version=None, neural_revision=None): + def snapshot(self, pose_revision=None, trail_version=None, neural_revision=None, motor_revision=None): with self.lock: state = { "sequence": self.sequence, "time": float(self.data.time), @@ -254,6 +280,12 @@ def snapshot(self, pose_revision=None, trail_version=None, neural_revision=None) state["neural_running"] = neural["status"] == "running" if neural_revision != neural["revision"]: state["neural"] = neural + if self.motor: + motor = self.motor.state + state["motor_revision"] = motor["revision"] + state["motor_running"] = self.motor.active + if motor_revision != motor["revision"]: + state["motor"] = motor return state def _run(self): @@ -275,8 +307,12 @@ def _run(self): steps = min(int(accumulator/self.model.opt.timestep), 250) remaining = steps while remaining: - block = min(10, remaining) if self.data.ntree_awake else remaining - mujoco.mj_step(self.model, self.data, nstep=block) + if self.motor and self.motor.active: + block = 1 + self.motor.step() + else: + block = min(10, remaining) if self.data.ntree_awake else remaining + mujoco.mj_step(self.model, self.data, nstep=block) remaining -= block self._sample_contacts() if self.probe_active and len(self.probe_trail) < 500: @@ -295,12 +331,13 @@ def _run(self): if now >= self.next_pose_time: mujoco.mj_kinematics(self.model, self.data) self._cache_fly_pose() - self.next_pose_time = now + (POSE_INTERVAL if self.data.ntree_awake else 1) + active = self.data.ntree_awake or (self.motor and self.motor.active) + self.next_pose_time = now + (POSE_INTERVAL if active else 1) self.sequence += 1 if now-measured_wall >= 1: self.ratio = (self.data.time-measured_sim)/(now-measured_wall) measured_wall, measured_sim = now, self.data.time - awake = bool(self.data.ntree_awake) and not self.paused + awake = (bool(self.data.ntree_awake) or bool(self.motor and self.motor.active)) and not self.paused # Enforce the worker's duty budget even if host quota accounting # is delayed. Slow physical time rather than growing a backlog. cpu_used = time.thread_time()-cpu_started @@ -308,8 +345,14 @@ def _run(self): cpu_used/PHYSICS_CPU_BUDGET-(time.monotonic()-now)) except Exception as exc: with self.lock: + if self.motor: + self.motor.finish("error", str(exc)) self.error = str(exc) self.paused = True + finally: + with self.lock: + if self.motor: + self.motor.finish() class Server(ThreadingHTTPServer): @@ -373,18 +416,20 @@ def do_GET(self): self.send_header("Content-Type", "text/event-stream") self.send_header("Cache-Control", "no-store") self.end_headers() - pose_revision = trail_version = neural_revision = None + pose_revision = trail_version = neural_revision = motor_revision = None while not env.stop.is_set(): - state = env.snapshot(pose_revision, trail_version, neural_revision) + state = env.snapshot(pose_revision, trail_version, neural_revision, motor_revision) payload = json.dumps(state, separators=(",", ":")) self.wfile.write(f"data: {payload}\n\n".encode()) self.wfile.flush() pose_revision = state["fly"]["pose_revision"] trail_version = (state["probe"]["generation"], state["probe"]["trail_count"]) neural_revision = state.get("neural_revision") + motor_revision = state.get("motor_revision") active = not state["fly"]["sleeping"] or (state["probe"]["active"] and not state["probe"]["contact_seen"]) interval = POSE_INTERVAL if active and not state["paused"] else 1 - env.stop.wait(min(interval, .1) if state.get("neural_running") else interval) + running = state.get("neural_running") or (state.get("motor_running") and not state["paused"]) + env.stop.wait(min(interval, .1) if running else interval) except (BrokenPipeError, ConnectionResetError, TimeoutError): pass finally: @@ -414,6 +459,7 @@ def do_GET(self): "/environment.js": ("web/environment.js", "text/javascript; charset=utf-8"), "/fly.js": ("web/fly.js", "text/javascript; charset=utf-8"), "/neural.js": ("web/neural.js", "text/javascript; charset=utf-8"), + "/motor.js": ("web/motor.js", "text/javascript; charset=utf-8"), "/neural.css": ("web/neural.css", "text/css; charset=utf-8"), "/dev-reload.js": ("web/dev-reload.js", "text/javascript; charset=utf-8"), "/style.css": ("web/style.css", "text/css; charset=utf-8"), @@ -468,10 +514,10 @@ def main(): server = Server(args.port, environment) environment.start() print(f"NETSPHERE ready on port {args.port}: " - f"{len(environment.world['geoms'])} world geoms; no neural controller.", flush=True) + f"{len(environment.world['geoms'])} world geoms; one physical fly.", flush=True) if environment.neural: print(f"Neural reference lab: {environment.neural.snapshot()['status']}; " - "body actuators disconnected.", flush=True) + f"MN9 motor link: {environment.motor.state['status']}.", flush=True) try: server.serve_forever() finally: diff --git a/scripts/test_environment.py b/scripts/test_environment.py index 3aced11..b93a4ae 100644 --- a/scripts/test_environment.py +++ b/scripts/test_environment.py @@ -84,8 +84,11 @@ def test_commands_reject_unbounded_or_ambiguous_values(self): def test_passive_anatomy_and_collision_contract(self): world = self.env.world - self.assertIsNone(world["neural_controller"]) - self.assertEqual(world["fly_body"]["neural_controller"], "disconnected") + if (world.get("motor") or {}).get("available"): + self.assertEqual(world["neural_controller"], "experimental MN9-to-rostrum link") + else: + self.assertIsNone(world["neural_controller"]) + self.assertEqual(world["fly_body"]["neural_controller"], "disconnected") self.assertEqual(len(world["fly_body"]["bodies"]), 67) self.assertEqual(len(world["geoms"]), 383) self.assertTrue(all(geom["collision"] for geom in world["geoms"])) diff --git a/scripts/validate_motor_bridge.py b/scripts/validate_motor_bridge.py new file mode 100644 index 0000000..16dda5c --- /dev/null +++ b/scripts/validate_motor_bridge.py @@ -0,0 +1,153 @@ +"""Bounded causal checks of the real graph, native actuator and shared clock.""" +import argparse +import json +from pathlib import Path +import resource +import time + +import mujoco +import numpy as np + +from fetch_neural_reference import sha256 +from motor_bridge import TICKS, PARAMETERS +from serve_environment import Environment + + +def validate(): + env = Environment() + bridge, model, data = env.motor, env.model, env.data + assert bridge and bridge.descriptor["available"], "Validated neural cache required" + # Independent test initial state. Runtime trials never reset the animal. + for _ in range(60): + mujoco.mj_step(model, data, nstep=1000) + if data.tree_asleep[env.fly_tree] >= 0: + break + assert data.tree_asleep[env.fly_tree] >= 0 + resting = mujoco.MjData(model) + mujoco.mj_copyData(resting, model, data) + other = np.arange(model.nu) != bridge.actuator + results = {} + for mode in ("baseline", "blocked", "stimulus"): + bridge.disable() + mujoco.mj_copyData(data, model, resting) + before = data.qpos.copy(), data.qvel.copy(), data.time + started, cpu_started = time.monotonic(), time.thread_time() + env.command({"action": "motor_trial", "mode": mode}) + np.testing.assert_array_equal(before[0], data.qpos) + np.testing.assert_array_equal(before[1], data.qvel) + assert before[2] == data.time + for command in ({"action": "motor_trial", "mode": "stimulus"}, + {"action": "neural_trial", "mode": "stimulus"}): + try: + env.command(command) + raise AssertionError("Concurrent assay accepted") + except ValueError: + pass + first_motor_tick = first_force_tick = None + peak_other = 0. + for tick in range(TICKS): + bridge.step() + peak_other = max(peak_other, float(np.max(np.abs(data.actuator_force[other])))) + if first_force_tick is None and data.actuator_force[bridge.actuator] != 0: + first_force_tick = tick + if bridge.active and first_motor_tick is None and bridge.neurons.spike_counts[bridge.motor_indices].any(): + first_motor_tick = tick + if mode == "stimulus" and first_motor_tick is None: + np.testing.assert_array_equal(data.qpos, resting.qpos) + result = bridge.state + assert result["status"] == "complete", result + assert not bridge.active and bridge.events is None + assert model.opt.disableflags & int(mujoco.mjtDisableBit.mjDSBL_ACTUATION) + assert not result["drive_enabled"] + assert abs(result["physical_ms"]-500) < 1e-5 + assert result["simulated_ms"] == 500 + assert len(result["history"]) == 100 + assert peak_other == 0, "A non-allowlisted actuator generated force" + assert not data.warning.number.any() and np.isfinite(data.qpos).all() + counts = [item["spikes"] for item in result["readouts"]] + if mode == "baseline": + assert counts == [0, 0] and result["total_spikes"] == 0 + else: + assert counts == [30, 17] and result["total_spikes"] == 5076 + if mode != "stimulus": + assert result["peak_force"] == 0 and result["peak_excursion_deg"] == 0 + np.testing.assert_array_equal(data.qpos, resting.qpos) + else: + assert first_force_tick == first_motor_tick == 255 + assert result["peak_excursion_deg"] > 10 + assert 0 < result["peak_force"] <= .1 + assert data.tree_asleep[env.fly_tree] < 0, "Sleeping animal did not wake" + results[mode] = { + key: result[key] for key in ("total_spikes", "input_spikes", "downstream_spikes", + "peak_force", "peak_excursion_deg", "physical_ms", "simulated_ms") + } + results[mode].update(mn9_spikes=counts, other_actuator_peak_force=peak_other, + first_force_ms=None if first_force_tick is None else first_force_tick*PARAMETERS.dt_ms, + wall_seconds=time.monotonic()-started, + cpu_seconds=time.thread_time()-cpu_started) + print(mode, json.dumps(results[mode]), flush=True) + # Cancellation must remove actuator authority, including after it has fired. + bridge.start("stimulus") + for _ in range(400): + bridge.step() + assert bridge.state["drive_enabled"] or data.actuator_force[bridge.actuator] != 0 + env.command({"action": "pause", "paused": True}) + # The gravity control also resumes the environment; both clocks must resume. + fly_qpos = data.qpos[:env.probe_qpos].copy() + env.command({"action": "drop"}) + assert not env.paused and not bridge.state["paused"] and bridge.paused_at is None + np.testing.assert_array_equal(data.qpos[:env.probe_qpos], fly_qpos) + env.command({"action": "pause", "paused": True}) + frozen = data.time, bridge.neurons.tick + env.command({"action": "motor_stop"}) + assert (data.time, round(bridge.state["simulated_ms"]/PARAMETERS.dt_ms)) == frozen + assert not bridge.active and not bridge.state["drive_enabled"] + assert model.opt.disableflags & int(mujoco.mjtDisableBit.mjDSBL_ACTUATION) + assert not np.signbit(data.qfrc_applied[bridge.dof]) + first = env.snapshot() + delta = env.snapshot(neural_revision=first["neural_revision"], motor_revision=first["motor_revision"]) + assert "neural" not in delta and "motor" not in delta + try: + env.command({"action": "motor_trial", "mode": "stimulus"}) + raise AssertionError("Motor trial started while paused") + except ValueError: + pass + env.command({"action": "pause", "paused": False}) + try: + env.command({"action": "motor_trial", "mode": "stimulus"}) + raise AssertionError("Motor trial started before physical rest") + except ValueError: + pass + # Enforced clock mismatch fails closed and reports an incomplete result. + bridge.start("baseline") + bridge.physical_start -= .001 + try: + bridge.step() + raise AssertionError("Clock divergence was accepted") + except RuntimeError: + pass + assert bridge.state["status"] == "error" and not bridge.active + assert model.opt.disableflags & int(mujoco.mjtDisableBit.mjDSBL_ACTUATION) + env.stop.set() + root = Path(__file__).resolve().parents[1] + return { + "schema": 1, "passed": True, + "code_sha256": {name: sha256(root/"scripts"/name) for name in + ("motor_bridge.py", "serve_environment.py", "neural_reference.py", + "validate_motor_bridge.py")}, + "cache_manifest_sha256": sha256(root/"out/neural-reference/manifest.json"), + "protocol": bridge.descriptor, "results": results, + "peak_rss_mib": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024, + "checks": ["shared clock", "native wake", "no pre-spike movement", + "motor block", "zero-input baseline", "actuator allowlist", + "concurrent assay rejection", "cancel while paused", "clock mismatch fails closed"], + } + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + report = validate() + args.output.write_text(json.dumps(report, indent=2)+"\n") + print(f"Motor validation passed; peak RSS {report['peak_rss_mib']:.1f} MiB", flush=True) diff --git a/web/environment.js b/web/environment.js index 5fa0bd4..ae6205e 100644 --- a/web/environment.js +++ b/web/environment.js @@ -24,7 +24,7 @@ let graphicsLost = false; let graphicsDevice = '', softwareGraphics = false, renderCostMs = 0; let cameraMovedSinceHud = false; let neuralPanel; -let fly, flyView = false, orbitYaw = -.95, orbitPitch = .45, orbitDistance = .65; +let fly, flyView = false, headView = false, orbitYaw = -.95, orbitPitch = .45, orbitDistance = .65; let scheduledFrame = 0, hudTimer = 0; const freeHelp = $('.navigation-help').innerHTML; const reducedMotion = matchMedia('(prefers-reduced-motion: reduce)').matches; @@ -87,8 +87,9 @@ async function command(payload) { function updateState(next) { if (!next.probe.trail) next.probe.trail = state?.probe.trail || []; if (!next.neural || (state?.neural?.revision ?? -1) > next.neural.revision) next.neural = state?.neural; + if (!next.motor || (state?.motor?.revision ?? -1) > next.motor.revision) next.motor = state?.motor; state = next; - neuralPanel?.update(next.neural); + neuralPanel?.update(next.neural, next.motor, next.paused, next.fly.sleeping); lastEvent = performance.now(); setConnected(!next.error); if (next.error) { @@ -106,7 +107,8 @@ function updateState(next) { if (flyView) orbitCamera(); invalidate(); } - $('#fly-status').textContent = next.fly.sleeping ? 'At rest' : next.paused ? 'Paused' : 'Settling'; + $('#fly-status').textContent = next.paused ? 'Paused' : next.fly.sleeping ? 'At rest' : next.motor_running ? 'Motor trial' : 'Settling'; + $('#fly-drive').textContent = next.motor?.drive_enabled ? 'Rostrum only' : 'Off'; if (!probe) return; const moved = probe.position.distanceToSquared(scratch.fromArray(next.probe.position)) > 1e-8; if (probe.visible !== next.probe.active || (next.probe.active && moved)) { @@ -290,12 +292,13 @@ function orient() { } function orbitCamera() { + const focus = headView ? fly.headPosition : fly.position; camera.position.set( Math.cos(orbitYaw) * Math.cos(orbitPitch), Math.sin(orbitYaw) * Math.cos(orbitPitch), Math.sin(orbitPitch), - ).multiplyScalar(orbitDistance * Math.max(1, .95 / camera.aspect)).add(fly.position); - camera.lookAt(fly.position); + ).multiplyScalar(orbitDistance * Math.max(1, .95 / camera.aspect)).add(focus); + camera.lookAt(focus); syncAngles(); invalidate(); } @@ -303,6 +306,7 @@ function orbitCamera() { function setView(id, immediate = false) { cameraMovedSinceHud = true; flyView = id === 'fly'; + headView = false; document.body.classList.toggle('fly-view', flyView); $('#fly-panel').hidden = !flyView; $('#gravity-panel').hidden = flyView; @@ -480,6 +484,13 @@ function updateHud() { } function bindControls() { + $('#motor-focus').addEventListener('click', () => { + setView('fly', true); + headView = true; + orbitDistance = .35; + orbitPitch = .25; + orbitCamera(); + }); document.querySelectorAll('[data-view]').forEach((button) => { button.addEventListener('click', () => setView(button.dataset.view)); }); @@ -625,7 +636,7 @@ async function start() { world = await response.json(); if (world.neural) { const { createNeuralPanel } = await import('./neural.js'); - neuralPanel = createNeuralPanel(world.neural, command); + neuralPanel = createNeuralPanel(world.neural, world.motor, command); } await buildWorld(); const [animal, initial] = await Promise.all([ diff --git a/web/fly.js b/web/fly.js index bd810ac..a0f960d 100644 --- a/web/fly.js +++ b/web/fly.js @@ -71,6 +71,7 @@ export async function createFly(description, scene) { let revision = -1; return { position, + headPosition: bones[description.bodies.indexOf('head')].position, get revision() { return revision; }, get visible() { return rig.visible; }, apply(state) { diff --git a/web/index.html b/web/index.html index b71d573..3fe29f7 100644 --- a/web/index.html +++ b/web/index.html @@ -4,7 +4,7 @@ - + NETSPHERE — Fly environment @@ -41,7 +41,7 @@

South gallery

—CM
Physical geometries—Gravity9.81 m/s²
- +
@@ -53,8 +53,8 @@

South gallery

@@ -63,17 +63,52 @@

South gallery

CONNECTOME / REFERENCE LAB

Inside the signal.

-

Loading neural reference…

+

+
+

Brain → body · sugar response

+

Loading motor link…

+
+
    +
  1. Sugar input21 neurons
    2. +
  2. ConnectomeFull graph
    3. +
  3. MN9 pairMotor spikes
    4. +
  4. ProboscisServo adapter
    5. +
+

One experimental motor link. MN9 spikes drive the rostrum through an engineered adapter. World sensing, walking and flight are not connected.

+

+
+ + + + + +
+

+ +
0 ms brain / 0 ms bodyRuns on demand
+
+ —MN9 spikes · L / R + —Peak joint movement + —Downstream spikes +
+

Actuator drive: Off

+ +

Measured samples every 5 ms. MN9 spikes use a cumulative scale; drive is 0–1; angle is in degrees. Dashed line: input ends. The trace stays visible after movement.

+
+
+
+ Antennal reference assay · separate clock +

Loading neural reference…

  1. StimulusDirect input
  2. ConnectomeLIF model
  3. ReadoutsNeural spikes
  4. MusclesDisconnected
-

The flybody is passive. Sensory mapping from the world and motor coupling are still pending. These neural trials have their own clock.

+

This antennal assay reads neural activity only. It has its own clock and does not drive muscles.

Antennal response

@@ -104,6 +139,7 @@

Measured activity

ReadoutTrial spikesLatest voltage +

Connectome-based simulation with a simplified neuron model. Numerical checks match Brian2; this does not establish biological equivalence or autonomous behavior. Published model and data ↗

diff --git a/web/motor.js b/web/motor.js new file mode 100644 index 0000000..c38d55b --- /dev/null +++ b/web/motor.js @@ -0,0 +1,100 @@ +const $ = (id) => document.getElementById(id); + +export function createMotorPanel(descriptor, command, busyChanged) { + const buttons = [...document.querySelectorAll('[data-motor-mode]')]; + let state, connected = true, otherBusy = false, paused = false, sleeping = false, pending = false, rendered = -1; + if (!descriptor?.available) { + $('motor-status').textContent = descriptor?.error || 'Motor link unavailable in this environment.'; + $('motor-controls').hidden = true; + return { update() {}, setConnected() {}, setOtherBusy() {}, render() {} }; + } + $('motor-protocol').textContent = `${descriptor.input_count} sugar-sensing neurons · ${descriptor.rate_hz} Hz for ${descriptor.stimulus_ms} ms, then 200 ms recovery. Neural and body time advance together.`; + $('fly-controller').textContent = 'MN9 → proboscis motor link available.'; + $('organism-controller').textContent = 'One neural motor link available'; + function controls() { + const running = state?.status === 'running'; + for (const button of buttons) { + button.disabled = !connected || otherBusy || running || pending || paused || !sleeping; + button.setAttribute('aria-pressed', String(state?.mode === button.dataset.motorMode)); + } + $('motor-stop').disabled = !connected || !running || pending; + $('motor-readiness').textContent = paused ? 'Resume the simulation to continue.' + : !running && !sleeping ? 'Waiting for the body to settle before the next trial.' : ''; + busyChanged(running || pending); + } + for (const button of buttons) { + button.addEventListener('click', async () => { + pending = true; + controls(); + try { await command({ action: 'motor_trial', mode: button.dataset.motorMode }); } + finally { pending = false; controls(); } + }); + } + $('motor-stop').addEventListener('click', () => command({ action: 'motor_stop' })); + + function drawHistory(history) { + const canvas = $('motor-timeline'), ctx = canvas.getContext('2d'); + ctx.clearRect(0, 0, canvas.width, canvas.height); + ctx.font = '10px monospace'; + const rows = [ + ['mn9_spikes', '#b8d6c1', 'MN9 spikes'], + ['drive', '#e3be80', 'Drive 0–1'], + ['angle_deg', '#b5b3e6', 'Angle °'], + ]; + rows.forEach(([key, color, label], row) => { + const top = row * 39 + 5, base = top + 27; + const values = history.map((item) => item[key]); + const low = key === 'angle_deg' ? Math.min(-75, ...values) : 0; + const high = key === 'drive' ? 1 : key === 'angle_deg' ? Math.max(15, ...values) : Math.max(1, ...values); + ctx.fillStyle = '#aab8bc'; + ctx.fillText(label, 0, top + 16); + ctx.strokeStyle = '#c3d9dc26'; + ctx.beginPath(); ctx.moveTo(85, base); ctx.lineTo(412, base); ctx.stroke(); + ctx.strokeStyle = color; + ctx.beginPath(); + history.forEach((item, i) => { + const x = 85 + item.ms / descriptor.total_ms * 327; + const y = base - (item[key] - low) / (high - low) * 25; + if (i) ctx.lineTo(x, y); else ctx.moveTo(x, y); + }); + ctx.stroke(); + }); + ctx.fillStyle = '#aab8bc'; + ctx.fillText('0', 85, 137); ctx.fillText('300', 272, 137); ctx.fillText('500 ms', 373, 137); + ctx.setLineDash([3, 4]); ctx.strokeStyle = '#c3d9dc55'; + ctx.beginPath(); ctx.moveTo(281.2, 0); ctx.lineTo(281.2, 119); ctx.stroke(); ctx.setLineDash([]); + } + + function render() { + if ($('neural-panel').hidden || document.hidden || !state || rendered === state.revision) return; + rendered = state.revision; + const idle = state.status === 'idle'; + const names = { stimulus: 'Sugar response', baseline: 'No input', blocked: 'Motor link blocked' }; + $('motor-status').textContent = idle ? 'Ready · one motor link' + : state.error || `${names[state.mode]} · ${state.paused && state.status === 'running' ? 'Paused' : state.status}`; + $('motor-progress').value = state.simulated_ms || 0; + $('motor-time').textContent = `${(state.simulated_ms || 0).toFixed(0)} ms brain / ${(state.physical_ms || 0).toFixed(0)} ms body`; + $('motor-wall').textContent = idle ? 'Runs on demand' : `${(state.wall_seconds || 0).toFixed(2)} s elapsed`; + $('motor-mn9').textContent = idle ? '—' : (state.readouts || []).map((item) => item.spikes).join(' / '); + $('motor-excursion').textContent = idle ? '—' : `${(state.peak_excursion_deg || 0).toFixed(1)}°`; + $('motor-downstream').textContent = idle ? '—' : Number(state.downstream_spikes || 0).toLocaleString('en-US'); + $('motor-drive').textContent = state.drive_enabled ? 'Enabled · rostrum only' + : state.status === 'running' && state.mode === 'blocked' ? 'Blocked' : 'Off'; + $('motor-workflow-brain').classList.toggle('measured', state.downstream_spikes > 0); + $('motor-workflow-mn9').classList.toggle('measured', (state.readouts || []).some((item) => item.spikes > 0)); + $('motor-workflow-body').classList.toggle('measured', state.peak_force > 0); + drawHistory(state.history || []); + } + return { + update(next, isPaused, isSleeping) { + paused = isPaused; + sleeping = isSleeping; + if (next) state = next; + controls(); + render(); + }, + setOtherBusy(value) { otherBusy = value; controls(); }, + setConnected(value) { connected = value; controls(); }, + render, + }; +} diff --git a/web/neural.css b/web/neural.css index e879145..7631146 100644 --- a/web/neural.css +++ b/web/neural.css @@ -40,6 +40,9 @@ .neural-legend{display:flex;gap:18px;font-size:9px;color:#adc6b8} .neural-legend span:last-child{color:#d39e84} #neural-timeline{width:100%;height:auto;display:block} +#motor-timeline{width:100%;height:auto;display:block} +.antennal-details{border-top:1px solid var(--line);margin-top:22px;padding-top:16px} +.antennal-details summary{font-size:11px;color:#bfd7ce;cursor:pointer;line-height:1.6} .neural-readout-table{width:100%;border-collapse:collapse;font:11px monospace;margin:10px 0} .neural-readout-table th,.neural-readout-table td{padding:8px 0;border-bottom:1px solid #c3d9dc18;text-align:right;font-weight:400} .neural-readout-table th:first-child{text-align:left} diff --git a/web/neural.js b/web/neural.js index a8810ef..d63302b 100644 --- a/web/neural.js +++ b/web/neural.js @@ -1,3 +1,5 @@ +import { createMotorPanel } from './motor.js'; + const $ = (id) => document.getElementById(id); const svgNS = 'http://www.w3.org/2000/svg'; const format = (number) => Number(number || 0).toLocaleString('en-US'); @@ -8,14 +10,18 @@ function svgElement(tag, attributes = {}) { return element; } -export function createNeuralPanel(descriptor, command) { +export function createNeuralPanel(descriptor, motorDescriptor, command) { const stylesheet = document.createElement('link'); stylesheet.rel = 'stylesheet'; stylesheet.href = '/neural.css'; document.head.append(stylesheet); const panel = $('neural-panel'), open = $('neural-open'); const buttons = [...panel.querySelectorAll('[data-neural-mode]')]; - let state = null, rendered = -1, pending = false, connected = true; + let state = null, rendered = -1, pending = false, connected = true, motorBusy = false; + const motor = createMotorPanel(motorDescriptor, command, (value) => { + motorBusy = value; + controls(); + }); const nodeElements = new Map(), readoutElements = new Map(); open.hidden = false; @@ -24,6 +30,7 @@ export function createNeuralPanel(descriptor, command) { document.body.classList.toggle('neural-open', value); open.setAttribute('aria-expanded', String(value)); if (value) { + motor.render(); render(); $('neural-close').focus(); } else open.focus(); @@ -33,10 +40,11 @@ export function createNeuralPanel(descriptor, command) { panel.addEventListener('keydown', (event) => { if (event.key === 'Escape') { event.stopPropagation(); show(false); } }); - document.addEventListener('visibilitychange', () => { if (!document.hidden) render(); }); + document.addEventListener('visibilitychange', () => { if (!document.hidden) { render(); motor.render(); } }); if (!descriptor.available) { - $('neural-status').textContent = descriptor.error || 'Neural reference unavailable.'; + $('neural-unavailable').textContent = descriptor.error || 'Neural reference unavailable.'; + $('neural-unavailable').hidden = false; $('neural-content').hidden = true; return { update() {}, setConnected() {} }; } @@ -101,7 +109,7 @@ export function createNeuralPanel(descriptor, command) { function controls() { for (const button of buttons) { - button.disabled = !connected || pending || state?.status === 'running'; + button.disabled = !connected || pending || motorBusy || state?.status === 'running'; button.setAttribute('aria-pressed', String(state?.mode === button.dataset.neuralMode)); } } @@ -109,9 +117,10 @@ export function createNeuralPanel(descriptor, command) { button.addEventListener('click', async () => { if (pending) return; pending = true; + motor.setOtherBusy(true); controls(); try { await command({ action: 'neural_trial', mode: button.dataset.neuralMode }); } - finally { pending = false; controls(); } + finally { pending = false; motor.setOtherBusy(state?.status === 'running'); controls(); } }); } @@ -181,12 +190,14 @@ export function createNeuralPanel(descriptor, command) { } return { - update(next) { + update(next, motorState, paused, sleeping) { + motor.update(motorState, paused, sleeping); + motor.setOtherBusy(next?.status === 'running' || pending); if (!next || next.revision === state?.revision) return; state = next; controls(); render(); }, - setConnected(value) { connected = value; controls(); }, + setConnected(value) { connected = value; motor.setConnected(value); controls(); }, }; }