diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d62c80..5623275 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,9 +17,22 @@ jobs: run: python -m compileall -q scripts - name: Shell syntax run: for f in scripts/*.sh; do bash -n "$f"; done + - name: Browser module syntax + run: | + node --version + for f in web/*.js; do node --check "$f"; done - name: Lockfile is hash-pinned run: | - test "$(grep -c -- '--hash=sha256:' requirements.lock)" -ge 60 - ! grep -E '^[A-Za-z0-9_.-]+==' requirements.lock | grep -v -- '--hash' >/dev/null + python - <<'PY' + from pathlib import Path + import re + for name, minimum in [("requirements.lock", 60), ("requirements.neural-tools.lock", 7)]: + entries = [line for line in Path(name).read_text().splitlines() + if line.strip() and not line.startswith("#")] + assert len(entries) >= minimum, name + assert all(re.fullmatch(r"[A-Za-z0-9_.-]+==[^ ]+ --hash=sha256:[a-f0-9]{64}", line) + for line in entries), f"Unpinned dependency in {name}" + print(f"{name}: {len(entries)} hash-pinned wheels") + PY - name: Patch applies to the pinned flybody commit run: scripts/fetch_flybody.sh diff --git a/.gitignore b/.gitignore index f03c02b..d621605 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ data/ __pycache__/ *.pyc .venv/ +node_modules/ # Local notes and agent/session files (never published) *.local.md diff --git a/Dockerfile.neural-tools b/Dockerfile.neural-tools new file mode 100644 index 0000000..3bb753d --- /dev/null +++ b/Dockerfile.neural-tools @@ -0,0 +1,9 @@ +# Extend the locally built, locked environment; no neural tooling in the viewer. +FROM fly-environment:0.1.0 +COPY requirements.neural-tools.lock /app/ +RUN pip install --no-cache-dir --only-binary=:all: --require-hashes \ + -r /app/requirements.neural-tools.lock && pip check +ENV MPLCONFIGDIR=/tmp/matplotlib \ + OPENBLAS_NUM_THREADS=1 \ + OMP_NUM_THREADS=1 +CMD ["python", "scripts/prepare_neural_reference.py"] diff --git a/Dockerfile.neural-tools.dockerignore b/Dockerfile.neural-tools.dockerignore new file mode 100644 index 0000000..79ce243 --- /dev/null +++ b/Dockerfile.neural-tools.dockerignore @@ -0,0 +1,3 @@ +* +!Dockerfile.neural-tools +!requirements.neural-tools.lock diff --git a/NOTICE b/NOTICE index ee29a15..e51d94c 100644 --- a/NOTICE +++ b/NOTICE @@ -13,5 +13,12 @@ Pretrained policies and the flight dataset come from the flybody Figshare deposit (https://doi.org/10.25378/janelia.25309105); see the deposit for its terms. They are downloaded by scripts/fetch_data.sh, not redistributed. +The neural reference uses the published FlyWire 630 data, equations and +antennal protocol from Drosophila_brain_model by Philip Shiu and Nico Spiller: +https://github.com/philshiu/Drosophila_brain_model +Copyright (c) 2023 Philip Shiu and Nico Spiller, MIT License. +The full notice is in licenses/drosophila-brain-model-MIT.txt. +Data are fetched separately and are not redistributed here. + BLAME! is a work by Tsutomu Nihei (Kodansha). This is a fan homage and is not affiliated with or endorsed by its authors or publishers. diff --git a/README.md b/README.md index ea464e1..fc2381e 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,14 @@ # fly-netsphere -**A physics-simulated fruit fly flying through a BLAME!-inspired megastructure.** +**A connectome reference lab and a physical fruit fly in a BLAME!-inspired megastructure.** -The animal is the anatomically detailed [flybody](https://github.com/TuragaLab/flybody) model of *Drosophila melanogaster* (Google DeepMind and HHMI Janelia, *Nature* 2025). Its pretrained flight controller runs on CUDA, MuJoCo Warp integrates the body and the wing aerodynamics at 20 kHz, and a geometric navigator steers the fly through a procedurally built, fully collidable interior: the NETSPHERE. Every frame of the recording comes from the integrated physical state. Nothing is composited, keyframed or teleported. +Explore the **NETSPHERE** in a local browser, inspect one anatomical flybody animal, +and stimulate a **127,400-neuron FlyWire reference model** while watching measured +neural activity. The browser currently keeps neural signals and body actuators +disconnected. [Open the development environment](#interactive-environment-in-the-browser) +or read the [neural model and its limits](docs/neural-reference.md). + +The animal is the anatomically detailed [flybody](https://github.com/TuragaLab/flybody) model of *Drosophila melanogaster* (Google DeepMind and HHMI Janelia, *Nature* 2025). In the separate recorded-flight pipeline, its pretrained controller runs on CUDA, MuJoCo Warp integrates the body and wing aerodynamics at 20 kHz, and a geometric navigator steers the fly through the collidable interior. Every frame of those recordings comes from the integrated physical state. ![A fruit fly in flight beside a pillar of the NETSPHERE](docs/media/hero.png) @@ -17,7 +23,9 @@ The full 60-second take, its metrics and the validation files are attached to th - It **is** whole-body physics: joints, wings with ellipsoid fluid forces, and the official DMPO flight policy (wingbeat pattern generator plus a residual MLP) driving the actuators. - It **is** a real 3D world: walls, pillars, ducts, cables and walkways with collision in the same MuJoCo model that integrates the fly. The camera moves through that space. -- It is **not** a brain simulation. No connectome (MaleCNS, FlyWire) is involved. +- The **recorded flight** uses the official MLP policy. The browser development lab + also runs stimulus-response trials on the published FlyWire 630 connectome; + those neural signals are **not yet connected to the body's muscles**. - The navigator is **not** learned vision. It reads the known world geometry and the measured position at 100 Hz and picks turns and climbs with clearance for wings and body. It only changes the reference command; it never writes the animal's pose or velocity. ## Results @@ -68,6 +76,49 @@ verify_take.py independent checks: duration, decoded frames, clearance, ha ## Quick start +### Interactive environment in the browser + +The local browser observatory contains **one passive flybody animal**, selectable +with a close-up orbit camera, plus three city viewpoints and a gravity/contact +experiment. Its **neural controller is disconnected and actuator drive disabled**: +it settles physically, with no autonomous walking or flight policy. +The **Neural activity** panel runs a separate FlyWire 630 reference assay with +127,400 neurons and all 14,687,178 stored directed connections. It shows calculated +spikes, selected real connections and the workflow up to the disconnected muscles. +Stimulate antennal neurons, compare a baseline or block sensory output. +Three.js renders measured body poses on demand; native MuJoCo sleep reduces idle +work. The backend is capped at 2 CPUs and 1 GiB. The browser requests +high-performance GPU graphics and targets 30 FPS while moving, with bounded +resolution; backend physics and neural computation remain on the CPU. +[Anatomy preparation, controls and limitations](docs/browser-environment.md) · +[Neural preparation, numerical validation and limitations](docs/neural-reference.md). + +```bash +pnpm-docker install --frozen-lockfile # provisioned Socket-protected Docker launcher +scripts/fetch_flybody.sh +docker compose --profile dev build environment-dev +mkdir -p out/browser-fly +docker compose --profile dev run --rm --no-deps --user "$(id -u):$(id -g)" \ + -v "$PWD/out/browser-fly:/cache" environment-dev \ + python scripts/prepare_browser_fly.py --output /cache +docker compose --profile dev up -d --no-build environment-dev +# Open http://localhost:8089 on the same machine. +``` + +There is **one development stack**, on port 8089. Frontend edits reload the page; +Python edits restart the backend inside Docker. Code is mounted from this checkout. +No host runtime or watcher is needed. Restarting the backend resets the development +simulation and current neural trial; a frontend reload preserves backend state. +The neural panel requires the separately prepared, numerically validated cache. + +This uses the existing Python image and a pinned, single-package frontend. +It is served locally, with no external CDN. The flight-recording commands below +remain separate from the browser environment. +The GPU recording service has an explicit `recording` profile; starting the +development profile does not start it. + +### Recorded flight + Requirements: Docker with the NVIDIA container runtime and an NVIDIA GPU (developed and validated on an RTX 4090). Nothing else is installed on the host; the fetch scripts use `curl`, `tar`, `unzip`, `patch` and `sha256sum`. ```bash @@ -122,6 +173,14 @@ scripts/ verify_take.py independent acceptance checks on states, geometry and video verify_avoidance.py compare baseline, shifted-obstacle and avoidance-off runs city_world.py procedural collidable megastructure + serve_environment.py local browser service, one passive fly and live gravity test + dev_environment.py container-owned backend watcher; one dev stack on port 8089 + neural_reference.py incremental LIF dynamics over the full FlyWire 630 graph + neural_lab.py bounded on-demand trials and measured activity telemetry + fetch_neural_reference.py, prepare_neural_reference.py, validate_neural_reference.py + passive_fly.py cached anatomy attachment, disabled actuator drive + prepare_browser_fly.py bounded anatomy preparation, full-resolution mass properties + test_environment.py physical contact, sleep/wake, state isolation and telemetry checks city_navigation.py receding-horizon geometric navigator flight_cuda.py controller and physics on CUDA (MuJoCo Warp) flight_mlp.py NumPy reference of the flight policy @@ -135,6 +194,8 @@ scripts/ render_fly.py, fly_in_the_city.py, assemble_blame.py earlier experiments, kept for reference patches/ flybody plotting imports become lazy (no matplotlib/IPython in the image) docs/ validation reports, roadmap, dependency provenance, media +web/ Three.js observatory, free camera and simulation controls +package.json, pnpm-lock.yaml pinned browser dependency (Socket-protected installation) Dockerfile, docker-compose.yml, requirements.txt, requirements.lock ``` @@ -146,6 +207,11 @@ Dockerfile, docker-compose.yml, requirements.txt, requirements.lock - flybody is fetched at commit `d015e9b` with a verified tarball hash. The only local change is [one patch](patches/flybody-lazy-plot-imports.patch) that defers the matplotlib and IPython imports so the package loads without them. - Policies and the flight dataset come from the flybody Figshare deposit ([10.25378/janelia.25309105](https://doi.org/10.25378/janelia.25309105)) and are hash-checked after download. - Every `metrics.json` records the SHA-256 of the scripts, the checkpoint, the recorded states and the compiled model, so a take can be traced to the exact code that produced it. +- GitHub CI checks Python, shell and browser-module syntax, verifies both Python + dependency locks and applies the anatomy import patch to the pinned upstream + source. It runs on pull requests and `main`; it does not allocate a GPU or + download the full neural dataset. Physical and neural comparisons run locally + in the bounded Docker environments described above. ## Roadmap @@ -155,6 +221,7 @@ Ideas after the validated minute, in order of preference: a vertical shaft cross - flybody: Vaxenburg et al., *Whole-body physics simulation of fruit fly locomotion*, Nature 643, 1312–1320 (2025). [Paper](https://www.nature.com/articles/s41586-025-09029-4) · [Code](https://github.com/TuragaLab/flybody) (Apache-2.0) · [Data](https://doi.org/10.25378/janelia.25309105). - [MuJoCo](https://github.com/google-deepmind/mujoco), [MuJoCo Warp](https://github.com/google-deepmind/mujoco_warp), [NVIDIA Warp](https://github.com/NVIDIA/warp) and [dm_control](https://github.com/google-deepmind/dm_control). +- Neural reference: [Shiu and Spiller's Drosophila brain model](https://github.com/philshiu/Drosophila_brain_model), using the published FlyWire 630 graph. MIT notice in [licenses/drosophila-brain-model-MIT.txt](licenses/drosophila-brain-model-MIT.txt). Brian2 is the numerical comparator; PyArrow prepares the graph. - The NETSPHERE is a fan homage to Tsutomu Nihei's *BLAME!*. Nothing here is official or affiliated. Licensed under the Apache License 2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE). diff --git a/docker-compose.yml b/docker-compose.yml index edbc0fa..c26d552 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,6 @@ services: fly: + profiles: ["recording"] build: . working_dir: /app volumes: @@ -19,5 +20,61 @@ services: gpus: all command: bash scripts/simulate_city.sh + neural-tools: + profiles: ["neural-tools"] + build: + context: . + dockerfile: Dockerfile.neural-tools + image: fly-neural-tools:0.1.0 + working_dir: /app + user: "${LOCAL_UID:-1000}:${LOCAL_GID:-1000}" + volumes: + - .:/app:ro + - ./data/neural-reference:/data + - ./out/neural-reference:/out + environment: + MUJOCO_GL: disable + OPENBLAS_NUM_THREADS: "1" + OMP_NUM_THREADS: "1" + init: true + read_only: true + tmpfs: + - /tmp:size=32m,mode=1777 + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + mem_limit: 512m + cpus: 0.5 + pids_limit: 32 + + environment-dev: + profiles: ["dev"] + build: . + image: fly-environment:0.1.0 + working_dir: /app + command: python scripts/dev_environment.py --port 8089 + volumes: + - .:/app:ro + ports: + - "127.0.0.1:8089:8089" + environment: + MUJOCO_GL: disable + OPENBLAS_NUM_THREADS: "1" + OMP_NUM_THREADS: "1" + FLY_DEV: "1" + FLY_NEURAL: "1" + init: true + read_only: true + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + mem_limit: 1g + cpus: 2.0 + pids_limit: 64 + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8089/health', timeout=3)"] + interval: 15s + timeout: 5s + retries: 3 + start_period: 30s + volumes: warp-cache: diff --git a/docs/browser-environment.md b/docs/browser-environment.md new file mode 100644 index 0000000..4f71f6f --- /dev/null +++ b/docs/browser-environment.md @@ -0,0 +1,230 @@ +# Browser environment + +The local observatory displays the existing NETSPHERE directly from its compiled +MuJoCo geometry, with one physical flybody animal on the lower walkway. +The initial **Fly 001** view follows its measured thorax: drag to orbit and scroll +to zoom. Select another view to explore the city. Drag to look around, use **W A S D** to move, **Q / E** to move +vertically, **Shift** to move faster and the mouse wheel to move forward/backward. +The three city view buttons return to known observation points. Touch screens have +movement buttons and support dragging to look. + +The animal is a **passive physical body with its neural controller disconnected**. +All actuator forces are disabled; anatomical springs, joints, gravity and contact +remain active. It settles under physics; it does not walk or fly autonomously. +The **Neural activity** panel opens a separate connectome reference assay: +antennal stimulus → FlyWire LIF dynamics → neural readouts → disconnected muscles. +Its structure view shows a selection of actual connections with schematic positions; +the entire published graph is retained in the simulation. Counts and voltages come +from calculated activity, and the timeline groups measured spikes in 5 ms bins. +The trial clock is separate from the physical clock. Physical contact, mouse +movement and camera direction do not stimulate the brain yet. +[Protocol, preparation and validation](neural-reference.md). +Select it with **Fly 001**, the specimen card, or its marker when visible. +Selection and camera movement never command its muscles. +The separate recorded-flight pipeline remains available. +The amber sphere is an explicitly initialized gravity experiment, not an animal. + +## Start + +Use the provisioned, Socket-protected Docker package launcher: + +```bash +pnpm-docker install --frozen-lockfile +scripts/fetch_flybody.sh # pinned anatomical source; policies are unnecessary here +docker compose --profile dev build environment-dev +mkdir -p out/browser-fly +docker compose --profile dev run --rm --no-deps --user "$(id -u):$(id -g)" \ + -v "$PWD/out/browser-fly:/cache" environment-dev \ + python scripts/prepare_browser_fly.py --output /cache +docker compose --profile dev up -d --no-build environment-dev +``` + +Open **http://localhost:8089** on the same machine. This is the only development +stack. The service binds to loopback; it is not a public deployment. +An SSH tunnel can forward port 8089 for remote use. +After installing dependencies, the page loads all scripts and textures locally. +No CDN, analytics, external fonts or browser extensions are required. Preparation +is done once; rebuild the cache after changing the source anatomy or generator. +Startup verifies the source XML, generator, physics and visual packet hashes. +The manifest also records source mesh hashes for provenance. + +```bash +docker compose logs --tail 30 environment-dev +docker compose stop environment-dev +``` + +Stopping the service ends the current session. There is no restart policy or +persistent recording in this first environment version. + +## Interactive development + +The detached `environment-dev` container uses a read-only bind mount of this +checkout. `scripts/dev_environment.py` checks backend source timestamps once per +second and restarts its child server after a Python edit. Syntax errors postpone +reload, leaving the previous process running and a diagnostic in container logs. +The injected `web/dev-reload.js` checks asset versions every 1.5 seconds while +visible, then reloads the page when frontend files or the backend instance change. +This is page live-reload and process restart, not module HMR. + +A backend restart starts a new development body simulation and ends any current +trial. A frontend reload preserves backend state. Neural preparation and Brian2 +validation are explicit one-shot tools; the watcher never reruns them on an edit. +After regenerating the neural cache, restart `environment-dev` to load it. +Docker configuration or watcher changes also require restarting/recreating that +one service. Reuse port 8089; do not launch a second viewer stack. +The GPU recording service belongs to the separate `recording` profile, so +activating the `dev` profile cannot start a flight recording in the background. + +## Simulation and rendering + +- `scripts/serve_environment.py` instantiates `City` from `city_world.py` and + attaches the cached fly anatomy and compiles its MJCF with MuJoCo 3.13.0. + The exported 383 world geoms retain their + primitive dimensions and compiled world transforms. Geometry is not rebuilt + procedurally in JavaScript. Positions and dimensions use centimetres. +- MuJoCo runs on the CPU at a 0.1 ms step, with gravity −981 cm/s², matching the + anatomical model's default passive/walking step. This is **not the 50 µs flight + physics timestep** and does not measure controlled flight or neural performance. +- Preparation processes one source geom at a time. Mass, centre of mass and + inertia are computed from the full-resolution source before visual reduction; + explicit principal inertias preserve even tiny foot tensors. Recompilation + checks mass and inertia to relative tolerance 1e-10. Collision primitives, + joints, passive parameters, tendons and sensor definitions remain intact. +- Only noncolliding visual surfaces are simplified. The current cache contains + 103,807 triangles (source: 272,550), with maximum original-vertex displacement + 15.2 µm, and a 2.56 MiB binary packet before HTTP compression. + The 67 physical segments have a total mass of approximately 0.985 mg. +- Native MuJoCo sleep is allowed for this disconnected stage, at the default + 0.001 cm/s tolerance. The attachment frame's rotational sleep length is corrected + from its distance to the world origin to a conservative anatomical collision + radius (0.286 cm). This changes sleep eligibility, not forces or integration. + Settling is integrated normally; no pose is frozen or reset to fake a rest state. + Sleep remains an approximation: the measured difference from awake dynamics + over 200 ms is about 10 µm, below the measured visual reduction displacement. + Stricter tolerances kept micro-adjustments active beyond 15 simulated seconds. + This accuracy/idle-cost tradeoff is specific to the passive viewer, not a + connectome or controlled-locomotion validation. + See [MuJoCo sleeping semantics](https://mujoco.readthedocs.io/en/stable/programming/simulation.html#sleeping-islands). + Connecting actuators later requires explicit wake handling. +- One backend simulation thread advances independently of browser tabs. Server + events target changed body poses at up to 30 Hz and idle status at 1 Hz. + Unchanged poses and trails are omitted per viewer; reconnect starts with a full + state. Native stepping runs in bounded blocks with no catch-up backlog. + A thread-CPU duty budget of 0.9 core leaves headroom for HTTP inside the + container ceiling. Integration releases the shared state lock after roughly + 6 ms of thread CPU, checked between native stepping blocks, so costly body + settling cannot hold up every state read for a full 250-step batch. + Only executed steps are subtracted from the bounded accumulator. + While settling, physical time may advance slower than wall + time; the footer reports the measured ratio. Resting bodies resume near 1×. + Pause and 0.25× speed affect the + shared simulation; the observer camera remains usable while paused. +- Three.js 0.180.0 uses WebGL 2 for the viewer. Static solids are batched by + material; the fly uses seven rigidly skinned material batches sharing one + skeleton. Its geometry is loaded once, and only segment transforms change. + Anatomy smaller than eight rendered pixels is culled; a selectable marker + identifies its position when unobstructed. Surface images come from the compiled model; lighting, fog, UV tiling + and perspective are observer presentation and do not simulate compound eyes. +- Rendering is on demand, without a perpetual animation callback: a stationary + camera and settled bodies produce no new + scene frames. Movement targets 30 FPS, paced on browser vsync without an extra + full-frame timer. The pixel ceiling is 1.5 million (360,000 for detected + software renderers, which also target 30 FPS). Hidden tabs stop drawing. + WebGL requests the browser's high-performance GPU. The footer identifies GPU, + software or undisclosed WebGL rendering. It reports `FPS` while moving the + camera, `UPDATES/S` for changed scene frames while stationary and `VIEW IDLE` + when nothing redraws. A low stationary update count does not measure the + renderer's maximum frame rate. + A GPU preference is a request, not a guarantee that hardware acceleration is + available in that browser. Container GPU access would not accelerate this + client-side WebGL renderer. + World shadow maps are 1024² and only recomputed when the probe moves. + The tiny fly receives direct scene lighting without a separate shadow pass. + Its close-up camera uses a smaller near plane and orbit controls. +- The camera is an observer with swept point-ray clearance against source solids. + It is not an embodied collision controller. View presets may move the observer + directly and never move a physical animal. +- “Test gravity” releases a 9 mm sphere with steel density (7.85 g/cm³) above `bridge_0`, preserving the + global simulation clock and the fly's position and velocity. The displayed trail + consists of measured positions in a fixed-size GPU buffer. + Releasing it again explicitly initializes a new test. The sphere has ordinary + contact/friction and finite solver penetration. + +The server only serves an allowlist of frontend assets and read endpoints: +`/api/world`, `/api/state`, `/api/events`, `/health`, plus `/api/dev-version` in +development mode. `POST /api/command` accepts bounded drop, pause, speed and +`neural_trial` commands with same-origin JSON. Neural trial mode must be +`stimulus`, `baseline` or `blocked`; one trial runs at a time. It cannot read +arbitrary project files or edit the scene. At most eight event streams are open +at once. The container has a read-only filesystem and limits of 2 CPUs, 1 GiB +RAM and 64 PIDs. These limits cover the backend; the browser is a separate process. +The neural worker has an additional duty budget of 0.15 core, runs only on request +and publishes changes at up to 10 Hz. It adds no continuous scene render loop. +This local server is not intended to be exposed directly to the internet. + +## Validation + +```bash +pnpm-docker run check +docker compose --profile dev exec environment-dev python scripts/test_environment.py +docker compose --profile dev exec environment-dev python -c \ + "import pathlib; [compile(p.read_bytes(), str(p), 'exec') for p in pathlib.Path('scripts').glob('*.py')]" +``` + +The physics checks isolate gravity in zero air for an analytic trajectory in centimetres, +check contact and the final height on the existing bridge, preserve the global +clock and animal state across probe resets, and reject unsupported commands. +They check the unchanged anatomical contracts, disabled actuation, fly contact, +native sleep and force-triggered wake, and compare sleep with 200 ms of awake +dynamics (segment drift must remain below the measured visual reduction +displacement, currently 15.2 µm). Delta telemetry is also checked. +Browser checks cover visible rendering, view changes, mouse/keyboard motion, +pause/resume, slow motion, the gravity experiment, reconnect and mobile layout. +The measured passive-viewer sample below predates the neural panel. The later +publication check repeated all seven physical checks and checked the JavaScript, +Python, shell and dependency locks. A short browser session exercised stimulus, +baseline and blocked sensory output, verified their displayed counts, and +inspected the panel on desktop and at 390×844 with no horizontal overflow. +There were no JavaScript exceptions or failed requests; a software graphics +context reset during viewport resizing recovered. The session was then closed. +Page/process live-reload and 30 FPS on the user's hardware remain separate from +those checks. + +Resource efficiency is an acceptance requirement. Keep the one-animal default +and existing container ceilings; measure CPU, RAM and browser rendering before +increasing population. Multiple animals would need independent dynamic state and +shared immutable surfaces/connectivity. No multi-animal runtime is enabled yet. +Neural activity, sensory channels and motor control must be validated separately +before describing this as connectome-controlled behavior. + +### Measured passive-viewer sample + +Earlier validation on 2026-09-12, one animal and one isolated browser, before the +resource increase (0.5 CPU / 512 MiB container, 0.25-core physics budget): + +| Workload | Observed | +|---|---:| +| Anatomy preparation peak RSS | 231.6 MiB | +| Server while settling, 24-second sample | 25–26% of one CPU core; about 131 MiB | +| Server at rest, 3-second sample | 8.34% of one CPU core; 137.5 MiB | +| Isolated Chrome at rest, 3-second sample | 0.33% of one CPU core; 495.3 MiB summed PSS | +| New scene frames during a stationary 2.2-second observation | 0 | +| Simulated clock advance during that observation | 2.001 seconds | +| Native settling time | 4.8 simulated seconds | +| Maximum segment drift in the 200 ms awake comparison | 10.062 µm | +| Triangles in the distant gallery after animal culling | 16,802 | + +Browser measurements include Chrome and its software renderer, with a 360,000-pixel +budget and 12 FPS ceiling. They are short samples, not guarantees for other tabs, +hardware or neural workloads. The backend allocates no GPU. The startup settling +phase runs slower than wall time under its CPU budget; idle physics subsequently +runs near 1×. Seven physics/transport checks passed. Browser checks also confirmed +orbit/zoom, selection and return, mobile framing, pause/resume, probe contact without +moving the animal, unchanged geometry counts at rest, and no console or request +errors. The isolated test browser was closed afterwards. + +The subsequent performance round increased the dev ceiling to 2 CPUs / 1 GiB +and the physics budget to 0.9 core, shortened lock-held batches, raised pose +delivery toward 30 Hz and requested high-performance browser graphics. Container +diagnostics are distinct from user-side FPS: the 30 FPS target still requires +measurement while moving the camera in the user's browser. diff --git a/docs/neural-reference.md b/docs/neural-reference.md new file mode 100644 index 0000000..2b7a63a --- /dev/null +++ b/docs/neural-reference.md @@ -0,0 +1,172 @@ +# Neural reference lab + +The development observatory at **http://localhost:8089** runs a bounded antennal +stimulus-response assay using the published **FlyWire 630** graph: **127,400 +neurons and 14,687,178 stored directed connections**, with their signed synapse +counts preserved. It uses the data and model parameters from +[Shiu and Spiller's reference repository at commit 91bdd1e](https://github.com/philshiu/Drosophila_brain_model/tree/91bdd1e7dcf193f3e7ca5a8933497fcef63b7960). +This is not the newer MaleCNS dataset. Specimens and releases remain distinct. + +The body remains passive. There is no motor mapping, walking pilot or flight +policy in the browser. The neural assay has its own clock and starts from rest +for each trial; it is not a continuously embodied animal. World-to-sensory mapping, +ventral nerve cord/muscle coupling and their validation remain future work. + +## Protocol and visible signals + +Open **Neural activity** in the Fly 001 card. Each trial lasts 150 ms of neural +time: 100 ms of input followed by 50 ms of recovery. The 146 published JON +antennal neurons come from the CE, F and D groups in the reference notebook. +Input is delivered directly to those neurons, not through simulated antenna +mechanics. A fixed seed (`20260912`) makes comparisons repeatable. + +| Control | Input | Intervention | +|---|---|---| +| Stimulate antennal neurons | 150 Hz per input neuron | Normal published connections | +| Baseline | No external input | Normal published connections | +| Block sensory output | Same input events as stimulus | Zero outgoing weights from input neurons | + +Silencing follows the reference code's outgoing-weight intervention. It does not +delete neurons. The JON neurons still spike in the blocked trial. + +The panel shows accumulated trial spikes, membrane voltages, a 5 ms binned +timeline and published aBN1, aDN1 and aDN2 readouts. The last two are descending +neurons; aBN1 is an interneuron. The schematic displays 21 selected actual neurons +and their connections, using no invented anatomical coordinates. The selection +does not prune the simulated graph. Node sizes/colors encode measured accumulated +spikes and remain visible after completion; no decorative animation runs at rest. +Excitatory/inhibitory edges show signed structural weights, not measured current +on each connection. + +The neural panel updates from the existing event stream without invalidating the +3D scene. It redraws its diagram/timeline only after a changed sample, while open +and visible. Muscle disconnection stays explicit in the workflow. + +## Model and numerical comparison + +`scripts/neural_reference.py` integrates the reference leaky integrate-and-fire +equations analytically in float64, using a 0.1 ms clock: + +```text +dv/dt = (-52 mV - v + g) / 20 ms +dg/dt = -g / 5 ms +spike when v > -45 mV; reset v = -52 mV and g = 0 +refractory period = 2.2 ms (0 for externally driven input neurons) +synaptic delay = 1.8 ms; signed synapse count × 0.275 mV +external input event = 250 × 0.275 mV added to v +``` + +The update order is groups → threshold detection → delayed synapses → external +input → resets. Refractory variables reject input and remain frozen, matching +Brian2. The clock resumes at `tick - last_spike >= refractory_ticks`; this +boundary was checked against Brian2's generated state updater. N=1 Poisson input +uses an independent Bernoulli draw per input neuron per step. + +The comparator is **Brian2 2.8.0.4** with NumPy code generation, installed only in +the disposable tooling container. The original repository specified Brian2 +2.5.1; this is not a claim to reproduce its exact historical software environment. +Its reset string also includes `w=0`, although no neuronal `w` variable is +declared in the supplied equations. The comparator retains the defined `v` and +`g` resets and omits that undeclared assignment. The downloaded reference Python +is retained as text and never imported or executed. + +`scripts/validate_neural_reference.py` compares membrane and synaptic states to +absolute tolerance 1e-8 mV, with **exact spike indices and ticks**, on excitation, +inhibition, recurrent and disconnected fixtures and an induced graph of the real +selected neurons. It also compares chunked versus continuous execution and +equilibrium skipping versus dense arithmetic. Deterministic input events are +identical in both implementations, avoiding unrelated RNG differences. + +The full graph then runs the three interventions. This is a numerical and causal +check of the implementation, **not biological validation**, a reproduction of +all paper figures, or a whole-graph Brian2 comparison. + +Measured on 2026-09-12: + +| Trial | Input spikes | Downstream spikes | aBN1 / aDN1 / aDN2 | +|---|---:|---:|---| +| Baseline | 0 | 0 | 0 / 0 / 0 | +| Antennal stimulus | 2,200 | 706 | 6 / 2 / 2 | +| Sensory output blocked | 2,200 | 0 | 0 / 0 / 0 | + +All numerical fixtures matched spike times, with maximum membrane discrepancy +7.11e-13 mV. The stimulated full graph had 383 neurons that spiked. The standalone +solver used about 0.213 CPU seconds for 150 ms of neural time. The budgeted assay +service took 1.552 seconds elapsed / 0.233 CPU seconds, and its published readouts +and 30 timeline bins matched the validated solver. These short, sparse-input +measurements used the earlier 0.5 CPU / 512 MiB container and do not predict +continuous whole-animal performance. + +Reports live locally in `out/neural-reference/validation.json` and +`service-validation.json`. A subsequent publication check exercised all three +controls in the browser and observed the same counts and readouts. Desktop and +390×844 mobile views were inspected, with a scrollable panel and no horizontal +overflow. No JavaScript exceptions or failed requests were recorded. The +software graphics context was lost/restored during the viewport change and the +view recovered. This browser check does not measure hardware-accelerated FPS. + +## Resources and preparation + +The viewer and neural worker share **one** development container, limited to +2 CPUs, 1 GiB and 64 PIDs, with no GPU compute allocation. Browser rendering +separately requests its high-performance GPU. The neural worker has a +0.15-core duty budget in addition to the physics worker's 0.9-core budget. +One on-demand trial can run at a time; limits are 90 seconds elapsed, 10 CPU +seconds and 100,000 spikes, checked every 25 steps. A limit stops the trial +explicitly with incomplete-result status instead of silently truncating it. +Small block overshoot is possible; Docker provides the outer CPU/memory limits. + +Connectivity is immutable and memory-mapped, approximately 114 MiB of packed +arrays. Dynamic state belongs to the trial. Neurons never reached by input stay +at exact equilibrium and need no arithmetic; once reached, they remain in the +integration set without activity cutoffs. No neurons or stored edges are removed. +The viewer loads neither PyArrow nor Brian2. Browser rendering resources remain +separate from Docker's limits. + +Preparation streams the Parquet graph in 65,536-row batches, with one Arrow +thread. Its measured peak RSS was 255.3 MiB; numerical validation peaked at +168.8 MiB. Sources are pinned to the reference Git commit, verified against Git +blob hashes, then recorded with SHA-256. Cache files and the numerical report +are checked at runtime. A missing or stale report disables the panel's controls; +there is no fake fallback. + +Build the existing `fly-environment:0.1.0` image and prepare the body as described +in [browser-environment.md](browser-environment.md), then prepare the neural cache. +Dependencies and runtime stay in Docker: + +```bash +mkdir -p data/neural-reference out/neural-reference + +# Tiny build context, with executable CPU/RAM limits on the legacy builder. +tar -cf - Dockerfile.neural-tools requirements.neural-tools.lock | + DOCKER_BUILDKIT=0 docker build --cpu-period=100000 --cpu-quota=50000 \ + --memory=512m --memory-swap=512m \ + -f Dockerfile.neural-tools -t fly-neural-tools:0.1.0 - + +LOCAL_UID="$(id -u)" LOCAL_GID="$(id -g)" \ + docker compose --profile neural-tools run --rm --no-deps neural-tools \ + python scripts/fetch_neural_reference.py +LOCAL_UID="$(id -u)" LOCAL_GID="$(id -g)" \ + docker compose --profile neural-tools run --rm --no-deps neural-tools \ + python scripts/prepare_neural_reference.py +LOCAL_UID="$(id -u)" LOCAL_GID="$(id -g)" \ + docker compose --profile neural-tools run --rm --no-deps neural-tools \ + python scripts/validate_neural_reference.py + +docker compose --profile dev up -d --no-build environment-dev +# If already running, reload the new immutable cache: +docker compose --profile dev restart environment-dev +``` + +The tools image extends the local base with hash-locked binary wheels for +Brian2, PyArrow and their required additions. It is a one-shot utility, not a +second persistent development stack. Its containers have 0.5 CPU / 512 MiB / +32 PIDs, no GPU, a read-only application mount and a 32 MiB temporary filesystem. +Only data/output mounts are writable, as the invoking user's UID/GID. +Downloads are capped at 4 MiB/s. No full spike raster or large state recording +is kept by the live lab. + +Development edits use the same checkout and port 8089. The page reloads for +frontend changes; Python changes restart the server in Docker. Backend restart +resets the development simulation and trial. Neural core changes require the +explicit numerical validation step before that new core can run in the lab. diff --git a/licenses/drosophila-brain-model-MIT.txt b/licenses/drosophila-brain-model-MIT.txt new file mode 100644 index 0000000..deca2d0 --- /dev/null +++ b/licenses/drosophila-brain-model-MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Philip Shiu and Nico Spiller + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/package.json b/package.json new file mode 100644 index 0000000..3818094 --- /dev/null +++ b/package.json @@ -0,0 +1,13 @@ +{ + "name": "fly-netsphere-browser", + "version": "0.1.0", + "private": true, + "type": "module", + "packageManager": "pnpm@11.24.0", + "scripts": { + "check": "node --check web/environment.js && node --check web/fly.js && node --check web/neural.js && node --check web/dev-reload.js" + }, + "dependencies": { + "three": "0.180.0" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml new file mode 100644 index 0000000..a81f156 --- /dev/null +++ b/pnpm-lock.yaml @@ -0,0 +1,24 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +pnpmfileChecksum: sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= + +importers: + + .: + dependencies: + three: + specifier: 0.180.0 + version: 0.180.0 + +packages: + + three@0.180.0: + resolution: {integrity: sha512-o+qycAMZrh+TsE01GqWUxUIKR1AL0S8pq7zDkYOQw8GqfX8b8VoCKYUoHbhiX5j+7hr8XsuHDVU6+gkQJQKg9w==} + +snapshots: + + three@0.180.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 0000000..9a31048 --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1,7 @@ +packages: + - . +minimumReleaseAge: 10080 +blockExoticSubdeps: true +ignoreScripts: true +strictDepBuilds: true +packageImportMethod: clone diff --git a/requirements.neural-tools.lock b/requirements.neural-tools.lock new file mode 100644 index 0000000..83d3338 --- /dev/null +++ b/requirements.neural-tools.lock @@ -0,0 +1,9 @@ +# Preparation/reference tools only. CPython 3.10, Linux x86_64 wheels from PyPI. +# The existing image supplies its hash-pinned NumPy, packaging and setuptools. +pyarrow==17.0.0 --hash=sha256:f7ae2de664e0b158d1607699a16a488de3d008ba99b3a7aa5de1cbc13574d047 +Brian2==2.8.0.4 --hash=sha256:b2655275f925f6ef0c900a4d3623a9958e2748c679da978fc543a8bd80752dde +Cython==3.0.12 --hash=sha256:bee2717e5b5f7d966d0c6e27d2efe3698c357aa4d61bb3201997c7a4f9fe485a +sympy==1.12 --hash=sha256:c3588cd4295d0c0f603d0f2ae780587e64e2efeedb3521e46b9bb1d08d184fa5 +mpmath==1.3.0 --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c +pyparsing==3.2.1 --hash=sha256:506ff4f4386c4cec0590ec19e6302d3aedb992fdc02c761e90416f158dacf8e1 +Jinja2==3.1.6 --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 diff --git a/scripts/dev_environment.py b/scripts/dev_environment.py new file mode 100644 index 0000000..ff3363b --- /dev/null +++ b/scripts/dev_environment.py @@ -0,0 +1,67 @@ +"""Container-owned development supervisor; no dependencies or host watcher.""" +import argparse +from pathlib import Path +import signal +import subprocess +import sys +import threading + +ROOT = Path(__file__).resolve().parents[1] +WATCHED = [ + "serve_environment.py", "passive_fly.py", "city_world.py", + "prepare_browser_fly.py", "fetch_neural_reference.py", + "neural_reference.py", "neural_lab.py", +] + + +def versions(): + return {name: (ROOT/"scripts"/name).stat().st_mtime_ns + for name in WATCHED if (ROOT/"scripts"/name).is_file()} + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--port", type=int, default=8089) + args = parser.parse_args() + stop = threading.Event() + for sig in (signal.SIGINT, signal.SIGTERM): + signal.signal(sig, lambda *_: stop.set()) + child = None + + def shutdown(): + if child is not None and child.poll() is None: + child.terminate() + try: + child.wait(timeout=5) + except subprocess.TimeoutExpired: + child.kill() + child.wait() + + def launch(): + print(f"Development server: port {args.port}; backend watcher active", flush=True) + return subprocess.Popen([sys.executable, "scripts/serve_environment.py", + "--port", str(args.port)], cwd=ROOT) + + previous = versions() + try: + child = launch() + while not stop.wait(1): + current = versions() + if current == previous: + continue + previous = current + try: + for name in current: + path = ROOT/"scripts"/name + compile(path.read_bytes(), name, "exec") + except SyntaxError as exc: + print(f"Reload postponed: {exc}", flush=True) + continue + shutdown() + child = launch() + finally: + shutdown() + + +if __name__ == "__main__": + main() diff --git a/scripts/fetch_neural_reference.py b/scripts/fetch_neural_reference.py new file mode 100644 index 0000000..53ffb98 --- /dev/null +++ b/scripts/fetch_neural_reference.py @@ -0,0 +1,102 @@ +"""Fetch data and read-only reference text at the published model's pinned commit.""" +import argparse +import ast +import csv +import hashlib +import json +from pathlib import Path +import time +import urllib.request + +COMMIT = "91bdd1e7dcf193f3e7ca5a8933497fcef63b7960" +BASE = f"https://raw.githubusercontent.com/philshiu/Drosophila_brain_model/{COMMIT}/" +FILES = { + "2023_03_23_completeness_630_final.csv": 3057611, + "2023_03_23_connectivity_630_final.parquet": 86630944, + "figures.ipynb": 33229, + "model.py": 11900, + "LICENSE": 1085, +} + + +def sha256(path): + value = hashlib.sha256() + with Path(path).open("rb") as stream: + for block in iter(lambda: stream.read(1024*1024), b""): + value.update(block) + return value.hexdigest() + + +def fetch(destination): + destination.mkdir(parents=True, exist_ok=True) + api = f"https://api.github.com/repos/philshiu/Drosophila_brain_model/git/trees/{COMMIT}" + with urllib.request.urlopen(api, timeout=30) as response: + tree = {item["path"]: item for item in json.load(response)["tree"]} + manifest = {"dataset": "FlyWire 630", "reference_commit": COMMIT, + "reference_url": BASE, "files": {}} + for name, size in FILES.items(): + if tree[name]["size"] != size: + raise ValueError(f"Unexpected source size: {name}") + # Reference code is retained as text and is never imported or executed. + target = destination/(name+".txt" if name.endswith(".py") else name) + if not target.is_file(): + pending = target.with_suffix(target.suffix+".part") + total = 0 + started = time.monotonic() + with urllib.request.urlopen(BASE+name, timeout=30) as response, pending.open("wb") as stream: + while block := response.read(256*1024): + total += len(block) + if total > size: + raise ValueError("Download exceeded its byte ceiling") + stream.write(block) + # Bound network/disk pressure to 4 MiB/s, with constant RAM. + time.sleep(max(0, total/(4*1024*1024)-(time.monotonic()-started))) + if total != size: + raise ValueError(f"Incomplete download: {name}") + pending.replace(target) + # A pinned Git blob also validates locally cached bytes. + value = hashlib.sha1(f"blob {size}\0".encode()) + with target.open("rb") as stream: + for block in iter(lambda: stream.read(1024*1024), b""): + value.update(block) + if target.stat().st_size != size or value.hexdigest() != tree[name]["sha"]: + raise ValueError(f"Source blob mismatch: {name}") + manifest["files"][name] = {"local_name": target.name, "bytes": size, + "sha256": sha256(target), "git_blob": tree[name]["sha"]} + print(f"Verified {name}: {size/1024**2:.2f} MiB", flush=True) + (destination/"sources.json").write_text(json.dumps(manifest, indent=2)+"\n") + return manifest + + +def antennal_identifiers(path): + """Read literal identifiers from the notebook; never execute notebook cells.""" + notebook = json.loads(Path(path).read_text()) + wanted = {"neu_JON_CE", "neu_JON_F", "neu_JON_D_m", + "id_DN1_1", "id_DN2_l", "id_aBN1", "three_inhibitory"} + values = {} + for cell in notebook["cells"]: + if cell["cell_type"] != "code": + continue + for node in ast.parse("".join(cell["source"])).body: + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target = node.targets[0] + if isinstance(target, ast.Name) and target.id in wanted: + value = ast.literal_eval(node.value) + items = value if isinstance(value, list) else [value] + if not all(type(item) is int and 0 < item < 2**63 for item in items): + raise ValueError("Invalid published neuron identifiers") + if target.id in values and values[target.id] != value: + raise ValueError("Conflicting published identifiers") + values[target.id] = value + if values.keys() != wanted: + raise ValueError("Incomplete published antennal protocol") + return values + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, default=Path("/data")) + args = parser.parse_args() + fetch(args.output) + protocol = antennal_identifiers(args.output/"figures.ipynb") + (args.output/"antennal-identifiers.json").write_text(json.dumps(protocol, indent=2)+"\n") diff --git a/scripts/neural_lab.py b/scripts/neural_lab.py new file mode 100644 index 0000000..e945637 --- /dev/null +++ b/scripts/neural_lab.py @@ -0,0 +1,154 @@ +"""On-demand, bounded neural assays; no connection to the body's actuators.""" +import json +from pathlib import Path +import threading +import time + +import numpy as np + +from fetch_neural_reference import sha256 +from neural_reference import CACHE, Graph, LIF, PARAMETERS, poisson_events + +TICKS = 1500 +STIMULUS_TICKS = 1000 +SEED = 20260912 +CPU_BUDGET = .15 +WALL_LIMIT = 90. +CPU_LIMIT = 10. +SPIKE_LIMIT = 100000 +MODES = ("stimulus", "baseline", "blocked") + + +class NeuralLab: + def __init__(self, stop, directory=CACHE): + self.stop = stop + self.lock = threading.Lock() + self.thread = None + self.graph = None + self.state = {"revision": 0, "status": "unavailable", "motor_connected": False} + self.descriptor = {"available": False, "dataset": "FlyWire 630", + "motor_connected": False} + try: + report = json.loads((directory/"validation.json").read_text()) + if (report["schema"] != 1 + or report["core_sha256"] != sha256(Path(__file__).with_name("neural_reference.py")) + or report["cache_manifest_sha256"] != sha256(directory/"manifest.json")): + raise ValueError("Neural validation is stale; rerun the reference assay.") + self.graph = Graph(directory) + manifest = self.graph.manifest + self.descriptor.update({ + "available": True, "neuron_count": self.graph.size, + "edge_count": len(self.graph.indices), "input_count": len(manifest["inputs"]), + "diagram": manifest["diagram"], "readouts": manifest["readouts"], + "reference_commit": manifest["reference_commit"], + "total_ms": TICKS*PARAMETERS.dt_ms, + "stimulus_ms": STIMULUS_TICKS*PARAMETERS.dt_ms, + "rate_hz": 150, "seed": SEED, "cpu_budget": CPU_BUDGET, + "wall_limit_seconds": WALL_LIMIT, + "numerical_reference": "Brian2 2.8.0.4", + }) + self.state = {"revision": 1, "status": "idle", "motor_connected": False, + "trial": 0, "history": [], "nodes": [], "readouts": []} + except (OSError, ValueError, KeyError) as exc: + self.graph = None + message = ("Neural reference cache is missing; prepare and validate it first." + if isinstance(exc, OSError) else str(exc)) + self.descriptor["error"] = message + self.state["error"] = message + + def snapshot(self): + # Published dictionaries are replaced, never modified by the worker. + with self.lock: + return self.state + + def start(self, mode): + if not isinstance(mode, str) or mode not in MODES: + raise ValueError("Neural mode must be stimulus, baseline or blocked") + with self.lock: + if self.graph is None: + raise ValueError(self.state["error"]) + if self.thread is not None and self.thread.is_alive(): + raise ValueError("A neural trial is already running") + if self.stop.is_set(): + raise ValueError("The environment is stopping") + trial = self.state["trial"]+1 + self.state = { + "revision": self.state["revision"]+1, "status": "running", + "motor_connected": False, "trial": trial, "mode": mode, + "simulated_ms": 0, "wall_seconds": 0, "cpu_seconds": 0, + "total_spikes": 0, "input_spikes": 0, "downstream_spikes": 0, + "neurons_that_spiked": 0, "history": [], "nodes": [], "readouts": [], + } + self.thread = threading.Thread(target=self._run, args=(mode,), + name="neural-assay", daemon=True) + self.thread.start() + + def _publish(self, values): + with self.lock: + self.state = {**self.state, **values, "revision": self.state["revision"]+1} + + def _run(self, mode): + started, cpu_started = time.monotonic(), time.thread_time() + try: + graph = self.graph + inputs = graph.manifest["inputs"] + model = LIF(graph, inputs, inputs if mode == "blocked" else ()) + events = poisson_events(inputs, TICKS, rate_hz=0 if mode == "baseline" else 150, + seed=SEED, stimulus_ticks=STIMULUS_TICKS) + sensory = np.zeros(graph.size, dtype=bool) + sensory[inputs] = True + descending = np.zeros(graph.size, dtype=bool) + descending[[r["index"] for r in graph.manifest["readouts"] + if r["role"] == "descending"]] = True + total = input_total = 0 + history = [] + bins = np.zeros(3, dtype=np.int64) + next_publish = 0. + for first in range(0, TICKS, 25): + if self.stop.is_set(): + self._publish({"status": "cancelled"}) + return + for external in events[first:first+25]: + spikes = model.step(external) + n_input = int(np.count_nonzero(sensory[spikes])) + n_descending = int(np.count_nonzero(descending[spikes])) + total += len(spikes) + input_total += n_input + bins += (n_input, len(spikes)-n_input, n_descending) + if model.tick % 50 == 0: + history.append({"ms": model.tick*PARAMETERS.dt_ms, + "input": int(bins[0]), "network": int(bins[1]), + "descending": int(bins[2])}) + bins[:] = 0 + cpu_used = time.thread_time()-cpu_started + wall_used = time.monotonic()-started + if total > SPIKE_LIMIT or cpu_used > CPU_LIMIT or wall_used > WALL_LIMIT: + raise RuntimeError("Trial stopped at its resource limit; results are incomplete.") + # Duty-cycle one worker; unused neurons and all graph files stay + # shared/read-only. No continuous assay runs while the UI is idle. + if self.stop.wait(max(0, cpu_used/CPU_BUDGET-wall_used)): + self._publish({"status": "cancelled"}) + return + now = time.monotonic() + complete = model.tick == TICKS + if now >= next_publish or complete: + nodes = [{"index": item["index"], + "spikes": int(model.spike_counts[item["index"]]), + "voltage_mv": round(float(model.v[item["index"]]), 4)} + for item in graph.manifest["diagram"]["nodes"]] + readouts = [{**item, "spikes": int(model.spike_counts[item["index"]]), + "voltage_mv": round(float(model.v[item["index"]]), 4)} + for item in graph.manifest["readouts"]] + self._publish({ + "status": "complete" if complete else "running", + "simulated_ms": model.tick*PARAMETERS.dt_ms, + "wall_seconds": round(now-started, 3), + "cpu_seconds": round(time.thread_time()-cpu_started, 4), + "total_spikes": total, "input_spikes": input_total, + "downstream_spikes": total-input_total, + "neurons_that_spiked": int(np.count_nonzero(model.spike_counts)), + "nodes": nodes, "readouts": readouts, "history": list(history), + }) + next_publish = now+.1 + except Exception as exc: + self._publish({"status": "error", "error": str(exc)}) diff --git a/scripts/neural_reference.py b/scripts/neural_reference.py new file mode 100644 index 0000000..49eb43d --- /dev/null +++ b/scripts/neural_reference.py @@ -0,0 +1,133 @@ +"""Incremental LIF reference using all published FlyWire 630 signed connections. + +Parameters/equations: Shiu & Spiller, MIT-licensed Drosophila_brain_model. +Integration is analytic, with Brian2's groups/thresholds/synapses/resets order. +This models neural responses; it has no authority over a physical actuator. +""" +from dataclasses import dataclass +import json +from pathlib import Path + +import numpy as np + +from fetch_neural_reference import sha256 + +ROOT = Path(__file__).resolve().parents[1] +CACHE = ROOT/"out/neural-reference" + + +@dataclass(frozen=True) +class Parameters: + dt_ms: float = .1 + rest_mv: float = -52. + threshold_mv: float = -45. + membrane_ms: float = 20. + synapse_ms: float = 5. + refractory_ms: float = 2.2 + delay_ms: float = 1.8 + synapse_mv: float = .275 + input_scale: float = 250. + + +PARAMETERS = Parameters() + + +class Graph: + def __init__(self, directory=CACHE): + self.manifest = json.loads((directory/"manifest.json").read_text()) + if self.manifest["schema"] != 1: + raise ValueError("Unsupported neural cache") + for name, item in self.manifest["files"].items(): + if (directory/name).stat().st_size != item["bytes"] or sha256(directory/name) != item["sha256"]: + raise ValueError(f"Neural cache checksum mismatch: {name}") + self.ids, self.indptr, self.indices, self.counts = ( + np.load(directory/name, mmap_mode="r", allow_pickle=False) + for name in ("ids.npy", "indptr.npy", "indices.npy", "counts.npy")) + self.size = len(self.ids) + if (self.size != self.manifest["neuron_count"] + or len(self.indices) != self.manifest["edge_count"] + or len(self.counts) != len(self.indices) or int(self.indptr[-1]) != len(self.indices)): + raise ValueError("Invalid neural graph dimensions") + + +class LIF: + """A trial owns dynamic state; read-only connectivity can be shared.""" + def __init__(self, graph, inputs=(), silenced=(), parameters=PARAMETERS, sparse=True): + self.graph, self.parameters = graph, parameters + n, p = graph.size, parameters + self.v = np.full(n, p.rest_mv, dtype=np.float64) + self.g = np.zeros(n, dtype=np.float64) + self.last_spike = np.full(n, -1000000, dtype=np.int64) + self.refractory = np.full(n, round(p.refractory_ms/p.dt_ms), dtype=np.int64) + self.inputs = np.asarray(inputs, dtype=np.int64) + self.refractory[self.inputs] = 0 + self.silenced = np.zeros(n, dtype=bool) + self.silenced[np.asarray(silenced, dtype=np.int64)] = True + self.delay = round(p.delay_ms/p.dt_ms) + self.pending = [np.empty(0, dtype=np.int64) for _ in range(self.delay+1)] + self.tick = 0 + self.spike_counts = np.zeros(n, dtype=np.uint32) + self.visited = np.zeros(n, dtype=bool) + self.active = np.arange(n, dtype=np.int64) if not sparse else self.inputs.copy() + self.visited[self.active] = True + self.alpha = np.exp(-p.dt_ms/p.membrane_ms) + self.beta = np.exp(-p.dt_ms/p.synapse_ms) + self.coupling = p.synapse_ms/(p.membrane_ms-p.synapse_ms)*(self.alpha-self.beta) + + def _activate(self, indices): + new = np.unique(indices[~self.visited[indices]]) + if len(new): + self.visited[new] = True + self.active = np.concatenate((self.active, new)) + + def step(self, external=()): + p, tick = self.parameters, self.tick + eligible = self.active[(tick-self.last_spike[self.active]) >= self.refractory[self.active]] + # Neurons never reached by an input are exactly at equilibrium, so omitting + # their arithmetic removes no connection, neuron or nonzero dynamic state. + self.v[eligible] = (p.rest_mv+(self.v[eligible]-p.rest_mv)*self.alpha + + self.g[eligible]*self.coupling) + self.g[eligible] *= self.beta + spikes = np.sort(eligible[self.v[eligible] > p.threshold_mv]) + self.last_spike[spikes] = tick + self.spike_counts[spikes] += 1 + slot = tick % len(self.pending) + arriving = self.pending[slot] + self.pending[slot] = np.empty(0, dtype=np.int64) + for source in arriving: + if self.silenced[source]: + continue + start, end = map(int, self.graph.indptr[source:source+2]) + targets = self.graph.indices[start:end] + receiving = (tick-self.last_spike[targets]) >= self.refractory[targets] + targets = targets[receiving] + weights = self.graph.counts[start:end][receiving] + nonzero = weights != 0 + targets, weights = targets[nonzero], weights[nonzero] + if len(targets): + np.add.at(self.g, targets, weights*p.synapse_mv) + self._activate(targets) + incoming = np.asarray(external, dtype=np.int64) + if len(incoming): + permitted = (tick-self.last_spike[incoming]) >= self.refractory[incoming] + incoming = incoming[permitted] + self.v[incoming] += p.synapse_mv*p.input_scale + self._activate(incoming) + self.v[spikes] = p.rest_mv + self.g[spikes] = 0. + self.pending[(tick+self.delay) % len(self.pending)] = spikes + self.tick += 1 + return spikes + + def advance(self, input_events): + return [self.step(event) for event in input_events] + + +def poisson_events(inputs, ticks, rate_hz=150., seed=20260912, stimulus_ticks=None): + """N=1 PoissonInput uses one Bernoulli draw per neuron per 0.1 ms step.""" + rng = np.random.default_rng(seed) + inputs = np.asarray(inputs, dtype=np.int64) + duration = ticks if stimulus_ticks is None else stimulus_ticks + return [inputs[rng.random(len(inputs)) < rate_hz*PARAMETERS.dt_ms/1000] + if tick < duration else np.empty(0, dtype=np.int64) + for tick in range(ticks)] diff --git a/scripts/passive_fly.py b/scripts/passive_fly.py new file mode 100644 index 0000000..69a5aed --- /dev/null +++ b/scripts/passive_fly.py @@ -0,0 +1,69 @@ +"""Attach one flybody anatomy with zero actuator drive and cached visual surfaces.""" +import json +from pathlib import Path + +from dm_control import mjcf +import numpy as np + +from prepare_browser_fly import CACHE, SOURCE, VERSION, digest + +FLY_ID = "fly_001" +FLY_START = [-44., -10., -7.41] + + +def attach_fly(city): + manifest_path = CACHE / "manifest.json" + if not manifest_path.is_file(): + raise RuntimeError("Prepare the bounded browser fly cache first; see docs/browser-environment.md") + manifest = json.loads(manifest_path.read_text()) + for key, path in ( + ("source_xml_sha256", SOURCE), + ("generator_sha256", Path(__file__).with_name("prepare_browser_fly.py")), + ("physics_sha256", CACHE/"physics.xml"), + ("surfaces_sha256", CACHE/"surfaces.bin"), + ): + if manifest[key] != digest(path): + raise RuntimeError(f"Browser fly cache mismatch: {key}; rebuild the cache") + if manifest["version"] != VERSION: + raise RuntimeError("Unsupported browser fly cache version") + anatomy = mjcf.from_path(str(CACHE/"physics.xml")) + anatomy.model = FLY_ID + anatomy.find("joint", "free").remove() + frame = city.mjcf_model.attach(anatomy) + frame.pos = FLY_START + free = frame.add("freejoint") + city.mjcf_model.compiler.boundmass = 0. + city.mjcf_model.compiler.boundinertia = 0. + # Walking contact parameters from flybody's Walking task, on the host surface. + city.mjcf_model.find("geom", "bridge_0").solref = [.001, 1] + city.mjcf_model.find("geom", "bridge_0").solimp = [.95, .99, .01] + return manifest, free.full_identifier + + +def initialize_fly(model, data): + # Fold the wings using their anatomical spring reference at initialization. + for joint in range(model.njnt): + if model.joint(joint).name.startswith(FLY_ID+"/wing_"): + address = int(model.jnt_qposadr[joint]) + data.qpos[address] = model.qpos_spring[address] + + +def describe_fly(model, manifest): + names = list(manifest["inertias"]) + body_ids = np.array([model.body(f"{FLY_ID}/{name}").id for name in names], dtype=np.int32) + body_index = {name: i for i, name in enumerate(names)} + surfaces = [ + {**geom, "body": body_index[geom["body"]]} for geom in manifest["visual_geoms"] + ] + return { + "id": FLY_ID, "label": "Fly 001", "source": "flybody", + "neural_controller": "disconnected", "actuator_drive": "disabled", + "bodies": names, "meshes": manifest["meshes"], "geoms": surfaces, + "surface_url": "/assets/fly-surfaces.bin", + "surface_bytes": manifest["surface_bytes"], + "surface_sha256": manifest["surfaces_sha256"], + "visual_triangles": manifest["visual_faces"], + "mass_mg": manifest["mass_grams"]*1000, + "max_visual_displacement_um": manifest["max_vertex_displacement_cm"]*10000, + "focus_body": body_index["thorax"], + }, body_ids diff --git a/scripts/prepare_browser_fly.py b/scripts/prepare_browser_fly.py new file mode 100644 index 0000000..eaad0c3 --- /dev/null +++ b/scripts/prepare_browser_fly.py @@ -0,0 +1,233 @@ +"""Build a bounded-memory flybody physics/visual cache, one source geom at a time. + +Mass properties are computed by MuJoCo from original, full-resolution geoms. +Only the separately rendered, non-colliding surfaces are vertex-clustered. +""" +import argparse +import copy +import gc +import hashlib +import json +from pathlib import Path +import resource +import time +import xml.etree.ElementTree as ET + +import mujoco +import numpy as np + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / "flybody/flybody/fruitfly/assets/fruitfly.xml" +CACHE = ROOT / "out/browser-fly" +VERSION = 1 + + +def digest(path): + result = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1024*1024), b""): + result.update(chunk) + return result.hexdigest() + + +def matrix(quat): + result = np.empty(9) + mujoco.mju_quat2Mat(result, quat) + return result.reshape(3, 3) + + +def text(values): + return " ".join(format(float(value), ".17g") for value in np.atleast_1d(values)) + + +def cluster(vertices, faces, target=1000): + """Bound vertex displacement explicitly; retain distinct material surfaces.""" + if len(faces) <= target: + return vertices.copy(), faces.copy(), 0. + cell = .00025 # centimetres: start at 2.5 micrometres. + low = vertices.min(axis=0) + while True: + keys = np.floor((vertices-low)/cell).astype(np.int32) + _, inverse = np.unique(keys, axis=0, return_inverse=True) + counts = np.bincount(inverse) + points = np.column_stack([ + np.bincount(inverse, weights=vertices[:, axis])/counts for axis in range(3) + ]).astype(np.float32) + triangles = inverse[faces] + keep = ((triangles[:, 0] != triangles[:, 1]) + & (triangles[:, 0] != triangles[:, 2]) + & (triangles[:, 1] != triangles[:, 2])) + triangles = triangles[keep] + _, unique = np.unique(np.sort(triangles, axis=1), axis=0, return_index=True) + triangles = triangles[np.sort(unique)] + error = float(np.linalg.norm(vertices-points[inverse], axis=1).max()) + if len(triangles) <= target or cell >= .0015: + if not len(triangles) or error > .0027: + raise ValueError("Visual simplification exceeded its 27 µm displacement bound") + used, remap = np.unique(triangles, return_inverse=True) + return points[used], remap.reshape(-1, 3).astype(np.int32), error + cell *= 1.35 + + +def defaults_by_class(root): + result = {} + + def visit(node, inherited): + current = dict(inherited) + geom = node.find("geom") + if geom is not None: + current.update(geom.attrib) + result[node.get("class", "")] = current + for child in node.findall("default"): + visit(child, current) + visit(root.find("default"), {}) + return result + + +def prepare(destination=CACHE): + destination.mkdir(parents=True, exist_ok=True) + document = ET.parse(SOURCE) + root = document.getroot() + meshes = {mesh.get("name"): mesh for mesh in root.findall("asset/mesh")} + materials = root.findall("asset/material") + inherited = defaults_by_class(root) + manifest = { + "version": VERSION, "generator_sha256": digest(__file__), + "source_xml_sha256": digest(SOURCE), "source_meshes": {}, + "visual_geoms": [], "inertias": {}, "meshes": [], + "original_faces": 0, "visual_faces": 0, "max_vertex_displacement_cm": 0., + } + # An incomplete build is never advertised as usable. + output = destination / "surfaces.bin" + started = time.monotonic() + count = 0 + with output.open("wb") as binary: + def visit(body, childclass="body"): + nonlocal count + childclass = body.get("childclass", childclass) + properties = [] + for geom in list(body.findall("geom")): + effective = dict(inherited[geom.get("class", childclass)]) + effective.update(geom.attrib) + massless = (float(effective.get("mass", "-1")) == 0 + or ("mass" not in effective and float(effective.get("density", "1000")) == 0)) + mesh_name = effective.get("mesh") + if massless and mesh_name is None: + continue + mini = ET.Element("mujoco") + compiler = copy.deepcopy(root.find("compiler")) + compiler.set("meshdir", str(SOURCE.parent)) + mini.append(compiler) + mini.append(copy.deepcopy(root.find("default"))) + asset = ET.SubElement(mini, "asset") + for material in materials: + asset.append(copy.deepcopy(material)) + if mesh_name: + asset.append(copy.deepcopy(meshes[mesh_name])) + source_mesh = SOURCE.parent/meshes[mesh_name].get("file") + manifest["source_meshes"][source_mesh.name] = digest(source_mesh) + sample = ET.SubElement(ET.SubElement(mini, "worldbody"), "body", + name="sample", childclass=childclass) + sample.append(copy.deepcopy(geom)) + # Static sample bodies allow zero-mass cosmetic geoms. + model = mujoco.MjModel.from_xml_string(ET.tostring(mini, encoding="unicode")) + if not massless: + mass = float(model.body_mass[1]) + com = model.body_ipos[1].copy() + rotation = matrix(model.body_iquat[1]) + inertia = rotation @ np.diag(model.body_inertia[1]) @ rotation.T + properties.append((mass, com, inertia)) + if mesh_name: + if int(model.geom_contype[0]) or int(model.geom_conaffinity[0]): + raise ValueError("Refusing to simplify collision geometry") + # Native compiled mesh coordinates include centering/alignment. + vertices, faces, error = cluster(model.mesh_vert, model.mesh_face) + vertices = (vertices @ matrix(model.geom_quat[0]).T + + model.geom_pos[0]).astype("= 0 else model.geom_rgba[0]) + manifest["visual_geoms"].append({ + "name": geom.get("name"), "body": body.get("name"), + "mesh": mesh_index, "rgba": rgba.tolist(), + "material": model.material(material_id).name if material_id >= 0 else "body", + }) + manifest["original_faces"] += int(model.nmeshface) + manifest["visual_faces"] += len(faces) + manifest["max_vertex_displacement_cm"] = max( + manifest["max_vertex_displacement_cm"], error) + body.remove(geom) + del model + gc.collect() + count += 1 + if count % 15 == 0: + print(f"Prepared {count} anatomy geoms; " + f"peak RSS {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024:.0f} MiB", + flush=True) + if properties: + total = sum(p[0] for p in properties) + com = sum(p[0]*p[1] for p in properties)/total + inertia = np.zeros((3, 3)) + for mass, center, tensor in properties: + delta = center-com + inertia += tensor+mass*(np.dot(delta, delta)*np.eye(3)-np.outer(delta, delta)) + # MuJoCo's fullinertia eigensolver uses an absolute tolerance + # that is too coarse for the fly's ~1e-13 g·cm² foot inertias. + # Store principal axes explicitly to avoid a second lossy solve. + diagonal, axes = np.linalg.eigh(inertia) + if np.linalg.det(axes) < 0: + axes[:, 0] *= -1 + quaternion = np.empty(4) + mujoco.mju_mat2Quat(quaternion, axes.ravel()) + ET.SubElement(body, "inertial", pos=text(com), mass=text(total), + quat=text(quaternion), diaginertia=text(diagonal)) + manifest["inertias"][body.get("name")] = { + "mass": total, "com": com.tolist(), "tensor": inertia.tolist(), + } + for child in body.findall("body"): + visit(child, childclass) + for body in root.findall("worldbody/body"): + visit(body) + for mesh in list(root.findall("asset/mesh")): + root.find("asset").remove(mesh) + physical = destination / "physics.xml" + ET.indent(document) + document.write(physical, encoding="unicode") + # Independent recompilation must reproduce the unsimplified mass properties. + model = mujoco.MjModel.from_xml_path(str(physical)) + for name, original in manifest["inertias"].items(): + body = model.body(name) + rotation = matrix(body.iquat) + np.testing.assert_allclose(body.mass, original["mass"], rtol=1e-10) + np.testing.assert_allclose(body.ipos, original["com"], rtol=1e-10, atol=1e-12) + tensor = np.asarray(original["tensor"]) + difference = rotation @ np.diag(body.inertia) @ rotation.T-tensor + if np.linalg.norm(difference) > 1e-10*np.linalg.norm(tensor): + raise ValueError(f"Inertia changed for {name}") + manifest.update( + mass_grams=float(model.body_mass.sum()), bodies=int(model.nbody)-1, + joints=int(model.njnt), physics_sha256=digest(physical), surfaces_sha256=digest(output), + surface_bytes=output.stat().st_size, seconds=time.monotonic()-started, + peak_rss_mib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024, + ) + temporary = destination / "manifest.pending.json" + temporary.write_text(json.dumps(manifest, separators=(",", ":"))) + temporary.replace(destination / "manifest.json") + print(json.dumps({key: manifest[key] for key in ( + "bodies", "joints", "mass_grams", "original_faces", "visual_faces", + "surface_bytes", "max_vertex_displacement_cm", "peak_rss_mib", "seconds")}), flush=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, default=CACHE) + prepare(parser.parse_args().output) diff --git a/scripts/prepare_neural_reference.py b/scripts/prepare_neural_reference.py new file mode 100644 index 0000000..5a4ea4c --- /dev/null +++ b/scripts/prepare_neural_reference.py @@ -0,0 +1,144 @@ +"""Stream the entire published FlyWire 630 graph into compact immutable CSR files.""" +import argparse +import csv +import json +from pathlib import Path +import resource +import time + +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq + +from fetch_neural_reference import COMMIT, antennal_identifiers, sha256 + +COLUMNS = ["Presynaptic_Index", "Postsynaptic_Index", "Excitatory x Connectivity"] + + +def prepare(source, destination): + started = time.monotonic() + destination.mkdir(parents=True, exist_ok=True) + provenance = json.loads((source/"sources.json").read_text()) + if provenance["reference_commit"] != COMMIT: + raise ValueError("Unexpected reference commit") + for item in provenance["files"].values(): + if sha256(source/item["local_name"]) != item["sha256"]: + raise ValueError("Reference data checksum mismatch") + with (source/"2023_03_23_completeness_630_final.csv").open() as stream: + reader = csv.reader(stream) + next(reader) + ids = np.fromiter((int(row[0]) for row in reader), dtype=np.uint64) + count = len(ids) + if not 100000 < count < 200000 or len(np.unique(ids)) != count: + raise ValueError("Unexpected neuron universe") + id_to_index = {int(value): i for i, value in enumerate(ids)} + protocol = antennal_identifiers(source/"figures.ipynb") + groups = {key: [id_to_index[x] for x in protocol[key]] + for key in ("neu_JON_CE", "neu_JON_F", "neu_JON_D_m")} + readouts = [ + {"index": id_to_index[protocol[key]], "id": str(protocol[key]), "label": label, "role": role} + for key, label, role in [ + ("id_aBN1", "aBN1", "interneuron"), + ("id_DN1_1", "aDN1", "descending"), + ("id_DN2_l", "aDN2", "descending"), + ] + ] + inputs = sorted({x for values in groups.values() for x in values}) + pa.set_cpu_count(1) + pa.set_io_thread_count(1) + parquet = pq.ParquetFile(source/"2023_03_23_connectivity_630_final.parquet") + edges = parquet.metadata.num_rows + if edges > 20000000: + raise ValueError("Graph exceeds the 20-million-edge preparation ceiling") + counts = np.zeros(count, dtype=np.uint64) + incoming = np.zeros(count, dtype=np.float64) + targets = np.array([item["index"] for item in readouts]) + signed_sum = absolute_sum = 0 + maximum = 0 + for batch in parquet.iter_batches(batch_size=65536, columns=COLUMNS, use_threads=False): + pre, post, weight = [column.to_numpy() for column in batch.columns] + if pre.min() < 0 or post.min() < 0 or max(pre.max(), post.max()) >= count: + raise ValueError("Connection index outside the published neuron universe") + maximum = max(maximum, int(np.abs(weight).max())) + if maximum >= 2**31: + raise ValueError("Signed synapse counts do not fit int32") + counts += np.bincount(pre, minlength=count).astype(np.uint64) + mask = np.isin(post, targets) + np.add.at(incoming, pre[mask], np.abs(weight[mask])) + signed_sum += int(weight.sum()) + absolute_sum += int(np.abs(weight).sum()) + pointers = np.concatenate((np.array([0], dtype=np.uint64), np.cumsum(counts))) + if int(pointers[-1]) != edges: + raise ValueError("Connection count changed during conversion") + np.save(destination/"ids.npy", ids, allow_pickle=False) + np.save(destination/"indptr.npy", pointers, allow_pickle=False) + post_out = np.lib.format.open_memmap(destination/"indices.npy", mode="w+", dtype="= 0), + "contact_seen": self.fly_contacts_seen, + "last_contact_count": self.fly_contact_count, + }, + "probe": { + "active": self.probe_active, "generation": self.probe_generation, + "position": self.data.xpos[self.probe_id].tolist(), + "contact_seen": self.contact_seen, + }, + } + if pose_revision != self.fly_pose_revision: + state["fly"]["poses_b64"] = self.fly_pose_bytes + if trail_version != (self.probe_generation, len(self.probe_trail)): + state["probe"]["trail"] = list(self.probe_trail) + state["probe"]["trail_count"] = len(self.probe_trail) + if self.neural: + neural = self.neural.snapshot() + state["neural_revision"] = neural["revision"] + state["neural_running"] = neural["status"] == "running" + if neural_revision != neural["revision"]: + state["neural"] = neural + return state + + def _run(self): + previous = time.monotonic() + measured_wall, measured_sim = previous, self.data.time + accumulator = 0. + wait = .01 + try: + while not self.stop.wait(wait): + now = time.monotonic() + cpu_started = time.thread_time() + elapsed, previous = now-previous, now + with self.lock: + if self.paused: + accumulator = 0. + else: + # Never skip physical time or accumulate an unbounded catch-up queue. + accumulator = min(accumulator+elapsed*self.speed, .025) + steps = min(int(accumulator/self.model.opt.timestep), 250) + remaining = steps + while remaining: + block = min(10, remaining) if self.data.ntree_awake else remaining + mujoco.mj_step(self.model, self.data, nstep=block) + remaining -= block + self._sample_contacts() + if self.probe_active and len(self.probe_trail) < 500: + p = self.data.xpos[self.probe_id] + if np.linalg.norm(p-self.probe_trail[-1]) > .15: + self.probe_trail.append(p.tolist()) + # Release the lock frequently enough to serve poses + # and commands while a costly articulated body settles. + if time.thread_time()-cpu_started >= PHYSICS_SLICE_CPU_SECONDS: + break + accumulator -= (steps-remaining)*self.model.opt.timestep + if not np.isfinite(self.data.qpos).all(): + raise RuntimeError("Non-finite physical state") + if self.data.warning.number.any(): + raise RuntimeError("Physics warning; simulation stopped") + if now >= self.next_pose_time: + mujoco.mj_kinematics(self.model, self.data) + self._cache_fly_pose() + self.next_pose_time = now + (POSE_INTERVAL if self.data.ntree_awake else 1) + self.sequence += 1 + if now-measured_wall >= 1: + self.ratio = (self.data.time-measured_sim)/(now-measured_wall) + measured_wall, measured_sim = now, self.data.time + awake = bool(self.data.ntree_awake) and not self.paused + # Enforce the worker's duty budget even if host quota accounting + # is delayed. Slow physical time rather than growing a backlog. + cpu_used = time.thread_time()-cpu_started + wait = max(.001 if awake else .01, + cpu_used/PHYSICS_CPU_BUDGET-(time.monotonic()-now)) + except Exception as exc: + with self.lock: + self.error = str(exc) + self.paused = True + + +class Server(ThreadingHTTPServer): + daemon_threads = True + allow_reuse_address = True + + def __init__(self, port, environment): + self.environment = environment + self.stream_slots = threading.BoundedSemaphore(8) + self.allowed_hosts = {f"localhost:{port}", f"127.0.0.1:{port}"} + super().__init__(("0.0.0.0", port), Handler) + + +class Handler(BaseHTTPRequestHandler): + """Allowlisted local assets and narrow simulation commands; no file browser.""" + + def setup(self): + super().setup() + self.connection.settimeout(10) + + def valid_host(self): + if self.headers.get("Host") not in self.server.allowed_hosts: + self.send_error(403) + return False + return True + + def respond(self, payload, content_type="application/json", status=200): + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(payload))) + self.send_header("Cache-Control", "no-store") + self.send_header("X-Content-Type-Options", "nosniff") + self.send_header("Referrer-Policy", "no-referrer") + self.end_headers() + try: + self.wfile.write(payload) + except (BrokenPipeError, ConnectionResetError): + pass + + def do_GET(self): + if not self.valid_host(): + return + path = urlsplit(self.path).path + env = self.server.environment + if path == "/api/dev-version" and DEV_MODE: + files = sorted((ROOT/"web").glob("*")) + revision = hashlib.sha256("".join( + f"{p.name}:{p.stat().st_mtime_ns}:{p.stat().st_size}" + for p in files if p.is_file()).encode()).hexdigest() + return self.respond(json.dumps({"instance": DEV_INSTANCE, "assets": revision}).encode()) + if path == "/api/world": + return self.respond(env.world_bytes) + if path in ("/api/state", "/health"): + state = env.snapshot() + return self.respond(json.dumps(state).encode(), status=503 if state["error"] else 200) + if path == "/api/events": + if not self.server.stream_slots.acquire(blocking=False): + return self.send_error(503, "Viewer limit reached") + try: + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-store") + self.end_headers() + pose_revision = trail_version = neural_revision = None + while not env.stop.is_set(): + state = env.snapshot(pose_revision, trail_version, neural_revision) + payload = json.dumps(state, separators=(",", ":")) + self.wfile.write(f"data: {payload}\n\n".encode()) + self.wfile.flush() + pose_revision = state["fly"]["pose_revision"] + trail_version = (state["probe"]["generation"], state["probe"]["trail_count"]) + neural_revision = state.get("neural_revision") + active = not state["fly"]["sleeping"] or (state["probe"]["active"] and not state["probe"]["contact_seen"]) + interval = POSE_INTERVAL if active and not state["paused"] else 1 + env.stop.wait(min(interval, .1) if state.get("neural_running") else interval) + except (BrokenPipeError, ConnectionResetError, TimeoutError): + pass + finally: + self.server.stream_slots.release() + return + if path == "/assets/fly-surfaces.bin": + compressed = "gzip" in self.headers.get("Accept-Encoding", "") + payload = env.fly_surfaces_gzip if compressed else env.fly_surfaces + self.send_response(200) + self.send_header("Content-Type", "application/octet-stream") + self.send_header("Content-Length", str(len(payload))) + self.send_header("Vary", "Accept-Encoding") + self.send_header("Cache-Control", "no-store") + if compressed: + self.send_header("Content-Encoding", "gzip") + self.end_headers() + try: + self.wfile.write(payload) + except (BrokenPipeError, ConnectionResetError): + pass + return + textures = {f"/textures/{i}.png": data for i, data in env.textures.items()} + if path in textures: + return self.respond(textures[path], "image/png") + assets = { + "/": ("web/index.html", "text/html; charset=utf-8"), + "/environment.js": ("web/environment.js", "text/javascript; charset=utf-8"), + "/fly.js": ("web/fly.js", "text/javascript; charset=utf-8"), + "/neural.js": ("web/neural.js", "text/javascript; charset=utf-8"), + "/neural.css": ("web/neural.css", "text/css; charset=utf-8"), + "/dev-reload.js": ("web/dev-reload.js", "text/javascript; charset=utf-8"), + "/style.css": ("web/style.css", "text/css; charset=utf-8"), + "/favicon.svg": ("web/favicon.svg", "image/svg+xml"), + "/vendor/three.module.js": ("node_modules/three/build/three.module.js", "text/javascript"), + "/vendor/three.core.js": ("node_modules/three/build/three.core.js", "text/javascript"), + "/vendor/LICENSE": ("node_modules/three/LICENSE", "text/plain"), + } + if path not in assets: + return self.send_error(404) + filename, content_type = assets[path] + try: + payload = (ROOT/filename).read_bytes() + if path == "/" and DEV_MODE: + payload = payload.replace(b"", b'') + self.respond(payload, content_type) + except FileNotFoundError: + self.send_error(503, "Browser assets unavailable; install locked dependencies first") + + def do_POST(self): + if not self.valid_host(): + return + host = self.headers.get("Host") + if (self.headers.get("Origin") != f"http://{host}" + or self.headers.get("Content-Type") != "application/json"): + return self.send_error(403) + if self.path != "/api/command": + return self.send_error(404) + try: + length = int(self.headers.get("Content-Length", "0")) + if not 0 < length <= 1024: + raise ValueError("Invalid request length") + command = json.loads(self.rfile.read(length)) + self.server.environment.command(command) + except (ValueError, UnicodeDecodeError) as exc: + return self.respond(json.dumps({"error": str(exc)}).encode(), status=400) + self.respond(json.dumps(self.server.environment.snapshot()).encode()) + + def log_message(self, fmt, *args): + # No per-frame logs; requests remain visible in browser network tooling. + if args and str(args[1]) not in ("200", "304"): + super().log_message(fmt, *args) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--port", type=int, default=8089) + args = parser.parse_args() + if not (ROOT/"node_modules/three/build/three.module.js").is_file(): + raise SystemExit("Missing browser dependencies. Run the guarded pnpm install first.") + environment = Environment() + server = Server(args.port, environment) + environment.start() + print(f"NETSPHERE ready on port {args.port}: " + f"{len(environment.world['geoms'])} world geoms; no neural controller.", flush=True) + if environment.neural: + print(f"Neural reference lab: {environment.neural.snapshot()['status']}; " + "body actuators disconnected.", flush=True) + try: + server.serve_forever() + finally: + environment.stop.set() + server.server_close() + + +if __name__ == "__main__": + main() diff --git a/scripts/test_environment.py b/scripts/test_environment.py new file mode 100644 index 0000000..3aced11 --- /dev/null +++ b/scripts/test_environment.py @@ -0,0 +1,167 @@ +"""Physical and transport checks for one disconnected flybody animal.""" +import copy +import unittest +import xml.etree.ElementTree as ET + +import mujoco +import numpy as np + +from serve_environment import Environment, PROBE_RADIUS, PROBE_START +from passive_fly import initialize_fly +from prepare_browser_fly import CACHE, SOURCE + + +class EnvironmentPhysicsTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.env = Environment() + + def setUp(self): + mujoco.mj_resetData(self.env.model, self.env.data) + initialize_fly(self.env.model, self.env.data) + self.env.fly_contacts_seen = False + self.env.command({"action": "drop"}) + + def advance(self, seconds): + for _ in range(round(seconds/self.env.model.opt.timestep/10)): + mujoco.mj_step(self.env.model, self.env.data, nstep=10) + self.env._sample_contacts() + + def test_free_fall_matches_centimetre_gravity(self): + # Compare with the analytic trajectory, allowing one integration step's error. + model, data = self.env.model, self.env.data + start_time = data.time + # Isolate gravity analytically. Runtime retains the source air density/ + # viscosity; the other contact/settling checks use those air parameters. + density, viscosity = model.opt.density, model.opt.viscosity + try: + model.opt.density = model.opt.viscosity = 0. + mujoco.mj_step(model, data, nstep=round(.1/model.opt.timestep)) + finally: + model.opt.density, model.opt.viscosity = density, viscosity + dt = data.time-start_time + expected = PROBE_START[2] - .5 * 981 * dt**2 + self.assertAlmostEqual(data.qpos[self.env.probe_qpos+2], expected, delta=981*model.opt.timestep*dt*.55) + self.assertAlmostEqual(data.qvel[self.env.probe_dof+2], -981*dt, places=7) + self.assertFalse(np.any(data.contact.geom[:data.ncon] == self.env.probe_geom)) + + def test_probe_lands_on_existing_bridge(self): + model, data = self.env.model, self.env.data + touched = set() + for _ in range(round(.8/model.opt.timestep/10)): + mujoco.mj_step(model, data, nstep=10) + for contact in data.contact[:data.ncon]: + if self.env.probe_geom in contact.geom: + touched.update(model.geom(int(g)).name for g in contact.geom) + self.assertIn("bridge_0", touched) + self.assertAlmostEqual(data.qpos[self.env.probe_qpos+2], -8 + .45 + PROBE_RADIUS, delta=.025) + self.assertLess(np.linalg.norm(data.qvel[self.env.probe_dof:self.env.probe_dof+3]), .01) + self.assertGreaterEqual(data.tree_asleep[model.body_treeid[self.env.probe_id]], 0) + # A second release must wake a sleeping particle naturally. + self.env.command({"action": "drop"}) + mujoco.mj_step(model, data, nstep=100) + self.assertLess(data.qpos[self.env.probe_qpos+2], PROBE_START[2]-.04) + + def test_drop_preserves_global_clock(self): + self.advance(.15) + self.env.data.time = 123.456 + q, v = self.env.probe_qpos, self.env.probe_dof + fly_qpos = self.env.data.qpos[:q].copy() + fly_qvel = self.env.data.qvel[:v].copy() + self.env.command({"action": "drop"}) + self.assertEqual(self.env.data.time, 123.456) + np.testing.assert_allclose(self.env.data.qpos[q:q+3], PROBE_START) + np.testing.assert_array_equal(self.env.data.qpos[:q], fly_qpos) + np.testing.assert_array_equal(self.env.data.qvel[:v], fly_qvel) + + def test_commands_reject_unbounded_or_ambiguous_values(self): + for command in [[], {"action": "speed", "value": True}, + {"action": "speed", "value": 100}, + {"action": "pause", "paused": "false"}, + {"action": "drop", "position": [0, 0, 0]}]: + with self.subTest(command=command), self.assertRaises(ValueError): + self.env.command(command) + + def test_passive_anatomy_and_collision_contract(self): + world = self.env.world + self.assertIsNone(world["neural_controller"]) + self.assertEqual(world["fly_body"]["neural_controller"], "disconnected") + self.assertEqual(len(world["fly_body"]["bodies"]), 67) + self.assertEqual(len(world["geoms"]), 383) + self.assertTrue(all(geom["collision"] for geom in world["geoms"])) + for view in world["views"]: + self.assertGreater(float(self.env.city.clearance(np.array([view["position"]]))[0]), .3) + # Cache reduction must leave joint, actuator, contact and passive contracts intact. + original, cached = ET.parse(SOURCE), ET.parse(CACHE/"physics.xml") + for tag in ["joint", "tendon", "actuator", "contact", "sensor"]: + # XML indentation is presentation, so compare trees without whitespace. + def canonical(tree): + node = copy.deepcopy(tree) + for child in node.iter(): + child.text = child.tail = None + return ET.tostring(node) + self.assertEqual([canonical(x) for x in original.iter(tag)], + [canonical(x) for x in cached.iter(tag)], tag) + original_geoms = {g.get("name"): g.attrib for g in original.iter("geom") if g.get("name") and not g.get("mesh")} + cached_geoms = {g.get("name"): g.attrib for g in cached.iter("geom") if g.get("name")} + self.assertEqual(original_geoms, cached_geoms) + self.advance(.8) + self.assertTrue(self.env.fly_contacts_seen) + self.assertTrue(np.isfinite(self.env.data.qpos).all()) + self.assertFalse(self.env.data.warning.number.any()) + self.assertFalse(self.env.data.actuator_force.any()) + self.assertGreater(self.env.data.xpos[self.env.fly_root_id, 2], -7.55) + self.assertLess(self.env.data.xpos[self.env.fly_root_id, 2], -7.4) + self.assertLess(self.env.fly_sleep_radius, .5) + self.assertGreater(self.env.fly_sleep_radius, .1) + + def test_native_sleep_bounds_drift_and_external_force_wakes_body(self): + model, data = self.env.model, self.env.data + for _ in range(60): + self.advance(.1) + if data.tree_asleep[self.env.fly_tree] >= 0: + break + self.assertGreaterEqual(data.tree_asleep[self.env.fly_tree], 0) + mujoco.mj_kinematics(model, data) + resting = data.xpos[self.env.fly_body_ids].copy() + position = data.qpos.copy() + # Compare against uninterrupted awake dynamics for 200 ms after settling. + flags = model.opt.enableflags + try: + model.opt.enableflags &= ~int(mujoco.mjtEnableBit.mjENBL_SLEEP) + reference = mujoco.MjData(model) + reference.qpos[:] = position + mujoco.mj_forward(model, reference) + mujoco.mj_step(model, reference, nstep=2000) + mujoco.mj_kinematics(model, reference) + drift = np.linalg.norm(reference.xpos[self.env.fly_body_ids]-resting, axis=1).max() + print(f"\nSleep: {data.time:.3f} simulated s; awake reference drift: {drift*10000:.3f} µm", + flush=True) + # This passive viewer accepts native sleep error below the measured + # visual reduction displacement, not sub-micrometre biomechanics. + self.assertLess(drift*10000, self.env.fly_description["max_visual_displacement_um"]) + finally: + model.opt.enableflags = flags + mujoco.mj_step(model, data, nstep=2000) + np.testing.assert_array_equal(data.qpos, position) + data.xfrc_applied[self.env.fly_root_id, 0] = .1 + mujoco.mj_step(model, data, nstep=100) + self.assertLess(data.tree_asleep[self.env.fly_tree], 0) + self.assertGreater(np.linalg.norm(data.qvel[:self.env.probe_dof]), 0) + self.assertFalse(data.warning.number.any()) + + def test_unchanged_poses_and_trail_are_not_retransmitted(self): + self.env._cache_fly_pose() + first = self.env.snapshot() + revision = first["fly"]["pose_revision"] + trail_version = (first["probe"]["generation"], first["probe"]["trail_count"]) + self.env._cache_fly_pose() + self.assertEqual(revision, self.env.fly_pose_revision) + delta = self.env.snapshot(revision, trail_version) + self.assertNotIn("poses_b64", delta["fly"]) + self.assertNotIn("trail", delta["probe"]) + self.assertIn("poses_b64", self.env.snapshot()["fly"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validate_neural_reference.py b/scripts/validate_neural_reference.py new file mode 100644 index 0000000..9272d41 --- /dev/null +++ b/scripts/validate_neural_reference.py @@ -0,0 +1,169 @@ +"""Bounded numerical comparison against Brian2 and real-graph intervention trials.""" +import argparse +import json +from pathlib import Path +import resource +import time +from types import SimpleNamespace + +import numpy as np + +from fetch_neural_reference import sha256 +from neural_reference import CACHE, Graph, LIF, PARAMETERS, poisson_events + + +def fixture_graph(n=12): + pre = np.array([0, 0, 1, 1, 2, 3, 3, 4, 5, 6, 7, 9], dtype=np.int64) + post = np.array([2, 3, 2, 4, 5, 2, 6, 5, 6, 7, 2, 10], dtype=np.uint32) + weights = np.array([90, 150, 120, 80, 190, -160, -60, 180, 140, 120, -95, 110], dtype=np.int32) + return SimpleNamespace(size=n, ids=np.arange(n), indices=post, counts=weights, + indptr=np.concatenate(([0], np.cumsum(np.bincount(pre, minlength=n))))) + + +def brian_reference(graph, events, inputs, silenced): + import brian2 as b + b.prefs.codegen.target = "numpy" + p = PARAMETERS + clock = b.Clock(dt=p.dt_ms*b.ms) + neurons = b.NeuronGroup( + graph.size, + """dv/dt = (-52*mV - v + g)/(20*ms) : volt (unless refractory) + dg/dt = -g/(5*ms) : volt (unless refractory) + rfc : second""", + threshold="v > -45*mV", reset="v = -52*mV; g = 0*mV", + refractory="rfc", method="linear", clock=clock) + neurons.v = -52*b.mV + neurons.g = 0*b.mV + neurons.rfc = 2.2*b.ms + neurons.rfc[np.array(inputs, dtype=np.int64)] = 0*b.ms + pre = np.repeat(np.arange(graph.size), np.diff(graph.indptr).astype(np.int64)) + synapses = b.Synapses(neurons, neurons, "w : volt", on_pre="g += w", + delay=1.8*b.ms, clock=clock) + synapses.connect(i=pre, j=np.asarray(graph.indices, dtype=np.int32)) + weights = graph.counts.astype(np.float64)*.275 + weights[np.isin(pre, silenced)] = 0 + synapses.w = weights*b.mV + table = np.zeros((len(events), graph.size)) + for tick, indices in enumerate(events): + table[tick, indices] = 1 + neurons.namespace["injected"] = b.TimedArray(table, dt=clock.dt) + drive = neurons.run_regularly("v += injected(t, i)*68.75*mV", + when="synapses", order=0) + states = b.StateMonitor(neurons, ["v", "g"], record=True, when="end", clock=clock) + spikes = b.SpikeMonitor(neurons) + network = b.Network(neurons, synapses, drive, states, spikes) + network.run(len(events)*clock.dt) + return (np.asarray(states.v/b.mV).T, np.asarray(states.g/b.mV).T, + np.rint(np.asarray(spikes.t/b.ms)/p.dt_ms).astype(int), + np.asarray(spikes.i).astype(int), b.__version__) + + +def compare(graph, events, inputs, silenced=()): + ref_v, ref_g, ref_t, ref_i, version = brian_reference(graph, events, inputs, silenced) + model = LIF(graph, inputs, silenced) + vs, gs, ticks, indices = [], [], [], [] + for tick, external in enumerate(events): + spikes = model.step(external) + vs.append(model.v.copy()) + gs.append(model.g.copy()) + ticks.extend([tick]*len(spikes)) + indices.extend(spikes) + error_v = float(np.max(np.abs(ref_v-np.array(vs)))) + error_g = float(np.max(np.abs(ref_g-np.array(gs)))) + np.testing.assert_allclose(vs, ref_v, rtol=0, atol=1e-8) + np.testing.assert_allclose(gs, ref_g, rtol=0, atol=1e-8) + np.testing.assert_array_equal(ticks, ref_t) + np.testing.assert_array_equal(indices, ref_i) + # Incremental API boundaries must not reset refractoriness or delayed spikes. + continued = LIF(graph, inputs, silenced) + for chunk in np.array_split(np.arange(len(events)), 7): + continued.advance([events[tick] for tick in chunk]) + np.testing.assert_array_equal(continued.v, model.v) + np.testing.assert_array_equal(continued.g, model.g) + np.testing.assert_array_equal(continued.spike_counts, model.spike_counts) + dense = LIF(graph, inputs, silenced, sparse=False) + dense.advance(events) + np.testing.assert_array_equal(dense.v, model.v) + np.testing.assert_array_equal(dense.g, model.g) + return {"brian2": version, "steps": len(events), "spikes": len(ticks), + "max_v_error_mv": error_v, "max_g_error_mv": error_g, + "spike_times_identical": True, "chunked_and_dense_identical": True} + + +def full_trial(graph, mode, ticks=1500): + inputs = graph.manifest["inputs"] + model = LIF(graph, inputs, inputs if mode == "blocked" else ()) + events = poisson_events(inputs, ticks, rate_hz=0 if mode == "baseline" else 150, + stimulus_ticks=1000) + start = time.monotonic() + cpu = time.process_time() + for external in events: + model.step(external) + if time.monotonic()-start > 90: + raise RuntimeError("Reference trial exceeded its 90-second wall ceiling") + noninput = model.spike_counts.copy() + noninput[inputs] = 0 + readouts = {item["label"]: int(model.spike_counts[item["index"]]) + for item in graph.manifest["readouts"]} + return {"mode": mode, "simulated_ms": ticks*PARAMETERS.dt_ms, + "stimulus_ms": 100, "wall_seconds": time.monotonic()-start, + "cpu_seconds": time.process_time()-cpu, + "total_spikes": int(model.spike_counts.sum()), + "downstream_spikes": int(noninput.sum()), + "neurons_that_spiked": int(np.count_nonzero(model.spike_counts)), + "readouts": readouts} + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--cache", type=Path, default=Path("/out")) + parser.add_argument("--fixtures-only", action="store_true") + args = parser.parse_args() + graph = fixture_graph() + events = poisson_events([0, 1], 800, rate_hz=220, seed=41) + # Include exact boundary ticks and convergent excitation/inhibition. + for tick in [0, 1, 18, 21, 22, 23, 44, 45]: + events[tick] = np.array([0, 1]) + results = [compare(graph, events, [0, 1]), compare(graph, events, [0, 1], [3])] + print(json.dumps({"numerical_fixtures": results}), flush=True) + if args.fixtures_only: + return + real = Graph(args.cache) + nodes = [item["index"] for item in real.manifest["diagram"]["nodes"]] + local = {node: i for i, node in enumerate(nodes)} + pre, post, weights = [], [], [] + for i, node in enumerate(nodes): + lo, hi = map(int, real.indptr[node:node+2]) + for target, weight in zip(real.indices[lo:hi], real.counts[lo:hi]): + if int(target) in local: + pre.append(i) + post.append(local[int(target)]) + weights.append(int(weight)) + induced = SimpleNamespace(size=len(nodes), ids=real.ids[nodes], + indices=np.array(post, dtype=np.uint32), + counts=np.array(weights, dtype=np.int32), + indptr=np.concatenate(([0], np.cumsum(np.bincount(pre, minlength=len(nodes)))))) + source_inputs = [local[x] for x in real.manifest["inputs"] if x in local] + results.append(compare(induced, poisson_events(source_inputs, 800), source_inputs)) + trials = [] + for mode in ("baseline", "stimulus", "blocked"): + result = full_trial(real, mode) + trials.append(result) + print(json.dumps(result), flush=True) + if trials[0]["total_spikes"] or trials[2]["downstream_spikes"]: + raise AssertionError("Activity appeared without a transmitted sensory input") + if not trials[1]["downstream_spikes"]: + raise AssertionError("No stimulus-driven propagation in the full graph") + report = { + "schema": 1, "core_sha256": sha256(Path(__file__).with_name("neural_reference.py")), + "cache_manifest_sha256": sha256(args.cache/"manifest.json"), + "numerical_fixtures": results, "full_graph_trials": trials, + "peak_rss_mib": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024, + "body_coupling": "disconnected", "biological_validation": "not established", + } + (args.cache/"validation.json").write_text(json.dumps(report, indent=2)+"\n") + print("Numerical and intervention checks passed", flush=True) + + +if __name__ == "__main__": + main() diff --git a/web/dev-reload.js b/web/dev-reload.js new file mode 100644 index 0000000..5fc41b2 --- /dev/null +++ b/web/dev-reload.js @@ -0,0 +1,16 @@ +// Development only: page reload for frontend edits; Python restarts in Docker. +let previous; +async function check() { + if (!document.hidden) { + try { + const response = await fetch('/api/dev-version', { cache: 'no-store' }); + if (response.ok) { + const current = await response.text(); + if (previous && previous !== current) { location.reload(); return; } + previous = current; + } + } catch { /* The supervisor may be restarting the backend. */ } + } + setTimeout(check, 1500); +} +check(); diff --git a/web/environment.js b/web/environment.js new file mode 100644 index 0000000..5fa0bd4 --- /dev/null +++ b/web/environment.js @@ -0,0 +1,668 @@ +import * as THREE from 'three'; +import { createFly } from './fly.js'; + +const $ = (selector) => document.querySelector(selector); +const canvas = $('#world'); +const keys = new Set(); +const up = new THREE.Vector3(0, 0, 1); +const forward = new THREE.Vector3(); +const right = new THREE.Vector3(); +const movement = new THREE.Vector3(); +const scratch = new THREE.Vector3(); +const projected = new THREE.Vector3(); +const markerPoint = new THREE.Vector2(); +const raycaster = new THREE.Raycaster(); +const scene = new THREE.Scene(); +const camera = new THREE.PerspectiveCamera(65, innerWidth / innerHeight, 0.04, 550); +camera.up.copy(up); +let renderer, world, state, geometries = [], frames = 0; +let lastEvent = 0, lastFrame = performance.now(), fpsStart = lastFrame, frameCount = 0; +let yaw = 0, pitch = 0, drag = null, transition = null, toastTimer; +let probe, trail, lastTrailLength = -1, lastGeneration = -1, connected = null; +let needsRender = true, lastRender = 0, renderLimit = 30, pixelBudget = 1500000; +let graphicsLost = false; +let graphicsDevice = '', softwareGraphics = false, renderCostMs = 0; +let cameraMovedSinceHud = false; +let neuralPanel; +let fly, flyView = false, orbitYaw = -.95, orbitPitch = .45, orbitDistance = .65; +let scheduledFrame = 0, hudTimer = 0; +const freeHelp = $('.navigation-help').innerHTML; +const reducedMotion = matchMedia('(prefers-reduced-motion: reduce)').matches; + +function invalidate() { + needsRender = true; + scheduleFrame(); +} + +function scheduleFrame() { + if (scheduledFrame || document.hidden || graphicsLost || !renderer) return; + scheduledFrame = requestAnimationFrame(render); +} + +function toast(text) { + $('#toast').textContent = text; + $('#toast').hidden = false; + clearTimeout(toastTimer); + toastTimer = setTimeout(() => { $('#toast').hidden = true; }, 3800); +} + +function fail(message) { + $('#loading').hidden = false; + $('#loading').replaceChildren(); + const title = document.createElement('strong'); + title.textContent = 'Could not open the environment'; + const explanation = document.createElement('span'); + explanation.textContent = message; + $('#loading').append(title, explanation); +} + +function setConnected(value) { + if (connected === value) return; + connected = value; + $('#connection-dot').className = `dot ${value ? 'live' : 'offline'}`; + $('#connection').textContent = value ? 'Environment connected' : 'Reconnecting…'; + for (const id of ['#drop', '#pause', '#speed']) $(id).disabled = !value; + neuralPanel?.setConnected(value); +} + +async function command(payload) { + try { + const response = await fetch('/api/command', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }); + if (!response.ok) { + const failure = await response.json().catch(() => ({})); + toast(failure.error || `Command unavailable (HTTP ${response.status}).`); + return false; + } + updateState(await response.json()); + return true; + } catch { + toast('The command did not reach the environment. Wait for reconnection and try again.'); + return false; + } +} + +function updateState(next) { + if (!next.probe.trail) next.probe.trail = state?.probe.trail || []; + if (!next.neural || (state?.neural?.revision ?? -1) > next.neural.revision) next.neural = state?.neural; + state = next; + neuralPanel?.update(next.neural); + lastEvent = performance.now(); + setConnected(!next.error); + if (next.error) { + $('#connection').textContent = 'Physics stopped'; + toast('The simulation has stopped. Check the environment service.'); + } + $('#pause').textContent = next.paused ? '▷' : 'Ⅱ'; + $('#pause').setAttribute('aria-label', next.paused ? 'Resume simulation' : 'Pause simulation'); + $('#pause').title = next.paused ? 'Resume simulation' : 'Pause simulation'; + $('#speed').value = String(next.speed); + const minutes = Math.floor(next.time / 60).toString().padStart(2, '0'); + $('#sim-time').textContent = `${minutes}:${(next.time % 60).toFixed(2).padStart(5, '0')}`; + $('#physical-rate').textContent = next.paused ? 'PAUSED' : `${next.realtime_ratio.toFixed(2)}× PHYSICS`; + if (fly && fly.apply(next.fly)) { + if (flyView) orbitCamera(); + invalidate(); + } + $('#fly-status').textContent = next.fly.sleeping ? 'At rest' : next.paused ? 'Paused' : 'Settling'; + if (!probe) return; + const moved = probe.position.distanceToSquared(scratch.fromArray(next.probe.position)) > 1e-8; + if (probe.visible !== next.probe.active || (next.probe.active && moved)) { + invalidate(); + renderer.shadowMap.needsUpdate = true; + } + probe.visible = trail.visible = next.probe.active; + probe.position.fromArray(next.probe.position); + if (next.probe.generation !== lastGeneration || next.probe.trail.length !== lastTrailLength) { + const attribute = trail.geometry.attributes.position; + for (let i = 0; i < next.probe.trail.length; i++) attribute.array.set(next.probe.trail[i], i * 3); + attribute.needsUpdate = true; + trail.geometry.setDrawRange(0, next.probe.trail.length); + lastTrailLength = next.probe.trail.length; + lastGeneration = next.probe.generation; + } + $('#probe-status').textContent = !next.probe.active + ? 'Test sphere · 9 mm diameter' + : next.probe.contact_seen ? 'Physical contact recorded on the walkway' : 'Sphere falling · measured trajectory'; + $('#probe-status').classList.toggle('contact', next.probe.contact_seen); +} + +function geometryFor(item, material) { + const [x, y, z] = item.size; + let geometry; + if (item.type === 'box') { + geometry = new THREE.BoxGeometry(x * 2, y * 2, z * 2); + if (material.name !== 'designation') { + const positions = geometry.attributes.position; + const normals = geometry.attributes.normal; + const uv = geometry.attributes.uv; + const tile = material.name === 'metal' || material.name === 'cladding' ? 7 : 18; + for (let i = 0; i < positions.count; i++) { + const nx = Math.abs(normals.getX(i)), ny = Math.abs(normals.getY(i)); + uv.setXY(i, (nx > .5 ? positions.getY(i) : positions.getX(i)) / tile, + (nx > .5 || ny > .5 ? positions.getZ(i) : positions.getY(i)) / tile); + } + } + } else if (item.type === 'plane') { + geometry = new THREE.PlaneGeometry(x * 2, y * 2); + const uv = geometry.attributes.uv; + for (let i = 0; i < uv.count; i++) uv.setXY(i, uv.getX(i) * 24, uv.getY(i) * 24); + } else if (item.type === 'cylinder') { + geometry = new THREE.CylinderGeometry(x, x, y * 2, x < .4 ? 8 : 24); + geometry.rotateX(Math.PI / 2); + } else if (item.type === 'capsule') { + geometry = new THREE.CapsuleGeometry(x, y * 2, 4, 8); + geometry.rotateX(Math.PI / 2); + } else if (item.type === 'sphere') { + geometry = new THREE.SphereGeometry(x, 24, 16); + } else { + throw new Error(`Unsupported geometry: ${item.type}`); + } + return geometry; +} + +async function buildWorld() { + const textures = new Map(); + const loader = new THREE.TextureLoader(); + await Promise.all(world.materials.filter((m) => m.texture).map(async (m) => { + if (textures.has(m.texture)) return; + const promise = loader.loadAsync(m.texture); + textures.set(m.texture, promise); + const texture = await promise; + texture.colorSpace = THREE.SRGBColorSpace; + texture.wrapS = texture.wrapT = THREE.RepeatWrapping; + texture.anisotropy = Math.min(renderer.capabilities.getMaxAnisotropy(), 8); + textures.set(m.texture, texture); + })); + const materials = world.materials.map((m) => { + const color = new THREE.Color().setRGB(...m.rgba.slice(0, 3)); + const mat = new THREE.MeshStandardMaterial({ + name: m.name, color, + roughness: m.name === 'metal' ? .66 : .91, + metalness: ['metal', 'cladding'].includes(m.name) ? .25 : .02, + map: m.texture ? textures.get(m.texture) : null, + emissive: m.emission ? color : 0, + emissiveIntensity: m.emission * 1.4, + }); + if (m.name === 'lamp') mat.emissive.setHex(0xffd09a); + return mat; + }); + for (const item of world.geoms) { + const material = materials[item.material]; + if (!material) throw new Error('World material is missing'); + const mesh = new THREE.Mesh(geometryFor(item, material), material); + mesh.name = item.name; + mesh.position.fromArray(item.position); + const r = item.rotation; + const matrix = new THREE.Matrix4().set( + ...r[0], 0, ...r[1], 0, ...r[2], 0, 0, 0, 0, 1, + ); + mesh.quaternion.setFromRotationMatrix(matrix); + mesh.castShadow = item.type !== 'plane'; + mesh.receiveShadow = true; + mesh.userData.physical = item; + mesh.updateMatrixWorld(true); + geometries.push(mesh); + } + // The world is static: batch by material while retaining exact source meshes + // for observer collision queries. Physical positions still come from MuJoCo. + for (const material of materials) { + const sources = geometries.filter((mesh) => mesh.material === material).map((mesh) => { + const geometry = mesh.geometry.index ? mesh.geometry.toNonIndexed() : mesh.geometry.clone(); + return geometry.applyMatrix4(mesh.matrixWorld); + }); + if (!sources.length) continue; + const merged = new THREE.BufferGeometry(); + for (const attribute of ['position', 'normal', 'uv']) { + const length = sources.reduce((sum, g) => sum + g.attributes[attribute].array.length, 0); + const buffer = new Float32Array(length); + let offset = 0; + for (const source of sources) { + buffer.set(source.attributes[attribute].array, offset); + offset += source.attributes[attribute].array.length; + } + merged.setAttribute(attribute, new THREE.BufferAttribute(buffer, attribute === 'uv' ? 2 : 3)); + } + for (const source of sources) source.dispose(); + const batch = new THREE.Mesh(merged, material); + batch.castShadow = batch.receiveShadow = true; + scene.add(batch); + } + scene.background = new THREE.Color(0x536674); + scene.fog = new THREE.FogExp2(0x536674, .009); + const hemisphere = new THREE.HemisphereLight(0xccdfeb, 0x34434b, 2.3); + hemisphere.position.copy(up); + scene.add(hemisphere); + const upper = new THREE.DirectionalLight(0xd6e6f1, 2.8); + upper.position.set(-28, -16, 174); + upper.target.position.set(0, -25, -15); + upper.castShadow = true; + upper.shadow.mapSize.set(1024, 1024); + Object.assign(upper.shadow.camera, { left: -92, right: 92, top: 110, bottom: -110, near: .5, far: 320 }); + upper.shadow.bias = -.00025; + upper.shadow.normalBias = .1; + scene.add(upper, upper.target); + const bounce = new THREE.DirectionalLight(0xa9c6da, .9); + bounce.position.set(-52, -38, 40); + bounce.target.position.set(0, -15, 10); + scene.add(bounce, bounce.target); + for (const x of [-45, -15, 15, 45]) { + const light = new THREE.PointLight(0xffbe77, 22, 17, 1.5); + light.position.set(x, -38, 11); + scene.add(light); + } + probe = new THREE.Mesh( + new THREE.SphereGeometry(world.probe.radius, 24, 16), + new THREE.MeshStandardMaterial({ color: 0xeaaa57, metalness: .28, roughness: .3, + emissive: 0xa85112, emissiveIntensity: .7 }), + ); + probe.castShadow = true; + probe.visible = false; + scene.add(probe); + const trailGeometry = new THREE.BufferGeometry(); + trailGeometry.setAttribute('position', new THREE.BufferAttribute(new Float32Array(1500), 3).setUsage(THREE.DynamicDrawUsage)); + trailGeometry.setDrawRange(0, 0); + trail = new THREE.Line(trailGeometry, new THREE.LineBasicMaterial({ + color: 0xffc276, transparent: true, opacity: .65, + })); + trail.visible = false; + trail.frustumCulled = false; + scene.add(trail); + scene.updateMatrixWorld(true); + $('#geom-count').textContent = String(world.geoms.length); +} + +function syncAngles() { + camera.getWorldDirection(forward); + yaw = Math.atan2(forward.y, forward.x); + pitch = Math.asin(THREE.MathUtils.clamp(forward.z, -.999, .999)); +} + +function orient() { + cameraMovedSinceHud = true; + scratch.set( + Math.cos(yaw) * Math.cos(pitch), Math.sin(yaw) * Math.cos(pitch), Math.sin(pitch), + ); + camera.lookAt(scratch.add(camera.position)); + invalidate(); +} + +function orbitCamera() { + camera.position.set( + Math.cos(orbitYaw) * Math.cos(orbitPitch), + Math.sin(orbitYaw) * Math.cos(orbitPitch), + Math.sin(orbitPitch), + ).multiplyScalar(orbitDistance * Math.max(1, .95 / camera.aspect)).add(fly.position); + camera.lookAt(fly.position); + syncAngles(); + invalidate(); +} + +function setView(id, immediate = false) { + cameraMovedSinceHud = true; + flyView = id === 'fly'; + document.body.classList.toggle('fly-view', flyView); + $('#fly-panel').hidden = !flyView; + $('#gravity-panel').hidden = flyView; + $('.navigation-help').innerHTML = flyView ? 'Drag to orbitScroll to zoom' : freeHelp; + camera.near = flyView ? .001 : .04; + camera.updateProjectionMatrix(); + document.querySelectorAll('[data-view]').forEach((button) => { + const selected = button.dataset.view === id; + button.classList.toggle('selected', selected); + button.setAttribute('aria-pressed', String(selected)); + }); + if (flyView) { + transition = null; + keys.clear(); + $('#view-title').textContent = 'Fly 001'; + $('#view-caption').textContent = 'One fruit fly on the lower walkway.'; + orbitCamera(); + return; + } + const view = world.views.find((v) => v.id === id); + if (!view) return; + invalidate(); + const destination = new THREE.Vector3(...view.position); + const target = new THREE.Vector3(...view.target); + const endCamera = camera.clone(); + endCamera.position.copy(destination); + endCamera.lookAt(target); + if (immediate || reducedMotion) { + camera.position.copy(destination); + camera.quaternion.copy(endCamera.quaternion); + transition = null; + syncAngles(); + } else { + transition = { + started: performance.now(), from: camera.position.clone(), to: destination, + startRotation: camera.quaternion.clone(), endRotation: endCamera.quaternion.clone(), + }; + } + $('#view-title').textContent = view.title; + $('#view-caption').textContent = view.caption; +} + +function moveCamera(delta) { + transition = null; + const distance = delta.length(); + if (!distance) return; + cameraMovedSinceHud = true; + raycaster.set(camera.position, scratch.copy(delta).normalize()); + raycaster.near = 0; + raycaster.far = distance + .3; + const hit = raycaster.intersectObjects(geometries, false)[0]; + if (hit) delta.multiplyScalar(Math.max(0, hit.distance - .3) / distance); + camera.position.add(delta); + camera.position.x = THREE.MathUtils.clamp(camera.position.x, -57, 57); + camera.position.y = THREE.MathUtils.clamp(camera.position.y, -40, 40); + camera.position.z = THREE.MathUtils.clamp(camera.position.z, -46, 175); + invalidate(); +} + +function drawMap() { + const map = $('#minimap'), ctx = map.getContext('2d'); + const scale = 2.7, ox = map.width / 2, oy = map.height / 2; + ctx.clearRect(0, 0, map.width, map.height); + ctx.strokeStyle = '#aac8cb12'; + ctx.lineWidth = 1; + for (let x = 21; x < map.width; x += 27) { + ctx.beginPath(); ctx.moveTo(x, 15); ctx.lineTo(x, map.height - 15); ctx.stroke(); + } + for (let y = 20; y < map.height; y += 27) { + ctx.beginPath(); ctx.moveTo(18, y); ctx.lineTo(map.width - 18, y); ctx.stroke(); + } + for (const item of world.geoms) { + if (item.type === 'plane') continue; + const [x, y, z] = item.position, [sx, sy, sz] = item.size; + const reaches = item.type === 'cylinder' ? sy : sz; + const active = Math.abs(camera.position.z - z) <= reaches; + ctx.fillStyle = active ? '#b1c8c32c' : '#b1c8c30a'; + ctx.strokeStyle = active ? '#accac573' : '#accac526'; + if (item.type === 'box') { + ctx.fillRect(ox + (x - sx) * scale, oy - (y + sy) * scale, sx * 2 * scale, sy * 2 * scale); + ctx.strokeRect(ox + (x - sx) * scale, oy - (y + sy) * scale, sx * 2 * scale, sy * 2 * scale); + } else if (item.type === 'cylinder') { + ctx.beginPath(); ctx.arc(ox + x * scale, oy - y * scale, Math.max(sx * scale, 1), 0, Math.PI * 2); + ctx.fill(); ctx.stroke(); + } + } + const x = ox + camera.position.x * scale, y = oy - camera.position.y * scale; + ctx.fillStyle = '#c4e4c82b'; + ctx.beginPath(); ctx.moveTo(x, y); ctx.arc(x, y, 29, -yaw - .5, -yaw + .5); ctx.closePath(); ctx.fill(); + ctx.fillStyle = '#c4e4c8'; + ctx.beginPath(); ctx.arc(x, y, 3.5, 0, Math.PI * 2); ctx.fill(); + if (fly) { + ctx.fillStyle = '#efc07c'; + ctx.beginPath(); ctx.arc(ox + fly.position.x * scale, oy - fly.position.y * scale, 3, 0, Math.PI * 2); ctx.fill(); + } + $('#coordinates').textContent = camera.position.toArray().map((p) => p.toFixed(1).padStart(5)).join(' / '); +} + +function render(now) { + scheduledFrame = 0; + if (document.hidden || graphicsLost) return; + if (!needsRender && !transition && !(keys.size && !flyView)) return; + // Pace on vsync. A full timeout followed by rAF can miss the target refresh. + // Only pending work schedules another callback; idle scenes still stop. + if (now - lastRender < 1000 / renderLimit - .5) { + scheduleFrame(); + return; + } + const dt = Math.min((now - lastFrame) / 1000, .05); + lastFrame = now; + if (transition) { + cameraMovedSinceHud = true; + needsRender = true; + const t = Math.min((now - transition.started) / 1100, 1); + const ease = t * t * (3 - 2 * t); + camera.position.lerpVectors(transition.from, transition.to, ease); + camera.quaternion.slerpQuaternions(transition.startRotation, transition.endRotation, ease); + if (t === 1) { transition = null; syncAngles(); } + } + if (keys.size && !flyView) { + camera.getWorldDirection(forward); + right.crossVectors(forward, up).normalize(); + const delta = movement.set(0, 0, 0); + if (keys.has('KeyW') || keys.has('ArrowUp')) delta.add(forward); + if (keys.has('KeyS') || keys.has('ArrowDown')) delta.sub(forward); + if (keys.has('KeyD') || keys.has('ArrowRight')) delta.add(right); + if (keys.has('KeyA') || keys.has('ArrowLeft')) delta.sub(right); + if (keys.has('KeyE')) delta.add(up); + if (keys.has('KeyQ')) delta.sub(up); + const speed = keys.has('ShiftLeft') || keys.has('ShiftRight') ? 30 : 12; + moveCamera(delta.normalize().multiplyScalar(dt * speed)); + } + if (needsRender) { + const started = performance.now(); + fly.updateVisibility(camera, canvas.height); + renderer.render(scene, camera); + frames++; frameCount++; + needsRender = false; + lastRender = now; + drawMap(); + updateMarker(); + renderCostMs = performance.now() - started; + } + if (transition || (keys.size && !flyView)) scheduleFrame(); +} + +function updateMarker() { + const marker = $('#fly-marker'); + marker.hidden = true; + if (flyView || fly.revision < 0) return; + projected.copy(fly.position).project(camera); + if (Math.abs(projected.x) > .96 || Math.abs(projected.y) > .92 || Math.abs(projected.z) > 1) return; + scratch.copy(fly.position).sub(camera.position); + const distance = scratch.length(); + raycaster.set(camera.position, scratch.normalize()); + raycaster.near = 0; + raycaster.far = Math.max(0, distance - .15); + if (raycaster.intersectObjects(geometries, false).length) return; + marker.style.left = `${(projected.x + 1) * innerWidth / 2}px`; + marker.style.top = `${(1 - projected.y) * innerHeight / 2}px`; + marker.hidden = false; +} + +function updateHud() { + if (!document.hidden) { + const now = performance.now(); + const rate = Math.round(frameCount * 1000 / (now - fpsStart)); + $('#fps').textContent = frameCount ? `${rate} ${cameraMovedSinceHud ? 'FPS' : 'UPDATES/S'}` : 'VIEW IDLE'; + $('#fps').title = 'FPS while moving the camera; changed scene updates per second while stationary. Target: 30 FPS during movement. Idle scenes do not redraw.'; + cameraMovedSinceHud = false; + frameCount = 0; + fpsStart = now; + if (connected && now - lastEvent > 3500) setConnected(false); + } +} + +function bindControls() { + document.querySelectorAll('[data-view]').forEach((button) => { + button.addEventListener('click', () => setView(button.dataset.view)); + }); + $('#fly-marker').addEventListener('click', () => setView('fly')); + $('#drop').addEventListener('click', async () => { + // Move the observer first so the fast physical event is actually visible. + setView('lower', true); + if (await command({ action: 'drop' })) toast('Sphere released. Use 0.25× to watch the fall in slow motion.'); + }); + $('#pause').addEventListener('click', () => command({ action: 'pause', paused: !state.paused })); + $('#speed').addEventListener('change', () => command({ action: 'speed', value: Number($('#speed').value) })); + $('#fullscreen').addEventListener('click', async () => { + try { + if (document.fullscreenElement) await document.exitFullscreen(); + else await document.documentElement.requestFullscreen(); + } catch { toast('This browser did not allow full screen.'); } + }); + canvas.addEventListener('pointerdown', (event) => { + if (event.button !== 0) return; + canvas.focus(); + transition = null; + syncAngles(); + drag = { id: event.pointerId, x: event.clientX, y: event.clientY, moved: 0 }; + canvas.setPointerCapture(event.pointerId); + }); + canvas.addEventListener('pointermove', (event) => { + if (!drag || event.pointerId !== drag.id) return; + const dx = event.clientX - drag.x, dy = event.clientY - drag.y; + drag.moved += Math.abs(dx) + Math.abs(dy); + if (flyView) { + cameraMovedSinceHud = true; + orbitYaw -= dx * .008; + orbitPitch = THREE.MathUtils.clamp(orbitPitch + dy * .008, .12, 1.48); + drag.x = event.clientX; drag.y = event.clientY; + orbitCamera(); + return; + } + yaw -= (event.clientX - drag.x) * .003; + pitch = THREE.MathUtils.clamp(pitch - (event.clientY - drag.y) * .003, -1.48, 1.48); + drag.x = event.clientX; drag.y = event.clientY; + orient(); + }); + const release = (event) => { + if (drag && drag.moved < 5 && !flyView && event.type === 'pointerup') { + markerPoint.set(event.clientX / innerWidth * 2 - 1, 1 - event.clientY / innerHeight * 2); + raycaster.setFromCamera(markerPoint, camera); + if (raycaster.ray.distanceToPoint(fly.position) < .18) { + // Selection changes only the observer; the marker already handles far views. + const distance = camera.position.distanceTo(fly.position); + raycaster.near = 0; raycaster.far = Math.max(0, distance - .18); + if (!raycaster.intersectObjects(geometries, false).length) setView('fly'); + } + } + drag = null; + }; + canvas.addEventListener('pointerup', release); + canvas.addEventListener('pointercancel', release); + canvas.addEventListener('lostpointercapture', release); + canvas.addEventListener('wheel', (event) => { + event.preventDefault(); + if (flyView) { + cameraMovedSinceHud = true; + orbitDistance = THREE.MathUtils.clamp(orbitDistance * Math.exp(event.deltaY * .001), .3, 3); + orbitCamera(); + return; + } + camera.getWorldDirection(forward); + moveCamera(movement.copy(forward).multiplyScalar(-THREE.MathUtils.clamp(event.deltaY * .012, -4, 4))); + }, { passive: false }); + const movementKeys = new Set(['KeyW', 'KeyA', 'KeyS', 'KeyD', 'KeyQ', 'KeyE', + 'ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight', 'ShiftLeft', 'ShiftRight']); + addEventListener('keydown', (event) => { + if (['SELECT', 'INPUT', 'TEXTAREA', 'BUTTON'].includes(document.activeElement.tagName)) return; + if (!flyView && movementKeys.has(event.code)) { + keys.add(event.code); event.preventDefault(); lastFrame = performance.now(); invalidate(); + } + }); + addEventListener('keyup', (event) => keys.delete(event.code)); + addEventListener('blur', () => { keys.clear(); drag = null; }); + document.addEventListener('visibilitychange', () => { + keys.clear(); lastFrame = performance.now(); + if (document.hidden) { + cancelAnimationFrame(scheduledFrame); + scheduledFrame = 0; + } else invalidate(); + }); + document.querySelectorAll('[data-move]').forEach((button) => { + button.addEventListener('pointerdown', (event) => { + event.preventDefault(); button.setPointerCapture(event.pointerId); + keys.add(button.dataset.move); + lastFrame = performance.now(); invalidate(); + }); + for (const name of ['pointerup', 'pointercancel', 'lostpointercapture']) { + button.addEventListener(name, () => keys.delete(button.dataset.move)); + } + }); + addEventListener('resize', () => { + camera.aspect = innerWidth / innerHeight; + camera.updateProjectionMatrix(); + renderer.setPixelRatio(Math.min(1, Math.sqrt(pixelBudget / (innerWidth * innerHeight)))); + renderer.setSize(innerWidth, innerHeight); + if (flyView) orbitCamera(); + invalidate(); + }); +} + +async function start() { + try { + renderer = new THREE.WebGLRenderer({ canvas, antialias: true, powerPreference: 'high-performance' }); + const gl = renderer.getContext(); + const debug = gl.getExtension('WEBGL_debug_renderer_info'); + graphicsDevice = debug ? gl.getParameter(debug.UNMASKED_RENDERER_WEBGL) : ''; + softwareGraphics = /swiftshader|llvmpipe|software/i.test(graphicsDevice); + if (softwareGraphics) { + pixelBudget = 360000; + } + $('#graphics-mode').textContent = `${softwareGraphics ? 'SOFTWARE' : graphicsDevice ? 'GPU' : 'WEBGL'} · 30 FPS TARGET`; + $('#graphics-mode').title = softwareGraphics + ? `${graphicsDevice}. Enable hardware acceleration in your browser to use the GPU.` + : graphicsDevice || 'The browser does not expose its graphics device.'; + renderer.setPixelRatio(Math.min(1, Math.sqrt(pixelBudget / (innerWidth * innerHeight)))); + renderer.setSize(innerWidth, innerHeight); + renderer.shadowMap.enabled = true; + renderer.shadowMap.type = THREE.PCFSoftShadowMap; + renderer.shadowMap.autoUpdate = false; + renderer.shadowMap.needsUpdate = true; + renderer.toneMapping = THREE.ACESFilmicToneMapping; + renderer.toneMappingExposure = 1.05; + canvas.addEventListener('webglcontextlost', (event) => { + event.preventDefault(); + graphicsLost = true; + fail('The graphics context was lost. Reload the page to resume viewing.'); + }); + canvas.addEventListener('webglcontextrestored', () => { + graphicsLost = false; + renderer.setSize(innerWidth, innerHeight); + renderer.shadowMap.needsUpdate = true; + invalidate(); + $('#loading').hidden = true; + }); + const response = await fetch('/api/world'); + if (!response.ok) throw new Error('The environment service did not respond.'); + world = await response.json(); + if (world.neural) { + const { createNeuralPanel } = await import('./neural.js'); + neuralPanel = createNeuralPanel(world.neural, command); + } + await buildWorld(); + const [animal, initial] = await Promise.all([ + createFly(world.fly_body, scene), + fetch('/api/state').then((r) => { if (!r.ok) throw new Error('Physics unavailable.'); return r.json(); }), + ]); + fly = animal; + $('#fly-mass').textContent = `${world.fly_body.mass_mg.toFixed(3)} mg`; + updateState(initial); + setView('fly', true); + bindControls(); + drawMap(); + const events = new EventSource('/api/events'); + events.onmessage = (event) => updateState(JSON.parse(event.data)); + events.onerror = () => setConnected(false); + addEventListener('pagehide', () => { + events.close(); clearInterval(hudTimer); + cancelAnimationFrame(scheduledFrame); + }, { once: true }); + $('#loading').hidden = !graphicsLost; + invalidate(); + hudTimer = setInterval(updateHud, 1000); + // Read-only observability for browser checks; no simulation control hidden here. + Object.defineProperty(window, 'environmentDebug', { get: () => ({ + ready: !graphicsLost, frames, camera: camera.position.toArray(), direction: camera.getWorldDirection(new THREE.Vector3()).toArray(), + renderedGeoms: geometries.length, modelHash: world.model_sha256, + simulation: state ? structuredClone(state) : null, connected, + drawCalls: renderer.info.render.calls, triangles: renderer.info.render.triangles, + renderLimit, pixelBudget, graphicsDevice, softwareGraphics, renderCostMs, + fly: { selected: flyView, visible: fly.visible, poseRevision: fly.revision, + position: fly.position.toArray(), bodies: world.fly_body.bodies.length }, + geometryCount: renderer.info.memory.geometries, textureCount: renderer.info.memory.textures, + }) }); + } catch (error) { + console.error(error); + fail(error.message || 'Check WebGL 2 support and reload the page.'); + } +} + +start(); diff --git a/web/favicon.svg b/web/favicon.svg new file mode 100644 index 0000000..9bf027a --- /dev/null +++ b/web/favicon.svg @@ -0,0 +1 @@ + diff --git a/web/fly.js b/web/fly.js new file mode 100644 index 0000000..bd810ac --- /dev/null +++ b/web/fly.js @@ -0,0 +1,96 @@ +import * as THREE from 'three'; + +// Rigid skinning: every surface vertex follows exactly one measured physical body. +// Seven material batches share one skeleton; no meshes are rebuilt per state. +export async function createFly(description, scene) { + const response = await fetch(description.surface_url); + if (!response.ok) throw new Error('The fly anatomy could not be loaded.'); + const binary = await response.arrayBuffer(); + if (binary.byteLength !== description.surface_bytes) throw new Error('Incomplete fly anatomy.'); + const hash = Array.from(new Uint8Array(await crypto.subtle.digest('SHA-256', binary)), + (byte) => byte.toString(16).padStart(2, '0')).join(''); + if (hash !== description.surface_sha256) throw new Error('Fly anatomy checksum mismatch.'); + const rig = new THREE.Group(); + const bones = description.bodies.map((name) => { + const bone = new THREE.Bone(); + bone.name = name; + rig.add(bone); + return bone; + }); + const skeleton = new THREE.Skeleton(bones, bones.map(() => new THREE.Matrix4())); + const groups = new Map(); + for (const geom of description.geoms) { + const key = geom.rgba.join(','); + if (!groups.has(key)) groups.set(key, []); + groups.get(key).push(geom); + } + const meshes = []; + for (const geoms of groups.values()) { + const count = geoms.reduce((n, g) => n + description.meshes[g.mesh].vertex_count, 0); + const indexCount = geoms.reduce((n, g) => n + description.meshes[g.mesh].index_count, 0); + const positions = new Float32Array(count * 3); + const indices = new Uint32Array(indexCount); + const skinIndex = new Uint16Array(count * 4); + const skinWeight = new Float32Array(count * 4); + let vertexOffset = 0, indexOffset = 0; + for (const geom of geoms) { + const source = description.meshes[geom.mesh]; + positions.set(new Float32Array(binary, source.vertex_offset, source.vertex_count * 3), vertexOffset * 3); + const sourceIndices = new Uint32Array(binary, source.index_offset, source.index_count); + for (let i = 0; i < source.index_count; i++) indices[indexOffset + i] = sourceIndices[i] + vertexOffset; + for (let i = vertexOffset; i < vertexOffset + source.vertex_count; i++) { + skinIndex[i * 4] = geom.body; + skinWeight[i * 4] = 1; + } + vertexOffset += source.vertex_count; + indexOffset += source.index_count; + } + const geometry = new THREE.BufferGeometry(); + geometry.setAttribute('position', new THREE.BufferAttribute(positions, 3)); + geometry.setAttribute('skinIndex', new THREE.BufferAttribute(skinIndex, 4)); + geometry.setAttribute('skinWeight', new THREE.BufferAttribute(skinWeight, 4)); + geometry.setIndex(new THREE.BufferAttribute(indices, 1)); + geometry.computeVertexNormals(); + const [r, g, b, a] = geoms[0].rgba; + const material = new THREE.MeshStandardMaterial({ + color: new THREE.Color(r, g, b), roughness: .65, metalness: .02, + transparent: a < 1, opacity: a, depthWrite: a === 1, + side: a < 1 ? THREE.DoubleSide : THREE.FrontSide, + }); + const mesh = new THREE.SkinnedMesh(geometry, material); + mesh.bind(skeleton, new THREE.Matrix4()); + mesh.frustumCulled = false; // Visibility is bounded by projected animal size below. + meshes.push(mesh); + rig.add(mesh); + } + rig.visible = false; + scene.add(rig); + const position = new THREE.Vector3(); + const bytes = new Uint8Array(bones.length * 7 * 4); + const poses = new Float32Array(bytes.buffer); + let revision = -1; + return { + position, + get revision() { return revision; }, + get visible() { return rig.visible; }, + apply(state) { + if (!state.poses_b64 || state.pose_revision === revision) return false; + const decoded = atob(state.poses_b64); + if (decoded.length !== bytes.length) throw new Error('Invalid physical pose packet.'); + for (let i = 0; i < bytes.length; i++) bytes[i] = decoded.charCodeAt(i); + for (let i = 0; i < bones.length; i++) { + const o = i * 7; + bones[i].position.fromArray(poses, o); + bones[i].quaternion.set(poses[o + 4], poses[o + 5], poses[o + 6], poses[o + 3]); + } + position.copy(bones[description.focus_body].position); + revision = state.pose_revision; + return true; + }, + updateVisibility(camera, height) { + const pixels = .3 * height / (2 * camera.position.distanceTo(position) * + Math.tan(THREE.MathUtils.degToRad(camera.fov / 2))); + rig.visible = revision >= 0 && pixels >= 8; + }, + }; +} diff --git a/web/index.html b/web/index.html new file mode 100644 index 0000000..b71d573 --- /dev/null +++ b/web/index.html @@ -0,0 +1,129 @@ + + + + + + + + NETSPHERE — Fly environment + + + + + + +
+ + +
+ + + NETSPHEREFLY ENVIRONMENT / 001 + +
Connecting
+ +
+ +
+ SECTOR 09 OBSERVATORY +

South gallery

+

Walkways, pillars and suspended cables.

+
+ + + + +
+
+ + + +
+
ENVIRONMENT PHYSICS

A real free fall.

+

Drop a sphere above the walkway to observe gravity and collision.

+ +

Test sphere · 9 mm diameter

+
+ + + + + + +
Building the spaceLoading the City's physical geometry…
+ +
+
+ +
SIMULATION TIME00:00.00
+ +
+
W A S D moveQ E heightDrag to look
+
— FPS30 FPS TARGET—
+
+
+ +
+
+ + diff --git a/web/neural.css b/web/neural.css new file mode 100644 index 0000000..e879145 --- /dev/null +++ b/web/neural.css @@ -0,0 +1,50 @@ +.neural-launch{margin-top:15px} +.neural-panel{position:absolute;right:24px;top:125px;bottom:103px;width:min(470px,calc(100vw - 40px));z-index:4;overflow:auto;overscroll-behavior:contain;background:#101c23f5;border:1px solid #c3d9dc40;border-radius:7px;padding:22px;color:#dae5e2;scrollbar-width:thin;box-shadow:0 12px 50px #0004} +.neural-open .map-panel{visibility:hidden} +.neural-heading{display:flex;align-items:flex-start;justify-content:space-between;gap:12px} +.neural-heading h2{font-size:25px;margin:8px 0 4px;font-weight:400} +.neural-heading .icon-button{flex-shrink:0;font-size:20px} +.neural-panel p{font-size:11px;line-height:1.65;color:#adbdc2;margin:8px 0} +.neural-panel h3{font-size:10px;letter-spacing:1.6px;font-weight:500;text-transform:uppercase;margin:21px 0 11px} +.neural-panel .neural-source{font:11px monospace;color:#bfd7ce} +.neural-panel .neural-boundary{border-left:2px solid #dcba80;padding:9px 12px;background:#dcba800b;color:#deceb3} +.neural-workflow{display:flex;gap:5px;align-items:stretch;margin:14px 0 10px;padding:0;list-style:none} +.neural-workflow li{flex:1;min-width:0;border:1px solid var(--line);padding:10px 5px;text-align:center;font-size:10px;border-radius:3px;position:relative} +.neural-workflow li+li::before{content:"→";position:absolute;left:-7px;top:11px;background:#101c23;color:#a6bdb6;font-size:10px} +.neural-workflow small{display:block;font-size:8px;color:#9baeb7;margin-top:5px;line-height:1.5} +.neural-workflow .measured{border-color:#b5d4bf99;background:#b5d4bf13} +.neural-workflow .disconnected{border-style:dashed;color:#dfc392} +.neural-actions{display:flex;flex-wrap:wrap;gap:6px;margin:13px 0} +.neural-actions button{border:1px solid var(--line);border-radius:4px;font-size:10px;background:#192930;padding:10px} +.neural-actions button:first-child{flex-basis:100%;background:#bfd5c4;color:#182b25;font-weight:600} +.neural-actions button[aria-pressed=true]{border-color:#bfd5c4} +.neural-actions button:disabled{cursor:default} +.neural-panel #neural-status{color:#d4e4dc;font-size:11px} +.neural-panel progress{appearance:none;display:block;width:100%;height:3px;border:0;margin:12px 0 9px;background:#26363b;accent-color:#b5d4bf} +.neural-panel progress::-webkit-progress-bar{background:#26363b} +.neural-panel progress::-webkit-progress-value{background:#b5d4bf} +.neural-panel progress::-moz-progress-bar{background:#b5d4bf} +.neural-timing{display:flex;flex-wrap:wrap;justify-content:space-between;gap:8px;font:9px monospace;color:#9fb4bb} +.neural-metrics{display:grid;grid-template-columns:repeat(3,1fr);gap:8px;margin:19px 0;padding:14px 0;border-top:1px solid var(--line);border-bottom:1px solid var(--line)} +.neural-metrics span{font-size:9px;color:#adbdc2} +.neural-metrics strong{display:block;font:22px monospace;color:#dce8de;margin-bottom:5px} +#neural-graph{display:block;width:100%;height:auto;overflow:visible} +#neural-graph>path{fill:none;stroke-width:.8;opacity:.45} +#neural-graph .graph-excitatory{stroke:#98bbaa} +#neural-graph .graph-inhibitory{stroke:#d39e84;stroke-dasharray:3 2} +.graph-column{font:8px sans-serif;letter-spacing:1.5px;fill:#91a9af} +.graph-node circle{fill:#23363e;stroke:#7f9c9d;stroke-width:1} +.graph-node text{font:8px monospace;fill:#9fb4b7} +.graph-node.has-spikes circle{fill:#b7d9c3;stroke:#d9eee0} +.graph-node.has-spikes text{fill:#dae8df} +.neural-legend{display:flex;gap:18px;font-size:9px;color:#adc6b8} +.neural-legend span:last-child{color:#d39e84} +#neural-timeline{width:100%;height:auto;display:block} +.neural-readout-table{width:100%;border-collapse:collapse;font:11px monospace;margin:10px 0} +.neural-readout-table th,.neural-readout-table td{padding:8px 0;border-bottom:1px solid #c3d9dc18;text-align:right;font-weight:400} +.neural-readout-table th:first-child{text-align:left} +.neural-readout-table thead{font:9px sans-serif;color:#a5b7be} +.neural-panel .neural-footnote{font-size:10px;margin-top:17px} +.neural-panel a{color:#bfd7ce;text-underline-offset:3px} +@media(max-height:650px) and (min-width:701px){.neural-panel{top:90px;bottom:99px}} +@media(max-width:700px){.neural-panel{top:90px;bottom:95px;right:12px;width:calc(100vw - 24px);padding:18px}.neural-open .intro,.neural-open #fly-panel{visibility:hidden}.neural-heading h2{font-size:23px}} diff --git a/web/neural.js b/web/neural.js new file mode 100644 index 0000000..a8810ef --- /dev/null +++ b/web/neural.js @@ -0,0 +1,192 @@ +const $ = (id) => document.getElementById(id); +const svgNS = 'http://www.w3.org/2000/svg'; +const format = (number) => Number(number || 0).toLocaleString('en-US'); + +function svgElement(tag, attributes = {}) { + const element = document.createElementNS(svgNS, tag); + for (const [name, value] of Object.entries(attributes)) element.setAttribute(name, value); + return element; +} + +export function createNeuralPanel(descriptor, command) { + const stylesheet = document.createElement('link'); + stylesheet.rel = 'stylesheet'; + stylesheet.href = '/neural.css'; + document.head.append(stylesheet); + const panel = $('neural-panel'), open = $('neural-open'); + const buttons = [...panel.querySelectorAll('[data-neural-mode]')]; + let state = null, rendered = -1, pending = false, connected = true; + const nodeElements = new Map(), readoutElements = new Map(); + open.hidden = false; + + function show(value) { + panel.hidden = !value; + document.body.classList.toggle('neural-open', value); + open.setAttribute('aria-expanded', String(value)); + if (value) { + render(); + $('neural-close').focus(); + } else open.focus(); + } + open.addEventListener('click', () => show(panel.hidden)); + $('neural-close').addEventListener('click', () => show(false)); + panel.addEventListener('keydown', (event) => { + if (event.key === 'Escape') { event.stopPropagation(); show(false); } + }); + document.addEventListener('visibilitychange', () => { if (!document.hidden) render(); }); + + if (!descriptor.available) { + $('neural-status').textContent = descriptor.error || 'Neural reference unavailable.'; + $('neural-content').hidden = true; + return { update() {}, setConnected() {} }; + } + $('neural-dataset').textContent = `${descriptor.dataset} · ${format(descriptor.neuron_count)} neurons`; + $('neural-graph-size').textContent = `${format(descriptor.edge_count)} directed connections in the simulation`; + $('neural-protocol').textContent = `${descriptor.input_count} antennal neurons · ${descriptor.rate_hz} Hz input for ${descriptor.stimulus_ms} ms, then 50 ms recovery. Each trial starts from rest with the same seed.`; + $('neural-provenance').href = `https://github.com/philshiu/Drosophila_brain_model/tree/${descriptor.reference_commit}`; + const diagram = $('neural-graph'); + const nodes = descriptor.diagram.nodes; + const readouts = new Set(descriptor.readouts.map((item) => item.index)); + const columns = [ + nodes.filter((item) => item.role === 'sensory'), + nodes.filter((item) => item.role !== 'sensory' && !readouts.has(item.index)), + nodes.filter((item) => readouts.has(item.index)), + ]; + const positions = new Map(); + columns.forEach((items, column) => { + const title = svgElement('text', { x: 62 + column * 148, y: 15, 'text-anchor': 'middle', class: 'graph-column' }); + title.textContent = ['INPUT', 'NETWORK', 'READOUTS'][column]; + diagram.append(title); + items.forEach((item, row) => { + positions.set(item.index, { x: 62 + column * 148, y: 34 + (row + .5) * 250 / items.length }); + }); + }); + for (const edge of descriptor.diagram.edges) { + const from = positions.get(edge.source), to = positions.get(edge.target); + const path = svgElement('path', { + d: `M${from.x},${from.y} C${from.x + 44},${from.y} ${to.x - 44},${to.y} ${to.x},${to.y}`, + class: edge.count > 0 ? 'graph-excitatory' : 'graph-inhibitory', + 'marker-end': edge.count > 0 ? 'url(#neural-arrow)' : 'url(#neural-stop)', + }); + const title = svgElement('title'); + title.textContent = `${edge.source} → ${edge.target}: ${edge.count} signed synapses`; + path.append(title); + diagram.append(path); + } + for (const item of nodes) { + const { x, y } = positions.get(item.index); + const group = svgElement('g', { class: 'graph-node' }); + const circle = svgElement('circle', { cx: x, cy: y, r: 5 }); + const label = svgElement('text', { x: x + 10, y: y + 3.5 }); + label.textContent = item.label; + const title = svgElement('title'); + title.textContent = `FlyWire ID ${item.id}`; + group.append(circle, label, title); + diagram.append(group); + nodeElements.set(item.index, { group, circle, title, item }); + } + $('neural-diagram-note').textContent = `${nodes.length} selected neurons · actual connections, schematic positions. Color and size show accumulated trial spikes.`; + for (const item of descriptor.readouts) { + const row = document.createElement('tr'); + const name = document.createElement('th'); + name.scope = 'row'; + name.textContent = item.label; + name.title = `${item.role} · FlyWire ID ${item.id}`; + const spikes = document.createElement('td'), voltage = document.createElement('td'); + spikes.textContent = voltage.textContent = '—'; + row.append(name, spikes, voltage); + $('neural-readouts').append(row); + readoutElements.set(item.index, { spikes, voltage }); + } + + function controls() { + for (const button of buttons) { + button.disabled = !connected || pending || state?.status === 'running'; + button.setAttribute('aria-pressed', String(state?.mode === button.dataset.neuralMode)); + } + } + for (const button of buttons) { + button.addEventListener('click', async () => { + if (pending) return; + pending = true; + controls(); + try { await command({ action: 'neural_trial', mode: button.dataset.neuralMode }); } + finally { pending = false; controls(); } + }); + } + + function drawHistory(history) { + const canvas = $('neural-timeline'), context = canvas.getContext('2d'); + const width = canvas.width, height = canvas.height; + context.clearRect(0, 0, width, height); + context.font = '10px monospace'; + const rows = [['input', '#a6d7ca', 'Input'], ['network', '#e3be80', 'Network'], + ['descending', '#b5b3e6', 'Descending']]; + rows.forEach(([key, color, label], row) => { + const top = row * 43 + 5, base = top + 27; + context.fillStyle = '#aab8bc'; + context.fillText(label, 0, top + 16); + context.strokeStyle = '#c3d9dc26'; + context.beginPath(); context.moveTo(85, base); context.lineTo(width - 8, base); context.stroke(); + const maximum = Math.max(1, ...history.map((bin) => bin[key])); + const step = (width - 93) / 30; + context.fillStyle = color; + history.forEach((bin, i) => { + const h = bin[key] / maximum * 24; + context.fillRect(85 + i * step, base - h, Math.max(1, step - 2), h); + }); + }); + context.fillStyle = '#aab8bc'; + context.fillText('0', 85, height - 3); + context.fillText('100', width - 138, height - 3); + context.fillText('150 ms', width - 46, height - 3); + context.setLineDash([3, 4]); + context.strokeStyle = '#c3d9dc55'; + const end = 85 + (width - 93) * 2 / 3; + context.beginPath(); context.moveTo(end, 0); context.lineTo(end, 128); context.stroke(); + context.setLineDash([]); + } + + function render() { + if (panel.hidden || document.hidden || !state || rendered === state.revision) return; + rendered = state.revision; + const idle = state.status === 'idle'; + const names = { stimulus: 'Antennal stimulus', baseline: 'No input', blocked: 'Sensory output blocked' }; + $('neural-status').textContent = idle ? 'Ready for a reference trial.' + : state.error || `${names[state.mode]} · ${state.status === 'running' ? 'Running' : state.status === 'complete' ? 'Complete' : state.status}`; + $('neural-progress').value = state.simulated_ms || 0; + $('neural-time').textContent = `${(state.simulated_ms || 0).toFixed(0)} / ${descriptor.total_ms} ms neural time`; + $('neural-wall').textContent = idle ? 'Runs on demand' : `${(state.wall_seconds || 0).toFixed(2)} s elapsed · ${(state.cpu_seconds || 0).toFixed(3)} CPU s`; + $('neural-spikes').textContent = idle ? '—' : format(state.total_spikes); + $('neural-downstream').textContent = idle ? '—' : format(state.downstream_spikes); + $('neural-active').textContent = idle ? '—' : format(state.neurons_that_spiked); + const measurements = new Map((state.nodes || []).map((node) => [node.index, node])); + for (const [index, element] of nodeElements) { + const value = measurements.get(index); + const count = value?.spikes || 0; + element.group.classList.toggle('has-spikes', count > 0); + element.circle.setAttribute('r', String(5 + Math.min(4, Math.log2(1 + count)))); + element.title.textContent = `FlyWire ID ${element.item.id} · ${count} trial spikes${value ? ` · ${value.voltage_mv.toFixed(2)} mV` : ''}`; + } + for (const [index, cells] of readoutElements) { + const value = (state.readouts || []).find((item) => item.index === index); + cells.spikes.textContent = value ? format(value.spikes) : '—'; + cells.voltage.textContent = value ? `${value.voltage_mv.toFixed(2)} mV` : '—'; + } + drawHistory(state.history || []); + const descending = (state.readouts || []).filter((item) => item.role === 'descending') + .reduce((sum, item) => sum + item.spikes, 0); + $('workflow-readout').classList.toggle('measured', descending > 0); + $('workflow-brain').classList.toggle('measured', state.downstream_spikes > 0); + } + + return { + update(next) { + if (!next || next.revision === state?.revision) return; + state = next; + controls(); + render(); + }, + setConnected(value) { connected = value; controls(); }, + }; +} diff --git a/web/style.css b/web/style.css new file mode 100644 index 0000000..418d4c9 --- /dev/null +++ b/web/style.css @@ -0,0 +1,16 @@ +:root{color-scheme:dark;font-family:Inter,"Segoe UI",Arial,sans-serif;color:#e5edeb;background:#11191d;font-synthesis:none;--muted:#aab8bc;--green:#b5d4bf;--line:#c3d9dc26} +*{box-sizing:border-box}body{margin:0;overflow:hidden}button,a,select{-webkit-tap-highlight-color:transparent}button,select{font:inherit}button{cursor:pointer;color:inherit}button:disabled{opacity:.45;cursor:wait}button:focus-visible,a:focus-visible,select:focus-visible,canvas:focus-visible{outline:2px solid var(--green);outline-offset:5px}button{transition:background .2s,color .2s,border-color .2s}button:hover{background:#e8f6f018}button.selected{color:#ecf7ef;background:#bed3c21c;border-color:#c3d8cd70}#observatory,#world,.vignette{position:fixed;inset:0;width:100%;height:100%}#world{display:block;touch-action:none;cursor:grab;outline:none}#world:active{cursor:grabbing}.vignette{pointer-events:none;background:linear-gradient(180deg,#0a121c95,transparent 35%,transparent 65%,#071015c4),linear-gradient(90deg,#09141a4d,transparent 60%)} +header{position:absolute;top:0;left:0;right:0;height:109px;display:flex;align-items:center;padding:0 42px;gap:22px;border-bottom:1px solid #dbf0ef12;background:linear-gradient(#101a2070,transparent)}.brand{display:flex;align-items:center;gap:15px;color:#edf3ec;text-decoration:none;font-size:19px;font-weight:650;letter-spacing:4px}.brand svg{width:30px;height:36px;fill:none;stroke:var(--green);stroke-width:1.15}.brand small{display:block;letter-spacing:2.6px;font-size:8px;color:#adbdc1;font-weight:400;margin-top:7px}.connection{margin-left:auto;font-size:11px;color:#c4d5ce;display:flex;gap:9px;align-items:center}.dot{width:6px;height:6px;background:#d3b384;border-radius:50%;box-shadow:0 0 12px #d3b38466}.dot.live{background:#b6d9b9;box-shadow:0 0 12px #b6d9b966}.dot.offline{background:#e0a48c}.icon-button{border:1px solid var(--line);background:#15212659;border-radius:5px;width:36px;height:36px;font-size:23px;display:grid;place-items:center} +.intro{position:absolute;top:151px;left:43px;max-width:500px;pointer-events:none}.eyebrow{font-size:9px;letter-spacing:2.1px;color:#bdcfcc;display:flex;align-items:center;gap:12px}.hairline{height:1px;width:27px;background:#b3cfcc7d}h1{font-weight:400;font-size:46px;letter-spacing:-1.8px;line-height:1.12;margin:18px 0 12px;text-shadow:0 2px 24px #0710169c}.intro p{font-size:12px;color:#bdc9cd;margin:0;line-height:1.65}.view-switcher{display:flex;gap:6px;margin-top:25px;pointer-events:auto}.view-switcher button{border:1px solid var(--line);border-radius:4px;background:#0c18275e;font-size:10px;padding:11px 13px;color:#c0cbce;backdrop-filter:blur(10px)}.view-switcher button span{font-size:8px;opacity:.6;margin-right:7px} +.map-panel{position:absolute;right:40px;top:151px;width:223px;background:#101c229e;border:1px solid var(--line);border-radius:6px;backdrop-filter:blur(14px);overflow:hidden}.panel-heading{display:flex;justify-content:space-between;padding:17px 17px 0;font-size:8px;letter-spacing:1.7px;color:#b4c8cc}.north{color:var(--green)}#minimap{width:100%;height:164px;display:block}.coordinates{display:flex;justify-content:space-between;font:9px monospace;padding:0 16px 15px;color:#9db2b8}.world-facts{padding:15px 16px;border-top:1px solid var(--line);display:flex;flex-direction:column;gap:10px}.world-facts>span{display:flex;justify-content:space-between;color:#aebdc1;font-size:10px}.world-facts strong{font:10px monospace;color:#d0deda}.organism{display:flex;gap:9px;padding:16px 14px;border-top:1px solid var(--line);font-size:10px;color:#c6d0ce;align-items:flex-start;background:#0b141629}.hollow-dot{width:6px;height:6px;border:1px solid #adbbb6;border-radius:50%;margin-top:3px;flex-shrink:0}.organism small{display:block;font-size:9px;color:#93a7af;margin-top:7px} +.lab-panel{position:absolute;bottom:121px;left:43px;width:253px;border-left:1px solid #bfd3d247;padding:3px 0 0 18px}.lab-panel .eyebrow{font-size:8px;letter-spacing:1.7px}h2{font-size:18px;font-weight:450;letter-spacing:-.4px;margin:10px 0}.lab-panel p{font-size:11px;line-height:1.7;color:#b0c0c7;max-width:220px;margin:0 0 16px}.primary{width:100%;padding:12px 13px;background:#c0d5c5;border:1px solid #d7e2d5;color:#182b25;border-radius:4px;font-size:11px;display:flex;align-items:center;gap:10px;font-weight:600}.primary span:last-child{margin-left:auto}.primary:hover{background:#d6e5d8}.lab-panel .probe-status{font-size:9px;color:#b8c9c6;margin:10px 0 0}.probe-status.contact{color:#e2c792} +footer{position:absolute;bottom:0;left:0;right:0;min-height:83px;border-top:1px solid var(--line);background:#0b171dae;backdrop-filter:blur(16px);padding:17px 40px;display:flex;align-items:center;justify-content:space-between;gap:24px}.clock-controls{display:flex;align-items:center;gap:16px}.clock{min-width:106px}.clock span{display:block;font-size:7px;letter-spacing:1.8px;color:#9fb0b7;margin-bottom:6px}.clock strong{font:17px monospace;color:#dfe9e8}.speed-control select{border:1px solid var(--line);border-radius:3px;background:#1a282e;color:#c1d0d1;font-size:10px;padding:8px;max-width:167px}.navigation-help{display:flex;gap:15px;align-items:center;font-size:9px;color:#b1c1c6}.navigation-help span{white-space:nowrap}kbd{font:8px monospace;letter-spacing:1px;border:1px solid #b5cbce40;padding:5px 6px;margin-right:5px;border-radius:3px;color:#c9d6d7}.performance{font:9px monospace;display:flex;flex-direction:column;gap:8px;min-width:69px;text-align:right;color:#a8bdc0}.performance span:first-child{color:var(--green)} +#loading{position:absolute;inset:0;display:flex;flex-direction:column;align-items:center;justify-content:center;gap:17px;background:#101c24;z-index:10;font-size:12px;color:#a9c1c6}#loading[hidden]{display:none}#loading strong{font-size:20px;font-weight:400;color:#e0ede6}.loader{width:28px;height:28px;border:1px solid #93baab35;border-top-color:#bed8c7;border-radius:50%;animation:spin 1s linear infinite}@keyframes spin{to{transform:rotate(360deg)}}#toast{position:absolute;bottom:108px;left:50%;transform:translateX(-50%);padding:12px 18px;border:1px solid var(--line);border-radius:4px;background:#13242ced;color:#e1ebe9;font-size:12px;max-width:90%;text-align:center}#crosshair{position:absolute;top:50%;left:50%;width:4px;height:4px;border:1px solid #d5eee475;border-radius:50%;pointer-events:none}.mobile-movement{display:none}.sr-only{position:absolute;width:1px;height:1px;overflow:hidden;clip:rect(0,0,0,0)} +@media(min-width:1600px){.intro{top:180px}h1{font-size:56px}.map-panel{top:180px}.lab-panel{bottom:140px}} +@media(max-width:1100px){.navigation-help{display:none}.map-panel{width:197px;right:25px}header,footer{padding-left:25px;padding-right:25px}.intro,.lab-panel{left:26px}} +@media(max-width:700px){header{height:79px;padding:0 19px;gap:12px}.brand{font-size:15px;letter-spacing:2.8px;gap:10px}.brand svg{width:23px}.brand small{font-size:6px;letter-spacing:1.9px}.connection{font-size:0;gap:0}.icon-button{width:32px;height:32px}.intro{top:110px;left:20px;right:20px}h1{font-size:35px;margin:14px 0 9px}.intro p{font-size:11px}.view-switcher{margin-top:20px}.view-switcher button{padding:10px 11px}.map-panel{display:none}.lab-panel{left:20px;bottom:155px;width:209px;padding-left:13px}.lab-panel h2{font-size:16px}.lab-panel p{font-size:10px}.lab-panel .eyebrow{font-size:7px}.primary{font-size:10px;padding:11px}footer{padding:14px 17px;min-height:80px;gap:8px}.clock-controls{gap:10px}.clock{min-width:85px}.clock strong{font-size:14px}.clock span{font-size:6px;letter-spacing:1px}.speed-control select{font-size:9px;width:91px;padding:7px 4px}.performance{min-width:53px;font-size:8px}.mobile-movement{position:absolute;bottom:98px;left:20px;display:flex;gap:8px}.mobile-movement button{background:#15252ad9;border:1px solid var(--line);border-radius:4px;width:39px;height:36px;touch-action:none}.lab-panel>p:not(.probe-status){max-width:192px}#toast{bottom:145px}} +@media(max-height:650px) and (min-width:701px){header{height:75px}.intro,.map-panel{top:99px}h1{font-size:32px}.lab-panel{bottom:104px}.lab-panel>p:not(.probe-status){display:none}.map-panel{width:181px}#minimap{height:125px}.organism{padding:10px}.world-facts{padding:10px 16px}} +@media(prefers-reduced-motion:reduce){*{animation:none!important;transition:none!important}} +[hidden]{display:none!important}.organism{width:100%;text-align:left;border:0;border-top:1px solid var(--line)}.specimen-facts{display:flex;flex-direction:column;gap:11px;font-size:10px;max-width:230px}.specimen-facts>span{display:flex;justify-content:space-between;gap:12px;color:#aebdc1}.specimen-facts strong{font-weight:400;color:#d0deda}#fly-marker{position:absolute;left:0;top:0;transform:translate(-50%,-50%);border:0;background:#13242ccd;color:#cee4cf;padding:6px 9px;border-radius:4px;font-size:12px}#fly-marker span{font-size:9px;margin-left:5px}.fly-view #crosshair,.fly-view .mobile-movement{display:none}.fly-view .navigation-help{font-size:10px}.fly-view .lab-panel{background:#101c2280;backdrop-filter:blur(8px);padding:15px 18px;border-radius:4px} +@media(max-width:700px){.view-switcher{gap:4px}.view-switcher button{padding:10px 9px}.view-switcher button span{display:none}.fly-view .lab-panel{bottom:104px;width:218px;padding:12px 14px}.fly-view .lab-panel h2{margin:7px 0}.fly-view .lab-panel p{margin-bottom:10px}.specimen-facts{font-size:9px;gap:7px}.fly-view .lab-panel .probe-status{margin:10px 0 0}} +.view-switcher button.selected{color:#ecf7ef;background:#bed3c22e;border-color:#c3d8cd70}