From 0a9cb8e93bd72dd1bba714bccc18a62f3ad90f40 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Tue, 16 Jun 2026 15:30:15 -0300 Subject: [PATCH 1/3] feat(plugin): in-plugin autoencoder training + dataset building (Stage 3) Port the upstream Python training pipeline into the live Rust plugin so users can build a dataset and train the autoencoder in-session, then play their own model. Previously only the frozen decoder shipped; training was offline-only. New modules: - autoencoder.rs: pure f64 full AE (encoder+decoder), manual backprop, Adam, BatchNorm. Fixes two upstream bugs: broken Adam bias correction (per-param step counter t) and no batch shuffle. Converges better; not a numeric match. - model_ops.rs: op-list export byte-compatible with decoder.json / train_export.py, so a trained model loads into decoder.rs unchanged. - dataset.rs: encode_onsets (exact corpus_encode.py port incl. banker's rounding) + encode_grid (capture live pattern). - audio.rs: symphonia decode + spectral-flux onset detection (rustfft). - training.rs: TrainShared + background task executor + arc-swap hot-swap. Wiring: - Decoder hot-swaps via ArcSwap (audio thread wait-free; never locks a Mutex). model_generation invalidates the regen cache on swap. - Trained model persists in DAW state via #[persist] trained_model; baked decoder.json stays the default/fallback. Restored in initialize(). - SharedState gains per-step substeps so the GUI can capture/encode patterns. - editor.rs: Training panel (capture / add audio / train / cancel / progress + loss / encode pattern->latent). Deps: arc-swap, rustfft, rfd, symphonia (wav+flac). Tests: 34 cargo tests (AE overfit gradient gate, Adam-fix, shuffle determinism, export<->decoder parity, corpus-encode parity with Python cases, onset sanity, end-to-end train->swap->persist->encode). clippy -D warnings clean; CLAP + VST3 bundle builds; both headless host scale-tests pass. --- README.md | 56 +- deepsteps-plugin/Cargo.lock | 1231 ++++++++++++++++- deepsteps-plugin/Cargo.toml | 4 + deepsteps-plugin/src/audio.rs | 262 ++++ deepsteps-plugin/src/autoencoder.rs | 683 +++++++++ deepsteps-plugin/src/dataset.rs | 177 +++ deepsteps-plugin/src/editor.rs | 165 ++- deepsteps-plugin/src/lib.rs | 66 +- deepsteps-plugin/src/model_ops.rs | 192 +++ deepsteps-plugin/src/params.rs | 12 +- deepsteps-plugin/src/shared.rs | 20 +- deepsteps-plugin/src/training.rs | 285 ++++ .../2026-06-16-runtime-training-design.md | 87 ++ 13 files changed, 3182 insertions(+), 58 deletions(-) create mode 100644 deepsteps-plugin/src/audio.rs create mode 100644 deepsteps-plugin/src/autoencoder.rs create mode 100644 deepsteps-plugin/src/dataset.rs create mode 100644 deepsteps-plugin/src/model_ops.rs create mode 100644 deepsteps-plugin/src/training.rs create mode 100644 docs/plans/2026-06-16-runtime-training-design.md diff --git a/README.md b/README.md index 99217c2..053ebef 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,8 @@ blog post [here](https://mct-master.github.io/masters-thesis/2024/05/14/alexanjw | Stage | What | State | |-------|------|-------| | **Stage 1** | Original openFrameworks standalone, building/running on Linux x86_64 (runtime only — offline training UI disabled) | Builds & runs on CachyOS/Arch with gcc 16 + openFrameworks 0.12.1. See [`docs/BUILDING-linux.md`](docs/BUILDING-linux.md). | -| **Stage 2** | Rust rewrite as a CLAP + VST3 **MIDI-generator** plugin (nih-plug). Reuses no C++/Pd/Python at runtime. | Working. 16 cargo + 8 pytest tests, clippy clean, `clap-validator` 18/0/3, `pluginval` (VST3) SUCCESS, and headless CLAP + VST3 host tests passing all 14 scales. CI green. | +| **Stage 2** | Rust rewrite as a CLAP + VST3 **MIDI-generator** plugin (nih-plug). Reuses no C++/Pd/Python at runtime. | Working. 34 cargo + 8 pytest tests, clippy clean, `clap-validator` 18/0/3, `pluginval` (VST3) SUCCESS, and headless CLAP + VST3 host tests passing all 14 scales. CI green. | +| **Stage 3** | **In-plugin training**: build a dataset and train the autoencoder live in the DAW, then play *your* model. The Python training pipeline reimplemented from scratch in Rust. | Working. Trains off the audio thread, hot-swaps the result, persists with the session. See [`docs/plans/2026-06-16-runtime-training-design.md`](docs/plans/2026-06-16-runtime-training-design.md). | The two stages share no runtime code. Stage 1 is the behavioural reference; Stage 2 is the plugin you actually install in a DAW. @@ -30,14 +31,39 @@ A **MIDI generator**: it emits notes; your host/synth makes the sound. It has ** internal clock** — it follows the **host transport** (tempo + playhead). Press play in your DAW and it sequences. -**How it works.** A frozen, offline-trained autoencoder **decoder** turns 4 latent -parameters into a 16-step pattern (which steps fire + a per-step "groove" sub-step -offset). The sequencer plays that pattern at 4 steps/beat (16 per bar), quantising -each step's pitch to a selected scale + key. +**How it works.** An autoencoder **decoder** turns 4 latent parameters into a 16-step +pattern (which steps fire + a per-step "groove" sub-step offset). The sequencer plays +that pattern at 4 steps/beat (16 per bar), quantising each step's pitch to a selected +scale + key. The plugin ships with a frozen, offline-trained decoder as the default, and +you can **train your own model in-session** (see *Training*, below). **Custom GUI** (egui editor): a 16-step grid with a live playhead and click-to-toggle cells (a click forces a step on/off, overriding the decoder until the next latent-driven -regeneration), plus sliders for the latent vector, per-step pitches, timing, and tuning. +regeneration), plus sliders for the latent vector, per-step pitches, timing, and tuning, +and a **Training** panel. + +## Training (in-plugin) + +The plugin reimplements the original's autoencoder *training* in pure Rust — no Python, +no offline step. In the **Training** panel: + +1. **Build a dataset.** *Capture pattern* snapshots the current 16-step grid (steps + + sub-step offsets) as a training sample, and/or *Add audio…* loads `.wav`/`.flac` files, + detects onsets (spectral-flux), and encodes each file as a sample. The dataset + accumulates across both sources. +2. **Train.** Set epochs/batch and press *Train*. Training runs on a background thread + (never the audio thread), showing a live epoch/loss progress bar; *Cancel* stops it. +3. **Play your model.** On finish the new decoder is **hot-swapped** into the audio path + (lock-free) and drives the latent sliders immediately. *Encode pattern → latent* runs + the encoder on the current grid to set the 4 latents to that pattern's latent code. + +The trained model is **saved with the DAW session** (and travels with presets); reloading +restores it. The baked default decoder remains the fallback when no model has been trained. + +> Faithfulness note: the Rust training fixes two bugs in the original Python (broken Adam +> bias correction; no batch shuffle), so it converges better but does not reproduce the +> Python numerically. The offline Python pipeline under `Deep_Steps_project/tools/` still +> exists for reference. **Parameters** (also host-automatable): Latent A–D, Gate length (ms), Sub-step scale, Sequence length (1–16), Key (0–11), Scale (14 options: Chromatic, @@ -93,14 +119,14 @@ clock and sequences off **incoming MIDI clock**. - **Step toggles are not preset-persisted.** Grid clicks override the decoder at runtime but are not saved in presets (they are runtime state, not params) and a latent move regenerates over them. Promoting them to params is a possible later pass. -- **Shipped weights are from a synthetic dataset.** The original never shipped trained - weights (it random-inits and only becomes meaningful after in-session training). - This port freezes an **offline-trained** decoder, but the committed - `deepsteps-plugin/weights/decoder.json` was trained on a deterministic *synthetic* - corpus (`Deep_Steps_project/tools/make_synth_dataset.py`), so patterns are - reproducible but not musically trained. Train your own from audio with - `Deep_Steps_project/tools/build_dataset.py` + `Deep_Steps_project/tools/train_export.py` - (uses [librosa](https://librosa.org) for onset detection). +- **Default weights are from a synthetic dataset.** The committed default decoder + `deepsteps-plugin/weights/decoder.json` was trained offline on a deterministic + *synthetic* corpus (`Deep_Steps_project/tools/make_synth_dataset.py`), so out-of-the-box + patterns are reproducible but not musically trained. Train your own **in the plugin** + (see *Training*) — or, offline, with the Python tools + `Deep_Steps_project/tools/build_dataset.py` + `train_export.py` ([librosa](https://librosa.org) + onsets). The in-plugin onset detector is spectral-flux based and intentionally not a + librosa clone, so audio-derived datasets are approximate. - **Two sequencer timing approximations** (flagged for A/B in [`deepsteps-plugin/NOTES-sequencer.md`](deepsteps-plugin/NOTES-sequencer.md) and [`VALIDATION.md`](deepsteps-plugin/VALIDATION.md)): the sub-step offset uses a @@ -111,7 +137,7 @@ clock and sequences off **incoming MIDI clock**. ## Validation See [`deepsteps-plugin/VALIDATION.md`](deepsteps-plugin/VALIDATION.md). Automated: -`cargo test` (16), `clap-validator` (18/0/3) and `pluginval` (VST3, strictness 8, +`cargo test` (34), `clap-validator` (18/0/3) and `pluginval` (VST3, strictness 8, SUCCESS), plus headless host scale tests that load the **shipped** binaries and assert all 14 scales quantise correctly through both plugin formats — `clap-host-test/` (CLAP) and `vst3-host-test/` (VST3). All run in CI on every push/PR. diff --git a/deepsteps-plugin/Cargo.lock b/deepsteps-plugin/Cargo.lock index 99343db..7963ff6 100644 --- a/deepsteps-plugin/Cargo.lock +++ b/deepsteps-plugin/Cargo.lock @@ -82,12 +82,198 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "170433209e817da6aae2c51aa0dd443009a613425dd041ebfb2492d1c4c11a25" +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "as-raw-xcb-connection" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "175571dd1d178ced59193a6fc02dde1b972eb0bc56c892cde9beeceac5bf0f6b" +[[package]] +name = "ashpd" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3af990a617932d416e83cf79e7335dd5247dcb0825995ca3274c17dab5b749d" +dependencies = [ + "async-fs", + "async-net", + "enumflags2", + "futures-channel", + "futures-util", + "rand", + "serde", + "serde_repr", + "url", + "zbus", +] + +[[package]] +name = "async-broadcast" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435a87a52755b8f27fcf321ac4f04b2802e337c8c4872923137471ec39c37532" +dependencies = [ + "event-listener", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-executor" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "pin-project-lite", + "slab", +] + +[[package]] +name = "async-fs" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034a681df4aed8b8edbd7fbe472401ecf009251c8b40556b304567052e294c5" +dependencies = [ + "async-lock", + "blocking", + "futures-lite", +] + +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix 1.1.4", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-net" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b948000fad4873c1c9339d60f2623323a0cfd3816e5181033c6a5cb68b2accf7" +dependencies = [ + "async-io", + "blocking", + "futures-lite", +] + +[[package]] +name = "async-process" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc50921ec0055cdd8a16de48773bfeec5c972598674347252c0399676be7da75" +dependencies = [ + "async-channel", + "async-io", + "async-lock", + "async-signal", + "async-task", + "blocking", + "cfg-if", + "event-listener", + "futures-lite", + "rustix 1.1.4", +] + +[[package]] +name = "async-recursion" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "async-signal" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52b5aaafa020cf5053a01f2a60e8ff5dccf550f0f77ec54a4e47285ac2bab485" +dependencies = [ + "async-io", + "async-lock", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix 1.1.4", + "signal-hook-registry", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -146,7 +332,7 @@ dependencies = [ "cocoa", "core-foundation", "keyboard-types", - "nix", + "nix 0.22.3", "objc", "raw-window-handle 0.5.2", "uuid", @@ -173,6 +359,15 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block2" version = "0.5.1" @@ -182,6 +377,19 @@ dependencies = [ "objc2", ] +[[package]] +name = "blocking" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -413,6 +621,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "crossbeam" version = "0.8.4" @@ -469,6 +686,16 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "cursor-icon" version = "1.2.0" @@ -479,10 +706,14 @@ checksum = "f27ae1dd37df86211c42e150270f82743308803d90a6f6e6651cd730d5e1732f" name = "deepsteps-plugin" version = "0.1.1" dependencies = [ + "arc-swap", "nih_plug", "nih_plug_egui", + "rfd", + "rustfft", "serde", "serde_json", + "symphonia", ] [[package]] @@ -491,12 +722,33 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dispatch" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b" +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "dlib" version = "0.5.3" @@ -579,6 +831,42 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "endi" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099" + +[[package]] +name = "enumflags2" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef" +dependencies = [ + "enumflags2_derive", + "serde", +] + +[[package]] +name = "enumflags2_derive" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "epaint" version = "0.31.1" @@ -624,6 +912,39 @@ version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -672,12 +993,66 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", +] + [[package]] name = "futures-core" version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + [[package]] name = "futures-task" version = "0.3.32" @@ -691,11 +1066,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "gethostname" version = "1.1.0" @@ -779,6 +1168,115 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -902,6 +1400,12 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.186" @@ -942,6 +1446,12 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + [[package]] name = "lock_api" version = "0.4.14" @@ -1129,18 +1639,58 @@ dependencies = [ "memoffset 0.6.5", ] +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.0", + "cfg-if", + "cfg_aliases", + "libc", + "memoffset 0.9.1", +] + [[package]] name = "nohash-hasher" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "num_enum" version = "0.7.6" @@ -1181,6 +1731,17 @@ dependencies = [ "malloc_buf", ] +[[package]] +name = "objc-foundation" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1add1b659e36c9607c7aab864a76c7a4c2760cd0cd2e120f3fb8b952c7e22bf9" +dependencies = [ + "block", + "objc", + "objc_id", +] + [[package]] name = "objc-sys" version = "0.3.5" @@ -1384,6 +1945,15 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "objc_id" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92d4ddb4bd7b50d730c215ff871754d0da6b2178849f8a2a2ab69712d0c073b" +dependencies = [ + "objc", +] + [[package]] name = "object" version = "0.37.3" @@ -1420,6 +1990,16 @@ dependencies = [ "libredox", ] +[[package]] +name = "ordered-stream" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aa2b01e1d916879f73a53d01d1d6cee68adbb31d6d9177a8cfce093cced1d50" +dependencies = [ + "futures-core", + "pin-project-lite", +] + [[package]] name = "owned_ttf_parser" version = "0.25.1" @@ -1429,6 +2009,12 @@ dependencies = [ "ttf-parser", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.5" @@ -1490,6 +2076,17 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + [[package]] name = "pkg-config" version = "0.3.33" @@ -1516,12 +2113,45 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "pollster" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22686f4785f02a4fcc856d3b3bb19bf6c8160d103f7a99cc258bddd0251dc7f2" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + [[package]] name = "powerfmt" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-crate" version = "3.5.0" @@ -1561,6 +2191,36 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + [[package]] name = "raw-window-handle" version = "0.5.2" @@ -1609,6 +2269,29 @@ dependencies = [ "winapi", ] +[[package]] +name = "rfd" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a73a7337fc24366edfca76ec521f51877b114e42dab584008209cca6719251" +dependencies = [ + "ashpd", + "block", + "dispatch", + "js-sys", + "log", + "objc", + "objc-foundation", + "objc_id", + "pollster", + "raw-window-handle 0.6.2", + "urlencoding", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows-sys 0.48.0", +] + [[package]] name = "rustc-demangle" version = "0.1.27" @@ -1624,6 +2307,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rustix" version = "0.38.44" @@ -1744,6 +2441,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "serde_spanned" version = "0.6.9" @@ -1754,55 +2462,166 @@ dependencies = [ ] [[package]] -name = "shlex" -version = "2.0.1" +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "slotmap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038" +dependencies = [ + "version_check", +] + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "smol_str" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd538fb6910ac1099850255cf94a94df6551fbdd602454387d0adb2d1ca6dead" +dependencies = [ + "serde", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] -name = "simd_cesu8" -version = "1.1.1" +name = "static_assertions" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + +[[package]] +name = "symphonia" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5773a4c030a19d9bfaa090f49746ff35c75dfddfa700df7a5939d5e076a57039" dependencies = [ - "rustc_version", - "simdutf8", + "lazy_static", + "symphonia-bundle-flac", + "symphonia-core", + "symphonia-format-riff", + "symphonia-metadata", ] [[package]] -name = "simdutf8" -version = "0.1.5" +name = "symphonia-bundle-flac" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +checksum = "c91565e180aea25d9b80a910c546802526ffd0072d0b8974e3ebe59b686c9976" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] [[package]] -name = "slab" -version = "0.4.12" +name = "symphonia-core" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" +checksum = "ea00cc4f79b7f6bb7ff87eddc065a1066f3a43fe1875979056672c9ef948c2af" +dependencies = [ + "arrayvec", + "bitflags 1.3.2", + "bytemuck", + "lazy_static", + "log", +] [[package]] -name = "slotmap" -version = "1.1.1" +name = "symphonia-format-riff" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038" +checksum = "c2d7c3df0e7d94efb68401d81906eae73c02b40d5ec1a141962c592d0f11a96f" dependencies = [ - "version_check", + "extended", + "log", + "symphonia-core", + "symphonia-metadata", ] [[package]] -name = "smallvec" -version = "1.15.2" +name = "symphonia-metadata" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +checksum = "36306ff42b9ffe6e5afc99d49e121e0bd62fe79b9db7b9681d48e29fa19e6b16" +dependencies = [ + "encoding_rs", + "lazy_static", + "log", + "symphonia-core", +] [[package]] -name = "smol_str" -version = "0.2.2" +name = "symphonia-utils-xiph" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd538fb6910ac1099850255cf94a94df6551fbdd602454387d0adb2d1ca6dead" +checksum = "ee27c85ab799a338446b68eec77abf42e1a6f1bb490656e121c6e27bfbab9f16" dependencies = [ - "serde", + "symphonia-core", + "symphonia-metadata", ] [[package]] @@ -1827,6 +2646,30 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix 1.1.4", + "windows-sys 0.61.2", +] + [[package]] name = "termcolor" version = "1.4.1" @@ -1908,6 +2751,16 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "toml" version = "0.7.8" @@ -1979,14 +2832,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "tracing-core" version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] [[package]] name = "ttf-parser" @@ -1994,6 +2872,23 @@ version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "uds_windows" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" +dependencies = [ + "memoffset 0.9.1", + "tempfile", + "windows-sys 0.61.2", +] + [[package]] name = "unicode-ident" version = "1.0.24" @@ -2006,6 +2901,31 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", + "serde_derive", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "uuid" version = "0.8.2" @@ -2210,6 +3130,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -2252,6 +3181,21 @@ dependencies = [ "windows_x86_64_msvc 0.42.2", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -2274,6 +3218,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -2286,6 +3236,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -2298,6 +3254,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -2316,6 +3278,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -2328,6 +3296,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -2340,6 +3314,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -2352,6 +3332,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -2426,6 +3412,12 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + [[package]] name = "x11" version = "2.21.0" @@ -2485,6 +3477,16 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec9e4a500ca8864c5b47b8b482a73d62e4237670e5b5f1d6b9e3cae50f28f2b" +[[package]] +name = "xdg-home" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec1cdab258fb55c0da61328dc52c8764709b249011b2cad0454c72f0bf10a1f6" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "xkbcommon-dl" version = "0.4.2" @@ -2511,6 +3513,91 @@ dependencies = [ "nih_plug_xtask", ] +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "zbus" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb97012beadd29e654708a0fdb4c84bc046f537aecfde2c3ee0a9e4b4d48c725" +dependencies = [ + "async-broadcast", + "async-executor", + "async-fs", + "async-io", + "async-lock", + "async-process", + "async-recursion", + "async-task", + "async-trait", + "blocking", + "enumflags2", + "event-listener", + "futures-core", + "futures-sink", + "futures-util", + "hex", + "nix 0.29.0", + "ordered-stream", + "rand", + "serde", + "serde_repr", + "sha1", + "static_assertions", + "tracing", + "uds_windows", + "windows-sys 0.52.0", + "xdg-home", + "zbus_macros", + "zbus_names", + "zvariant", +] + +[[package]] +name = "zbus_macros" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "267db9407081e90bbfa46d841d3cbc60f59c0351838c4bc65199ecd79ab1983e" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.117", + "zvariant_utils", +] + +[[package]] +name = "zbus_names" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b9b1fef7d021261cc16cba64c351d291b715febe0fa10dc3a443ac5a5022e6c" +dependencies = [ + "serde", + "static_assertions", + "zvariant", +] + [[package]] name = "zerocopy" version = "0.8.52" @@ -2531,8 +3618,100 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zvariant" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2084290ab9a1c471c38fc524945837734fbf124487e105daec2bb57fd48c81fe" +dependencies = [ + "endi", + "enumflags2", + "serde", + "static_assertions", + "url", + "zvariant_derive", +] + +[[package]] +name = "zvariant_derive" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73e2ba546bda683a90652bac4a279bc146adad1386f25379cf73200d2002c449" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.117", + "zvariant_utils", +] + +[[package]] +name = "zvariant_utils" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51bcff7cc3dbb5055396bcf774748c3dab426b4b8659046963523cee4808340" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] diff --git a/deepsteps-plugin/Cargo.toml b/deepsteps-plugin/Cargo.toml index 6f1cc3b..b25a7e7 100644 --- a/deepsteps-plugin/Cargo.toml +++ b/deepsteps-plugin/Cargo.toml @@ -15,6 +15,10 @@ nih_plug = { git = "https://github.com/robbert-vdh/nih-plug.git", rev = "f36931f nih_plug_egui = { git = "https://github.com/robbert-vdh/nih-plug.git", rev = "f36931f7af4646065488a9845d8f8c2f95252c23" } serde = { version = "1", features = ["derive"] } serde_json = "1" +arc-swap = "1" +rustfft = "6" +rfd = "0.14" +symphonia = { version = "0.5", default-features = false, features = ["wav", "flac"] } [profile.release] lto = "thin" diff --git a/deepsteps-plugin/src/audio.rs b/deepsteps-plugin/src/audio.rs new file mode 100644 index 0000000..f09beb7 --- /dev/null +++ b/deepsteps-plugin/src/audio.rs @@ -0,0 +1,262 @@ +//! Audio-file ingestion for the dataset builder: decode a file to mono f32 +//! (symphonia), detect onsets with a spectral-flux + adaptive-peak-pick +//! detector (rustfft), then encode to a 32-dim training sample. +//! +//! This is deliberately NOT a librosa clone (project plan, decision 2): the +//! file-derived dataset is approximate. The detector is a single self-contained +//! function with exposed constants for tuning. + +use std::path::Path; + +use rustfft::{num_complex::Complex, FftPlanner}; + +use crate::dataset::encode_onsets; + +const FRAME: usize = 1024; +const HOP: usize = 512; +/// Adaptive-threshold moving-average window, in frames. +const THRESH_WIN: usize = 8; +/// Threshold = mean(window) * THRESH_MULT + THRESH_DELTA (on normalized flux). +const THRESH_MULT: f32 = 1.5; +const THRESH_DELTA: f32 = 0.04; +/// Minimum gap between detected onsets, in frames (refractory period). +const MIN_GAP_FRAMES: usize = 3; + +#[derive(Debug)] +pub enum DecodeError { + Io(String), + Unsupported(String), + Decode(String), + Empty, +} + +impl std::fmt::Display for DecodeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + DecodeError::Io(s) => write!(f, "io error: {s}"), + DecodeError::Unsupported(s) => write!(f, "unsupported audio: {s}"), + DecodeError::Decode(s) => write!(f, "decode error: {s}"), + DecodeError::Empty => write!(f, "decoded audio was empty"), + } + } +} + +/// Decode an audio file to mono f32 samples + sample rate. Channels are +/// averaged. Supports the formats enabled in symphonia's Cargo features. +pub fn decode_audio(path: &Path) -> Result<(Vec, u32), DecodeError> { + use symphonia::core::audio::SampleBuffer; + use symphonia::core::codecs::DecoderOptions; + use symphonia::core::formats::FormatOptions; + use symphonia::core::io::MediaSourceStream; + use symphonia::core::meta::MetadataOptions; + use symphonia::core::probe::Hint; + + let file = std::fs::File::open(path).map_err(|e| DecodeError::Io(e.to_string()))?; + let mss = MediaSourceStream::new(Box::new(file), Default::default()); + + let mut hint = Hint::new(); + if let Some(ext) = path.extension().and_then(|e| e.to_str()) { + hint.with_extension(ext); + } + + let probed = symphonia::default::get_probe() + .format(&hint, mss, &FormatOptions::default(), &MetadataOptions::default()) + .map_err(|e| DecodeError::Unsupported(e.to_string()))?; + let mut format = probed.format; + + let track = format + .tracks() + .iter() + .find(|t| t.codec_params.codec != symphonia::core::codecs::CODEC_TYPE_NULL) + .ok_or_else(|| DecodeError::Unsupported("no decodable track".into()))?; + let track_id = track.id; + let sample_rate = track.codec_params.sample_rate.unwrap_or(44_100); + + let mut decoder = symphonia::default::get_codecs() + .make(&track.codec_params, &DecoderOptions::default()) + .map_err(|e| DecodeError::Unsupported(e.to_string()))?; + + let mut mono: Vec = Vec::new(); + let mut sample_buf: Option> = None; + + loop { + let packet = match format.next_packet() { + Ok(p) => p, + Err(symphonia::core::errors::Error::IoError(e)) + if e.kind() == std::io::ErrorKind::UnexpectedEof => + { + break + } + Err(e) => return Err(DecodeError::Decode(e.to_string())), + }; + if packet.track_id() != track_id { + continue; + } + match decoder.decode(&packet) { + Ok(audio_buf) => { + let spec = *audio_buf.spec(); + let channels = spec.channels.count().max(1); + if sample_buf.is_none() { + sample_buf = + Some(SampleBuffer::::new(audio_buf.capacity() as u64, spec)); + } + let buf = sample_buf.as_mut().unwrap(); + buf.copy_interleaved_ref(audio_buf); + // Downmix interleaved channels to mono. + for frame in buf.samples().chunks(channels) { + let sum: f32 = frame.iter().copied().sum(); + mono.push(sum / channels as f32); + } + } + Err(symphonia::core::errors::Error::DecodeError(_)) => continue, // skip bad packet + Err(e) => return Err(DecodeError::Decode(e.to_string())), + } + } + + if mono.is_empty() { + return Err(DecodeError::Empty); + } + Ok((mono, sample_rate)) +} + +/// Detect onset sample positions in mono audio via spectral flux + adaptive +/// peak picking. Returns positions in samples, ascending. +pub fn detect_onsets(mono: &[f32], _sr: u32) -> Vec { + if mono.len() < FRAME + HOP { + return Vec::new(); + } + let mut planner = FftPlanner::::new(); + let fft = planner.plan_fft_forward(FRAME); + + // Hann window. + let window: Vec = (0..FRAME) + .map(|n| { + let w = (std::f32::consts::PI * n as f32 / (FRAME as f32 - 1.0)).sin(); + w * w + }) + .collect(); + + let n_frames = (mono.len() - FRAME) / HOP + 1; + let bins = FRAME / 2; + let mut prev_mag = vec![0.0f32; bins]; + let mut flux = vec![0.0f32; n_frames]; + let mut scratch = vec![Complex::new(0.0f32, 0.0); FRAME]; + + for (t, f) in flux.iter_mut().enumerate() { + let start = t * HOP; + for i in 0..FRAME { + scratch[i] = Complex::new(mono[start + i] * window[i], 0.0); + } + fft.process(&mut scratch); + let mut sf = 0.0f32; + for k in 0..bins { + let mag = scratch[k].norm(); + let d = mag - prev_mag[k]; + if d > 0.0 { + sf += d; + } + prev_mag[k] = mag; + } + *f = sf; + } + + // Normalize flux to [0,1]. + let max = flux.iter().copied().fold(0.0f32, f32::max); + if max <= 0.0 { + return Vec::new(); + } + for v in &mut flux { + *v /= max; + } + + // Adaptive peak pick: local maximum, above moving-average threshold, with a + // refractory gap. + let mut onsets = Vec::new(); + let mut last_onset: Option = None; + for t in 1..n_frames - 1 { + let lo = t.saturating_sub(THRESH_WIN); + let hi = (t + THRESH_WIN + 1).min(n_frames); + let mean: f32 = flux[lo..hi].iter().copied().sum::() / (hi - lo) as f32; + let thresh = mean * THRESH_MULT + THRESH_DELTA; + + let is_peak = flux[t] > flux[t - 1] && flux[t] >= flux[t + 1] && flux[t] > thresh; + if is_peak { + if let Some(prev) = last_onset { + if t - prev < MIN_GAP_FRAMES { + // Keep the stronger of the two close peaks. + if flux[t] > flux[prev] { + onsets.pop(); + onsets.push(t * HOP); + last_onset = Some(t); + } + continue; + } + } + onsets.push(t * HOP); + last_onset = Some(t); + } + } + onsets +} + +/// Decode a file and encode it to a single 32-dim training sample. The whole +/// file is treated as one bar (matching `corpus_encode.py`'s `bar_length=1`). +/// Returns `Ok(None)` when the file decodes but has no detectable onsets. +pub fn file_to_sample(path: &Path) -> Result, DecodeError> { + let (mono, sr) = decode_audio(path)?; + let onsets = detect_onsets(&mono, sr); + if onsets.is_empty() { + return Ok(None); + } + let onsets_i64: Vec = onsets.iter().map(|&o| o as i64).collect(); + Ok(Some(encode_onsets(&onsets_i64, mono.len() as i64))) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A synthetic click train should yield onsets near the click positions. + #[test] + fn detects_click_train() { + let sr = 44_100u32; + let len = 44_100; // 1 second + let mut sig = vec![0.0f32; len]; + let clicks = [5_000usize, 16_000, 28_000, 39_000]; + for &c in &clicks { + // Short decaying burst so a frame sees a clear energy rise. + for i in 0..256 { + if c + i < len { + let env = 1.0 - (i as f32 / 256.0); + // broadband-ish: alternate sign + sig[c + i] = if i % 2 == 0 { env } else { -env }; + } + } + } + let onsets = detect_onsets(&sig, sr); + assert!(!onsets.is_empty(), "no onsets detected"); + + // Every real click should have a detected onset within ~1.5 hops. + let tol = (HOP as f32 * 1.5) as usize + FRAME; // detection lags by ~a frame + for &c in &clicks { + let found = onsets.iter().any(|&o| { + let d = if o > c { o - c } else { c - o }; + d <= tol + }); + assert!(found, "no onset near click {c}; got {onsets:?}"); + } + // Should not produce a flood of spurious onsets. + assert!(onsets.len() <= clicks.len() + 2, "too many onsets: {onsets:?}"); + } + + #[test] + fn silence_has_no_onsets() { + let onsets = detect_onsets(&vec![0.0f32; 44_100], 44_100); + assert!(onsets.is_empty()); + } + + #[test] + fn short_signal_is_safe() { + assert!(detect_onsets(&[0.0f32; 100], 44_100).is_empty()); + } +} diff --git a/deepsteps-plugin/src/autoencoder.rs b/deepsteps-plugin/src/autoencoder.rs new file mode 100644 index 0000000..2cd6efc --- /dev/null +++ b/deepsteps-plugin/src/autoencoder.rs @@ -0,0 +1,683 @@ +//! From-scratch autoencoder training, ported from the upstream Python +//! `Deep_Steps_project/bin/data/AE_init.py` (itself adapted from +//! ML-From-Scratch). Pure f64, no external ML deps, no nih-plug deps so it is +//! unit-testable in isolation. +//! +//! Architecture (input 32, latent 4): +//! encoder: Dense(32->16) Relu BN Dense(16->8) Relu BN Dense(8->4) +//! decoder: Dense(4->8) Relu BN Dense(8->16) Relu BN Dense(16->32) Sigmoid +//! +//! Two deliberate fixes over the Python original (see project plan, decision 2): +//! * Adam bias correction uses a real per-parameter step counter `t` +//! (`1 - b1^t`), where the Python divides by the constant `1 - b1`. +//! * Batches are shuffled each epoch (the Python iterates sequentially). +//! +//! The exported op list (see `model_ops.rs`) feeds `decoder.rs` unchanged. + +/// Minimal row-major dense matrix used for whole-batch forward/backward. +/// Dimensions are tiny (<=32) so naive loops are fine. +#[derive(Clone)] +pub struct Mat { + pub rows: usize, + pub cols: usize, + pub data: Vec, +} + +impl Mat { + fn zeros(rows: usize, cols: usize) -> Self { + Mat { rows, cols, data: vec![0.0; rows * cols] } + } + #[inline] + fn at(&self, r: usize, c: usize) -> f64 { + self.data[r * self.cols + c] + } + #[inline] + fn set(&mut self, r: usize, c: usize, v: f64) { + self.data[r * self.cols + c] = v; + } + fn col_mean(&self) -> Vec { + let mut m = vec![0.0; self.cols]; + for r in 0..self.rows { + for c in 0..self.cols { + m[c] += self.at(r, c); + } + } + for v in &mut m { + *v /= self.rows as f64; + } + m + } + /// Population variance (ddof=0), matching numpy `np.var`. + fn col_var(&self, mean: &[f64]) -> Vec { + let mut v = vec![0.0; self.cols]; + for r in 0..self.rows { + for c in 0..self.cols { + let d = self.at(r, c) - mean[c]; + v[c] += d * d; + } + } + for x in &mut v { + *x /= self.rows as f64; + } + v + } +} + +// --------------------------------------------------------------------------- +// PRNG: SplitMix64 -> deterministic init + shuffle, no `rand` dependency. +// --------------------------------------------------------------------------- +pub struct Rng { + state: u64, +} + +impl Rng { + pub fn new(seed: u64) -> Self { + // Avoid the all-zero fixed point. + Rng { state: seed ^ 0x9E37_79B9_7F4A_7C15 } + } + #[inline] + fn next_u64(&mut self) -> u64 { + self.state = self.state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + /// Uniform f64 in [0, 1). + #[inline] + fn next_f64(&mut self) -> f64 { + // Top 53 bits -> [0,1). + (self.next_u64() >> 11) as f64 / (1u64 << 53) as f64 + } + /// Uniform f64 in [-limit, limit). + #[inline] + fn uniform(&mut self, limit: f64) -> f64 { + (self.next_f64() * 2.0 - 1.0) * limit + } + /// Uniform integer in [0, n). + #[inline] + fn below(&mut self, n: usize) -> usize { + (self.next_u64() % n as u64) as usize + } +} + +// --------------------------------------------------------------------------- +// Adam (fixed bias correction). +// --------------------------------------------------------------------------- +#[derive(Clone)] +struct Adam { + lr: f64, + b1: f64, + b2: f64, + eps: f64, + m: Vec, + v: Vec, + t: u32, +} + +impl Adam { + fn new(lr: f64, b1: f64, b2: f64, n: usize) -> Self { + Adam { lr, b1, b2, eps: 1e-8, m: vec![0.0; n], v: vec![0.0; n], t: 0 } + } + /// In-place `w -= lr * m_hat / (sqrt(v_hat) + eps)` with real bias correction. + fn step(&mut self, w: &mut [f64], grad: &[f64]) { + self.t += 1; + let bc1 = 1.0 - self.b1.powi(self.t as i32); + let bc2 = 1.0 - self.b2.powi(self.t as i32); + for i in 0..w.len() { + self.m[i] = self.b1 * self.m[i] + (1.0 - self.b1) * grad[i]; + self.v[i] = self.b2 * self.v[i] + (1.0 - self.b2) * grad[i] * grad[i]; + let m_hat = self.m[i] / bc1; + let v_hat = self.v[i] / bc2; + w[i] -= self.lr * m_hat / (v_hat.sqrt() + self.eps); + } + } +} + +// --------------------------------------------------------------------------- +// Layers. +// --------------------------------------------------------------------------- +#[derive(Clone, Copy, PartialEq)] +pub enum ActKind { + Relu, + Sigmoid, +} + +enum Layer { + Dense { + n_in: usize, + n_out: usize, + w: Vec, // row-major (n_in x n_out): w[i*n_out + j] + b: Vec, // (n_out) + w_opt: Adam, + b_opt: Adam, + x_cache: Mat, + }, + Activation { + kind: ActKind, + in_cache: Mat, + }, + BatchNorm { + gamma: Vec, + beta: Vec, + eps: f64, + momentum: f64, + running_mean: Option>, + running_var: Option>, + g_opt: Adam, + b_opt: Adam, + x_centered: Mat, + stddev_inv: Vec, + }, +} + +impl Layer { + fn dense(n_in: usize, n_out: usize, lr: f64, b1: f64, b2: f64, rng: &mut Rng) -> Layer { + // Xavier uniform per fan-in: U(-1/sqrt(n_in), +1/sqrt(n_in)); bias zeros. + let limit = 1.0 / (n_in as f64).sqrt(); + let mut w = vec![0.0; n_in * n_out]; + for x in &mut w { + *x = rng.uniform(limit); + } + Layer::Dense { + n_in, + n_out, + w, + b: vec![0.0; n_out], + w_opt: Adam::new(lr, b1, b2, n_in * n_out), + b_opt: Adam::new(lr, b1, b2, n_out), + x_cache: Mat::zeros(0, 0), + } + } + + fn bn(dim: usize, momentum: f64, lr: f64, b1: f64, b2: f64) -> Layer { + Layer::BatchNorm { + gamma: vec![1.0; dim], + beta: vec![0.0; dim], + eps: 0.01, + momentum, + running_mean: None, + running_var: None, + g_opt: Adam::new(lr, b1, b2, dim), + b_opt: Adam::new(lr, b1, b2, dim), + x_centered: Mat::zeros(0, 0), + stddev_inv: vec![0.0; dim], + } + } + + fn forward(&mut self, x: &Mat, training: bool) -> Mat { + match self { + Layer::Dense { n_in, n_out, w, b, x_cache, .. } => { + debug_assert_eq!(x.cols, *n_in); + *x_cache = x.clone(); + let mut y = Mat::zeros(x.rows, *n_out); + for r in 0..x.rows { + for j in 0..*n_out { + let mut acc = b[j]; + for i in 0..*n_in { + acc += x.at(r, i) * w[i * *n_out + j]; + } + y.set(r, j, acc); + } + } + y + } + Layer::Activation { kind, in_cache } => { + *in_cache = x.clone(); + let mut y = Mat::zeros(x.rows, x.cols); + for idx in 0..x.data.len() { + let v = x.data[idx]; + y.data[idx] = match kind { + ActKind::Relu => { + if v >= 0.0 { + v + } else { + 0.0 + } + } + ActKind::Sigmoid => 1.0 / (1.0 + (-v).exp()), + }; + } + y + } + Layer::BatchNorm { + gamma, + beta, + eps, + momentum, + running_mean, + running_var, + x_centered, + stddev_inv, + .. + } => { + // Running stats initialize on the first forward (matches Python). + if running_mean.is_none() { + *running_mean = Some(x.col_mean()); + let m = running_mean.as_ref().unwrap(); + *running_var = Some(x.col_var(m)); + } + let (mean, var) = if training { + let bmean = x.col_mean(); + let bvar = x.col_var(&bmean); + // running = momentum*running + (1-momentum)*batch + let rm = running_mean.as_mut().unwrap(); + let rv = running_var.as_mut().unwrap(); + for c in 0..rm.len() { + rm[c] = *momentum * rm[c] + (1.0 - *momentum) * bmean[c]; + rv[c] = *momentum * rv[c] + (1.0 - *momentum) * bvar[c]; + } + (bmean, bvar) + } else { + (running_mean.clone().unwrap(), running_var.clone().unwrap()) + }; + + *x_centered = Mat::zeros(x.rows, x.cols); + for c in 0..x.cols { + stddev_inv[c] = 1.0 / (var[c] + *eps).sqrt(); + } + let mut out = Mat::zeros(x.rows, x.cols); + for r in 0..x.rows { + for c in 0..x.cols { + let xc = x.at(r, c) - mean[c]; + x_centered.set(r, c, xc); + let x_norm = xc * stddev_inv[c]; + out.set(r, c, gamma[c] * x_norm + beta[c]); + } + } + out + } + } + } + + fn backward(&mut self, grad: &Mat) -> Mat { + match self { + Layer::Dense { n_in, n_out, w, b, w_opt, b_opt, x_cache } => { + let rows = grad.rows; + // grad_w[i,j] = sum_r x[r,i]*grad[r,j]; grad_b[j] = sum_r grad[r,j] + let mut grad_w = vec![0.0; *n_in * *n_out]; + let mut grad_b = vec![0.0; *n_out]; + for r in 0..rows { + for j in 0..*n_out { + let g = grad.at(r, j); + grad_b[j] += g; + for i in 0..*n_in { + grad_w[i * *n_out + j] += x_cache.at(r, i) * g; + } + } + } + // accum_out = grad @ W^T, using the pre-update W. + let mut out = Mat::zeros(rows, *n_in); + for r in 0..rows { + for i in 0..*n_in { + let mut acc = 0.0; + for j in 0..*n_out { + acc += grad.at(r, j) * w[i * *n_out + j]; + } + out.set(r, i, acc); + } + } + w_opt.step(w, &grad_w); + b_opt.step(b, &grad_b); + out + } + Layer::Activation { kind, in_cache } => { + let mut out = Mat::zeros(grad.rows, grad.cols); + for idx in 0..grad.data.len() { + let x = in_cache.data[idx]; + let d = match kind { + ActKind::Relu => { + if x >= 0.0 { + 1.0 + } else { + 0.0 + } + } + ActKind::Sigmoid => { + let s = 1.0 / (1.0 + (-x).exp()); + s * (1.0 - s) + } + }; + out.data[idx] = grad.data[idx] * d; + } + out + } + Layer::BatchNorm { + gamma, beta, g_opt, b_opt, x_centered, stddev_inv, .. + } => { + let bs = grad.rows as f64; + let cols = grad.cols; + let gamma_old = gamma.clone(); + + // grad_gamma[c] = sum_r grad*x_norm ; grad_beta[c] = sum_r grad + let mut grad_gamma = vec![0.0; cols]; + let mut grad_beta = vec![0.0; cols]; + // per-column sums needed by the batchnorm backward formula + let mut sum_grad = vec![0.0; cols]; + let mut sum_grad_xc = vec![0.0; cols]; + for r in 0..grad.rows { + for c in 0..cols { + let g = grad.at(r, c); + let xc = x_centered.at(r, c); + let x_norm = xc * stddev_inv[c]; + grad_gamma[c] += g * x_norm; + grad_beta[c] += g; + sum_grad[c] += g; + sum_grad_xc[c] += g * xc; + } + } + g_opt.step(gamma, &grad_gamma); + b_opt.step(beta, &grad_beta); + + let mut out = Mat::zeros(grad.rows, cols); + for r in 0..grad.rows { + for c in 0..cols { + let g = grad.at(r, c); + let xc = x_centered.at(r, c); + let val = (1.0 / bs) + * gamma_old[c] + * stddev_inv[c] + * (bs * g + - sum_grad[c] + - xc * stddev_inv[c] * stddev_inv[c] * sum_grad_xc[c]); + out.set(r, c, val); + } + } + out + } + } + } +} + +// --------------------------------------------------------------------------- +// Autoencoder: encoder layers + decoder layers in one chain, split at enc_len. +// --------------------------------------------------------------------------- +pub struct Autoencoder { + layers: Vec, + enc_len: usize, + pub input_dim: usize, + pub latent_dim: usize, +} + +/// Adam/optimizer hyperparameters, matching AE_init.py `Adam(lr=0.01, b1=0.5)`. +const LR: f64 = 0.01; +const B1: f64 = 0.5; +const B2: f64 = 0.999; +const BN_MOMENTUM: f64 = 0.8; + +impl Autoencoder { + pub fn new(seed: u64) -> Self { + let input_dim = 32; + let latent_dim = 4; + let mut rng = Rng::new(seed); + let mut layers = Vec::new(); + + // Encoder 32 -> 16 -> 8 -> 4 + layers.push(Layer::dense(32, 16, LR, B1, B2, &mut rng)); + layers.push(Layer::Activation { kind: ActKind::Relu, in_cache: Mat::zeros(0, 0) }); + layers.push(Layer::bn(16, BN_MOMENTUM, LR, B1, B2)); + layers.push(Layer::dense(16, 8, LR, B1, B2, &mut rng)); + layers.push(Layer::Activation { kind: ActKind::Relu, in_cache: Mat::zeros(0, 0) }); + layers.push(Layer::bn(8, BN_MOMENTUM, LR, B1, B2)); + layers.push(Layer::dense(8, 4, LR, B1, B2, &mut rng)); + let enc_len = layers.len(); + + // Decoder 4 -> 8 -> 16 -> 32 + layers.push(Layer::dense(4, 8, LR, B1, B2, &mut rng)); + layers.push(Layer::Activation { kind: ActKind::Relu, in_cache: Mat::zeros(0, 0) }); + layers.push(Layer::bn(8, BN_MOMENTUM, LR, B1, B2)); + layers.push(Layer::dense(8, 16, LR, B1, B2, &mut rng)); + layers.push(Layer::Activation { kind: ActKind::Relu, in_cache: Mat::zeros(0, 0) }); + layers.push(Layer::bn(16, BN_MOMENTUM, LR, B1, B2)); + layers.push(Layer::dense(16, 32, LR, B1, B2, &mut rng)); + layers.push(Layer::Activation { kind: ActKind::Sigmoid, in_cache: Mat::zeros(0, 0) }); + + Autoencoder { layers, enc_len, input_dim, latent_dim } + } + + fn forward_range(&mut self, x: &Mat, start: usize, end: usize, training: bool) -> Mat { + let mut out = x.clone(); + for l in start..end { + out = self.layers[l].forward(&out, training); + } + out + } + + /// Full reconstruction forward over the whole chain. + fn forward_all(&mut self, x: &Mat, training: bool) -> Mat { + let n = self.layers.len(); + self.forward_range(x, 0, n, training) + } + + fn backward_all(&mut self, grad: &Mat) { + let mut g = grad.clone(); + for l in (0..self.layers.len()).rev() { + g = self.layers[l].backward(&g); + } + } + + /// Train on `data` (each row a 32-dim sample). `on_epoch(epoch, avg_loss)` + /// is called after every epoch; returning `false` cancels training. + /// Returns the number of completed epochs. + pub fn fit( + &mut self, + data: &[[f32; 32]], + epochs: usize, + batch: usize, + seed: u64, + mut on_epoch: impl FnMut(usize, f64) -> bool, + ) -> usize { + let n = data.len(); + if n == 0 { + return 0; + } + let batch = batch.max(1); + let mut rng = Rng::new(seed ^ 0xD1B5_4A32_D192_ED03); + let mut order: Vec = (0..n).collect(); + + for epoch in 0..epochs { + // Fisher-Yates shuffle each epoch (fix vs Python's sequential batches). + for i in (1..n).rev() { + let j = rng.below(i + 1); + order.swap(i, j); + } + + let mut epoch_loss = 0.0; + let mut n_batches = 0; + let mut start = 0; + while start < n { + let end = (start + batch).min(n); + let rows = end - start; + // Build batch matrix (target == input for an autoencoder). + let mut x = Mat::zeros(rows, 32); + for (r, &idx) in order[start..end].iter().enumerate() { + for c in 0..32 { + x.set(r, c, data[idx][c] as f64); + } + } + let recon = self.forward_all(&x, true); + // MSE 0.5*(y-yhat)^2 averaged over all elements (loss report). + let mut s = 0.0; + let mut grad = Mat::zeros(rows, 32); + for idx in 0..x.data.len() { + let d = x.data[idx] - recon.data[idx]; + s += 0.5 * d * d; + grad.data[idx] = -d; // -(y - yhat), elementwise, not averaged + } + epoch_loss += s / (x.data.len() as f64); + self.backward_all(&grad); + + n_batches += 1; + start = end; + } + let avg = epoch_loss / n_batches as f64; + if !on_epoch(epoch, avg) { + return epoch + 1; + } + } + epochs + } + + /// Encode a single 32-dim sample to a 4-dim latent (BN in inference mode). + pub fn encode(&mut self, x: &[f64; 32]) -> [f64; 4] { + let mut m = Mat::zeros(1, 32); + m.data.copy_from_slice(x); + let out = self.forward_range(&m, 0, self.enc_len, false); + let mut z = [0.0; 4]; + z.copy_from_slice(&out.data[..4]); + z + } + + /// Decode a 4-dim latent to a 32-dim output through the decoder layers + /// (BN in inference mode). Mirrors the exported `Decoder::forward`; used by + /// tests to check export parity. + pub fn decode(&mut self, z: &[f64; 4]) -> [f64; 32] { + let mut m = Mat::zeros(1, 4); + m.data.copy_from_slice(z); + let out = self.forward_range(&m, self.enc_len, self.layers.len(), false); + let mut y = [0.0; 32]; + y.copy_from_slice(&out.data[..32]); + y + } + + /// Index range of decoder layers, for the exporter (`model_ops.rs`). + pub fn decoder_range(&self) -> std::ops::Range { + self.enc_len..self.layers.len() + } + /// Index range of encoder layers, for the exporter. + pub fn encoder_range(&self) -> std::ops::Range { + 0..self.enc_len + } + /// Read-only layer accessor for the exporter. + pub(crate) fn layer_export(&self, i: usize) -> LayerView<'_> { + match &self.layers[i] { + Layer::Dense { n_in, n_out, w, b, .. } => { + LayerView::Dense { n_in: *n_in, n_out: *n_out, w, b } + } + Layer::Activation { kind, .. } => LayerView::Activation(*kind), + Layer::BatchNorm { gamma, beta, eps, running_mean, running_var, .. } => { + LayerView::BatchNorm { + gamma, + beta, + eps: *eps, + running_mean: running_mean.as_deref(), + running_var: running_var.as_deref(), + } + } + } + } +} + +/// Borrowed view of a layer's trained parameters, used by `model_ops::export_*`. +pub(crate) enum LayerView<'a> { + Dense { n_in: usize, n_out: usize, w: &'a [f64], b: &'a [f64] }, + Activation(ActKind), + BatchNorm { + gamma: &'a [f64], + beta: &'a [f64], + eps: f64, + running_mean: Option<&'a [f64]>, + running_var: Option<&'a [f64]>, + }, +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Adam with real bias correction differs from the buggy constant-divisor + /// form on the very first step. Locks in the fix (decision 2). + #[test] + fn adam_bias_correction_is_applied() { + let mut a = Adam::new(0.01, 0.5, 0.999, 1); + let mut w = [0.0]; + a.step(&mut w, &[1.0]); + // Correct: m=0.5, m_hat=0.5/(1-0.5)=1.0; v=0.001, v_hat=0.001/(1-0.999)=1.0; + // w -= 0.01 * 1.0 / (1.0 + 1e-8) ~= -0.01. + assert!((w[0] + 0.01).abs() < 1e-6, "got {}", w[0]); + } + + /// The real gradient-correctness gate: a tiny fixed dataset must be driven + /// to near-zero reconstruction loss. Only correct backprop converges here. + #[test] + fn overfits_tiny_dataset() { + let data = [ + [0.0f32; 32], + { + let mut v = [0.0f32; 32]; + v[0] = 1.0; + v[5] = 1.0; + v[16] = 0.5; + v + }, + { + let mut v = [0.0f32; 32]; + for i in 0..16 { + v[i] = (i % 2) as f32; + } + v + }, + { + // First 16 steps on, no substeps. + let mut v = [0.0f32; 32]; + for x in v.iter_mut().take(16) { + *x = 1.0; + } + v + }, + ]; + let mut ae = Autoencoder::new(42); + let mut last = f64::INFINITY; + ae.fit(&data, 1500, 4, 7, |_, loss| { + last = loss; + true + }); + assert!(last < 1e-3, "did not converge: final loss {last}"); + + // Reconstruction (infer mode) should match each input closely. + let mut x = Mat::zeros(data.len(), 32); + for (r, row) in data.iter().enumerate() { + for c in 0..32 { + x.set(r, c, row[c] as f64); + } + } + let recon = ae.forward_all(&x, false); + let mut worst = 0.0f64; + for idx in 0..x.data.len() { + worst = worst.max((x.data[idx] - recon.data[idx]).abs()); + } + assert!(worst < 0.1, "infer-mode reconstruction off by {worst}"); + } + + /// Cancellation: returning false from the callback stops early. + #[test] + fn fit_can_be_cancelled() { + let data = [[0.0f32; 32], [1.0f32; 32]]; + let mut ae = Autoencoder::new(1); + let done = ae.fit(&data, 100, 2, 1, |e, _| e < 4); + assert_eq!(done, 5, "should stop after epoch 4 returns false"); + } + + /// Shuffle is deterministic given a seed; different seeds diverge. + #[test] + fn shuffle_is_seed_deterministic() { + let data: Vec<[f32; 32]> = (0..20) + .map(|i| { + let mut v = [0.0f32; 32]; + v[i % 32] = 1.0; + v + }) + .collect(); + let run = |seed: u64| { + let mut ae = Autoencoder::new(99); + let mut losses = Vec::new(); + ae.fit(&data, 5, 4, seed, |_, l| { + losses.push(l); + true + }); + losses + }; + assert_eq!(run(123), run(123), "same seed must reproduce"); + assert_ne!(run(123), run(456), "different seeds should diverge"); + } +} diff --git a/deepsteps-plugin/src/dataset.rs b/deepsteps-plugin/src/dataset.rs new file mode 100644 index 0000000..bce1a72 --- /dev/null +++ b/deepsteps-plugin/src/dataset.rs @@ -0,0 +1,177 @@ +//! Dataset construction for in-app training. A dataset is a `Vec<[f32; 32]>`, +//! each row the 32-dim representation the autoencoder trains on: 16 onset +//! one-hots + 16 substep timing offsets. +//! +//! `encode_onsets` is an exact port of `Deep_Steps_project/tools/corpus_encode.py` +//! (the audio-file path), pinned by the ported `test_corpus_encode.py` cases. +//! `encode_grid` captures the plugin's current step pattern (the user-pattern +//! path). Audio decoding + onset detection live in `dataset_audio` (added next). + +const PER_QUARTER_NOTE: i64 = 48; +const SIXTEENTHS_DIV: i64 = 16; // bar_length = 1 +const PPQN_PER_BAR: i64 = PER_QUARTER_NOTE * 4; // 192 + +/// Round-half-to-even, matching Python's built-in `round()` / `np.round`, so +/// the encoding agrees with the upstream pipeline on half-way onset positions. +fn banker_round(x: f64) -> f64 { + let floor = x.floor(); + let diff = x - floor; + if (diff - 0.5).abs() < 1e-9 { + if (floor as i64).rem_euclid(2) == 0 { + floor + } else { + floor + 1.0 + } + } else { + x.round() + } +} + +/// Encode onset sample positions within one bar to a 32-dim vector. +/// `onsets`: integer sample positions; `dur`: bar length in samples. +/// Port of `corpus_encode.encode_onsets`. +pub fn encode_onsets(onsets: &[i64], dur: i64) -> [f32; 32] { + let mut out = [0.0f32; 32]; + let ppqn_timebase = banker_round(dur as f64 / PPQN_PER_BAR as f64) as i64; + let sixteenths = banker_round(dur as f64 / SIXTEENTHS_DIV as f64) as i64; + // Degenerate bar (dur too small): nothing to encode. + if ppqn_timebase < 1 || sixteenths < 1 { + return out; + } + + // Round onsets to nearest 16th; drop onsets landing on the same step as the + // immediately preceding (kept) onset. + let mut onset_points_rounded: Vec = Vec::new(); + let mut kept: Vec = Vec::new(); + let mut previous: Option = None; + for &onset in onsets { + let r = banker_round(onset as f64 / sixteenths as f64) as i64; + if Some(r) != previous { + onset_points_rounded.push(r); + previous = Some(r); + kept.push(onset); + } + // else: duplicate step, dropped (not added to `kept`) + } + + // One-hot over in-range steps. + for &o in &onset_points_rounded { + if (0..SIXTEENTHS_DIV).contains(&o) { + out[o as usize] = 1.0; + } + } + + // Substep timing offsets for each kept onset, in the order they appear. + let mut substeps: Vec = Vec::with_capacity(kept.len()); + for &o in &kept { + let ppqn_onset = o.div_euclid(ppqn_timebase) * ppqn_timebase; + let nearest = banker_round(o as f64 / sixteenths as f64) as i64 * sixteenths; + let ss = (ppqn_onset - nearest).div_euclid(ppqn_timebase); + substeps.push(((ss + 6) as f32) / 12.0); + } + + // Reindex substeps back onto the 16-step grid (only where a step is on). + let mut j = 0; + for step in 0..16 { + if out[step] == 1.0 && j < substeps.len() { + out[16 + step] = substeps[j]; + j += 1; + } + } + out +} + +/// Encode the plugin's current step pattern into a 32-dim training sample. +/// `mask`: per-step on/off bits; `substeps`: the decoder's per-step substep +/// values (already in the `[0,1]` domain the model emits and trains on). +pub fn encode_grid(mask: u16, substeps: &[f64; 16]) -> [f32; 32] { + let mut out = [0.0f32; 32]; + for i in 0..16 { + if mask & (1 << i) != 0 { + out[i] = 1.0; + out[16 + i] = substeps[i] as f32; + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + // Ported verbatim from tools/test_corpus_encode.py. + + #[test] + fn single_bar_onsets_to_32dim() { + let dur = 192; + let sixteenth = dur / 16; // 12 + let onsets: Vec = [0, 4, 8, 12].iter().map(|s| s * sixteenth).collect(); + let v = encode_onsets(&onsets, dur); + for i in [0, 4, 8, 12] { + assert_eq!(v[i], 1.0); + assert!((v[16 + i] - 0.5).abs() < 1e-6, "on-grid substep should be 0.5"); + } + let onhot_sum: f32 = v[..16].iter().sum(); + assert_eq!(onhot_sum, 4.0); + assert_eq!(v[16 + 1], 0.0); // empty step carries substep 0 + } + + #[test] + fn off_grid_substep_not_half() { + let dur = 192; + let onset = 4 * 12 + 5; // 53 + let v = encode_onsets(&[onset], dur); + assert_eq!(v[4], 1.0); + let onhot_sum: f32 = v[..16].iter().sum(); + assert_eq!(onhot_sum, 1.0); + let expected = (5.0 + 6.0) / 12.0; + assert!((expected - 0.5f32).abs() > 1e-6); + assert!((v[16 + 4] - expected).abs() < 1e-6); + } + + #[test] + fn duplicate_step_dropped() { + let dur = 192; + let v = encode_onsets(&[4 * 12, 4 * 12 + 2], dur); // [48, 50] both round to step 4 + assert_eq!(v[4], 1.0); + let onhot_sum: f32 = v[..16].iter().sum(); + assert_eq!(onhot_sum, 1.0); + } + + #[test] + fn out_of_range_step_excluded() { + let dur = 192; + let v = encode_onsets(&[0, 16 * 12], dur); // [0, 192]; 192 rounds to step 16, excluded + assert_eq!(v[0], 1.0); + let onhot_sum: f32 = v[..16].iter().sum(); + assert_eq!(onhot_sum, 1.0); + } + + #[test] + fn banker_round_matches_python() { + assert_eq!(banker_round(0.5), 0.0); + assert_eq!(banker_round(1.5), 2.0); + assert_eq!(banker_round(2.5), 2.0); + assert_eq!(banker_round(-2.5), -2.0); + assert_eq!(banker_round(4.4167), 4.0); + } + + #[test] + fn grid_roundtrip() { + let mask = (1 << 0) | (1 << 3) | (1 << 15); + let mut substeps = [0.0f64; 16]; + substeps[0] = 0.5; + substeps[3] = 0.9; + substeps[15] = 0.25; + let v = encode_grid(mask, &substeps); + assert_eq!(v[0], 1.0); + assert_eq!(v[3], 1.0); + assert_eq!(v[15], 1.0); + assert_eq!(v[1], 0.0); + assert!((v[16] - 0.5).abs() < 1e-6); + assert!((v[16 + 3] - 0.9).abs() < 1e-6); + assert!((v[16 + 15] - 0.25).abs() < 1e-6); + // off steps carry no substep + assert_eq!(v[16 + 1], 0.0); + } +} diff --git a/deepsteps-plugin/src/editor.rs b/deepsteps-plugin/src/editor.rs index 063fae5..1313e74 100644 --- a/deepsteps-plugin/src/editor.rs +++ b/deepsteps-plugin/src/editor.rs @@ -7,33 +7,52 @@ //! `ParamSetter`. use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering::Relaxed}; -use nih_plug::prelude::Editor; +use nih_plug::prelude::{AsyncExecutor, Editor, ParamSetter}; use nih_plug_egui::{create_egui_editor, egui, resizable_window::ResizableWindow, widgets::ParamSlider}; use crate::params::DeepStepsParams; use crate::shared::{SharedState, NO_STEP}; +use crate::training::{Task, TrainShared, TrainStatus}; /// State handed to every egui frame. struct EditorState { params: Arc, shared: Arc, + train: Arc, + exec: AsyncExecutor, + /// Training hyperparameters, editable in the GUI (default to the upstream + /// `train_export.py` values). Atomics so the panel can mutate them while the + /// surrounding `EditorState` is only borrowed immutably (the egui frame). + epochs: AtomicUsize, + batch: AtomicUsize, } pub fn create( params: Arc, shared: Arc, + train: Arc, + exec: AsyncExecutor, ) -> Option> { let egui_state = params.editor_state.clone(); create_egui_editor( egui_state, - EditorState { params, shared }, + EditorState { + params, + shared, + train, + exec, + epochs: AtomicUsize::new(200), + batch: AtomicUsize::new(16), + }, |_ctx, _state| {}, |ctx, setter, state| { - // Keep the playhead animating while the transport plays. When stopped - // (current == NO_STEP) let egui idle instead of spinning at full - // framerate — a grid click still triggers its own repaint. - if state.shared.current() != NO_STEP { + // Keep repainting while the playhead moves or a training run is in + // progress (so the progress bar advances). Otherwise let egui idle. + if state.shared.current() != NO_STEP + || matches!(state.train.status(), TrainStatus::Running | TrainStatus::Ingesting) + { ctx.request_repaint(); } @@ -88,6 +107,8 @@ pub fn create( labeled(ui, scale, "Scale", |ui| ui.add(ParamSlider::for_param(&p.scale, setter))); }); + training_section(ui, setter, state); + egui::CollapsingHeader::new("Pitches") .default_open(true) .show(ui, |ui| { @@ -116,6 +137,138 @@ pub fn create( ) } +/// The "Training" panel: build a dataset (capture live patterns or ingest audio +/// files), train an autoencoder on a background thread with a live progress bar, +/// and encode the current pattern back into the latent sliders. All heavy work +/// is dispatched via `exec.execute_background`; the audio thread is untouched. +fn training_section(ui: &mut egui::Ui, setter: &ParamSetter, state: &EditorState) { + // Clone the shared handles out so the closures below only borrow `state` for + // the editable `epochs`/`batch` fields (avoids overlapping borrows). + let train = state.train.clone(); + let shared = state.shared.clone(); + let params = state.params.clone(); + let exec = state.exec.clone(); + + egui::CollapsingHeader::new("Training") + .default_open(false) + .show(ui, |ui| { + let status = train.status(); + let busy = matches!(status, TrainStatus::Running | TrainStatus::Ingesting); + let n = train.dataset_len(); + + ui.horizontal(|ui| { + ui.label(format!("Dataset: {n}")); + if ui.button("Capture pattern").clicked() { + let v = crate::dataset::encode_grid(shared.mask(), &shared.substeps()); + if let Ok(mut d) = train.dataset.lock() { + d.push(v); + } + } + if ui.add_enabled(!busy, egui::Button::new("Add audio…")).clicked() { + if let Some(files) = rfd::FileDialog::new() + .add_filter("audio", &["wav", "flac"]) + .pick_files() + { + if let Ok(mut q) = train.pending_paths.lock() { + q.extend(files); + } + exec.execute_background(Task::IngestAudio); + } + } + if ui.add_enabled(n > 0 && !busy, egui::Button::new("Clear")).clicked() { + if let Ok(mut d) = train.dataset.lock() { + d.clear(); + } + } + }); + + ui.horizontal(|ui| { + // Atomics edited via a local copy, written back after the widget. + let mut epochs = state.epochs.load(Relaxed); + let mut batch = state.batch.load(Relaxed); + ui.label("Epochs"); + if ui.add(egui::DragValue::new(&mut epochs).range(1..=5000)).changed() { + state.epochs.store(epochs, Relaxed); + } + ui.label("Batch"); + if ui.add(egui::DragValue::new(&mut batch).range(1..=512)).changed() { + state.batch.store(batch, Relaxed); + } + }); + + ui.horizontal(|ui| { + if ui + .add_enabled(n > 0 && !busy, egui::Button::new("Train")) + .clicked() + { + exec.execute_background(Task::Train { + epochs: state.epochs.load(Relaxed), + batch: state.batch.load(Relaxed), + // Fixed seed -> reproducible training for a given dataset. + seed: 0x5_1EED, + }); + } + if status == TrainStatus::Running + && ui.button("Cancel").clicked() + { + train.cancel.store(true, Relaxed); + } + }); + + if status == TrainStatus::Running { + let e = train.epoch.load(Relaxed); + let t = train.total_epochs.load(Relaxed).max(1); + ui.add( + egui::ProgressBar::new(e as f32 / t as f32) + .text(format!("epoch {e}/{t} loss {:.4}", train.last_loss())), + ); + } + + ui.label(format!("Status: {}", status_text(status))); + + ui.horizontal(|ui| { + let trained = train.has_trained_model(); + ui.label(format!( + "Model: {}", + if trained { "Trained" } else { "Default (baked)" } + )); + if ui + .add_enabled(trained, egui::Button::new("Encode pattern → latent")) + .clicked() + { + let grid = crate::dataset::encode_grid(shared.mask(), &shared.substeps()); + let x: Vec = grid.iter().map(|&v| v as f64).collect(); + if let Some(z) = train.encode(&x) { + set_latents(setter, ¶ms, z); + } + } + }); + }); +} + +fn status_text(s: TrainStatus) -> &'static str { + match s { + TrainStatus::Idle => "idle", + TrainStatus::Ingesting => "ingesting audio…", + TrainStatus::Running => "training…", + TrainStatus::Done => "done", + TrainStatus::Cancelled => "cancelled", + TrainStatus::Error => "error (empty dataset?)", + } +} + +/// Write a latent vector into the 4 latent params as a single automation gesture +/// each, clamped to the params' `[0,1]` range (the encoder output is unbounded). +fn set_latents(setter: &ParamSetter, p: &DeepStepsParams, z: [f64; 4]) { + let targets = [&p.latent_a, &p.latent_b, &p.latent_c, &p.latent_d]; + for (param, &v) in targets.iter().zip(z.iter()) { + let val = (v as f32).clamp(0.0, 1.0); + setter.begin_set_parameter(*param); + setter.set_parameter(*param, val); + setter.end_set_parameter(*param); + } +} + /// One labelled row: fixed-width label + the widget. Label box and row height /// scale with `scale` so they keep pace with the scaled font size. fn labeled(ui: &mut egui::Ui, scale: f32, label: &str, add: impl FnOnce(&mut egui::Ui) -> egui::Response) { diff --git a/deepsteps-plugin/src/lib.rs b/deepsteps-plugin/src/lib.rs index 768bb49..555c12d 100644 --- a/deepsteps-plugin/src/lib.rs +++ b/deepsteps-plugin/src/lib.rs @@ -1,16 +1,23 @@ +pub mod audio; +pub mod autoencoder; +pub mod dataset; pub mod decoder; pub mod editor; +pub mod model_ops; pub mod params; pub mod sequencer; pub mod shared; +pub mod training; use nih_plug::prelude::*; use std::sync::Arc; +use std::sync::atomic::Ordering::Relaxed; use decoder::Decoder; use params::{DeepStepsParams, ScaleParam}; use sequencer::{quantize, schedule_step, steps_in_range, Scale, STEP_BEATS}; use shared::{pack, SharedState, NO_STEP}; +use training::{Task, TrainShared}; /// A NoteOff scheduled to fire in a future process block. `remaining` is the /// sample count from the *current* block's start until the NoteOff should fire; @@ -20,9 +27,15 @@ struct PendingOff { remaining: i64, } -struct DeepSteps { +pub struct DeepSteps { params: Arc, - decoder: Decoder, + /// Cross-thread training state. Owns the live (hot-swappable) decoder the + /// audio thread runs, plus the dataset and progress atomics. + train: Arc, + /// Last `model_generation` the audio thread acted on. When it falls behind + /// `train.model_generation` a swap happened, so `last_latent` is invalidated + /// to force a regenerate with the new model. + model_gen_seen: u64, /// Last latent vector that produced `steps`/`substeps`. Initialised to NaN so /// the first `maybe_regen` always regenerates (NaN != anything). last_latent: [f64; 4], @@ -45,9 +58,12 @@ impl Default for DeepSteps { Decoder::empty() } }; + let params = Arc::new(DeepStepsParams::default()); + let train = Arc::new(TrainShared::new(decoder, params.trained_model.clone())); Self { - params: Arc::new(DeepStepsParams::default()), - decoder, + params, + train, + model_gen_seen: 0, last_latent: [f64::NAN; 4], substeps: [0.0; 16], shared: Arc::new(SharedState::default()), @@ -77,8 +93,16 @@ impl DeepSteps { } } - /// Re-run the decoder iff the latent params changed since last call. + /// Re-run the decoder iff the latent params changed since last call, or the + /// model was hot-swapped by a finished training run. fn maybe_regen(&mut self) { + // A model swap bumps the generation; invalidate the cache so the new + // decoder regenerates even if the latent vector is unchanged. + let gen = self.train.model_generation.load(Relaxed); + if gen != self.model_gen_seen { + self.model_gen_seen = gen; + self.last_latent = [f64::NAN; 4]; + } let p = &self.params; let z = [ p.latent_a.value() as f64, @@ -87,11 +111,14 @@ impl DeepSteps { p.latent_d.value() as f64, ]; if z != self.last_latent { - let (s, ss) = self.decoder.generate(&z); + // Wait-free load of the live decoder (audio thread never locks). + let decoder = self.train.model.load(); + let (s, ss) = decoder.generate(&z); // Publish the freshly-decoded pattern as the playback source of // truth. This overwrites any user grid toggles — moving a latent // regenerates, which is the intended generative behaviour. self.shared.set_mask(pack(&s)); + self.shared.set_substeps(&ss); self.substeps = ss; self.last_latent = z; } @@ -121,14 +148,20 @@ impl Plugin for DeepSteps { const SAMPLE_ACCURATE_AUTOMATION: bool = true; type SysExMessage = (); - type BackgroundTask = (); + type BackgroundTask = Task; + + /// Runs background tasks (training, audio ingestion) off the audio and GUI + /// threads. Queried once after construction; captures the shared state. + fn task_executor(&mut self) -> TaskExecutor { + training::executor(self.train.clone()) + } fn params(&self) -> Arc { self.params.clone() } - fn editor(&mut self, _async_executor: AsyncExecutor) -> Option> { - editor::create(self.params.clone(), self.shared.clone()) + fn editor(&mut self, async_executor: AsyncExecutor) -> Option> { + editor::create(self.params.clone(), self.shared.clone(), self.train.clone(), async_executor) } fn initialize( @@ -138,6 +171,21 @@ impl Plugin for DeepSteps { _context: &mut impl InitContext, ) -> bool { self.sample_rate = buffer_config.sample_rate; + + // State has already been restored at this point: if the host loaded a + // trained model, hot-swap it into the audio + encode paths (overriding + // the baked default). The bump forces `maybe_regen` to use it. + if let Ok(slot) = self.params.trained_model.lock() { + if let Some(tm) = slot.as_ref() { + if let Ok(dec) = model_ops::to_decoder(&tm.decoder) { + self.train.model.store(Arc::new(dec)); + } + if let Ok(enc) = model_ops::to_decoder(&tm.encoder) { + self.train.encoder.store(Arc::new(Some(enc))); + } + self.train.model_generation.fetch_add(1, Relaxed); + } + } true } diff --git a/deepsteps-plugin/src/model_ops.rs b/deepsteps-plugin/src/model_ops.rs new file mode 100644 index 0000000..699705d --- /dev/null +++ b/deepsteps-plugin/src/model_ops.rs @@ -0,0 +1,192 @@ +//! Serializable op-list model export, byte-compatible with `weights/decoder.json` +//! and the Python `train_export.py` output. A model trained in-app serializes to +//! exactly the format `decoder::Decoder::from_json_str` already consumes, so the +//! audio-thread inference path needs no changes to run an in-app-trained model. +//! +//! The encoder is exported in the same op-list shape; at runtime it is loaded as +//! a `Decoder` too (the forward pass is a generic layer chain — only `generate()` +//! assumes a 32-dim output, which the encode path does not call). + +use serde::{Deserialize, Serialize}; + +use crate::autoencoder::{ActKind, Autoencoder, LayerView}; +use crate::decoder::Decoder; + +/// One layer in the exported op list. Field names/tags match `decoder.rs`'s +/// private `Op` and `train_export.py` exactly (note `b` is nested 1xN). +#[derive(Serialize, Deserialize, Clone)] +#[serde(tag = "op")] +pub enum ExportOp { + #[serde(rename = "dense")] + #[allow(non_snake_case)] + Dense { W: Vec>, b: Vec> }, + #[serde(rename = "relu")] + Relu, + #[serde(rename = "sigmoid")] + Sigmoid, + #[serde(rename = "bn")] + Bn { + gamma: Vec, + beta: Vec, + running_mean: Vec, + running_var: Vec, + eps: f64, + }, +} + +/// A full exported model (`{latent_dim, input_dim, ops}`), the persistence and +/// hot-swap unit. +#[derive(Serialize, Deserialize, Clone)] +pub struct ModelExport { + pub latent_dim: usize, + pub input_dim: usize, + pub ops: Vec, +} + +/// Both halves of a trained autoencoder, serialized into DAW state (`#[persist]`). +#[derive(Serialize, Deserialize, Clone)] +pub struct TrainedModel { + pub decoder: ModelExport, + pub encoder: ModelExport, +} + +#[derive(Debug)] +pub enum ExportError { + /// A BatchNorm layer was exported before any forward pass populated its + /// running stats (mirrors the Python guard in `train_export.py`). + UntrainedBatchNorm, +} + +impl std::fmt::Display for ExportError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ExportError::UntrainedBatchNorm => write!( + f, + "BatchNorm running stats are None -- train (fit) before exporting" + ), + } + } +} + +fn export_range( + ae: &Autoencoder, + range: std::ops::Range, +) -> Result { + let mut ops = Vec::new(); + for i in range { + match ae.layer_export(i) { + LayerView::Dense { n_in, n_out, w, b } => { + // Reshape row-major w[i*n_out+j] -> nested [n_in][n_out]; bias 1xN. + let mut rows = Vec::with_capacity(n_in); + for r in 0..n_in { + rows.push(w[r * n_out..(r + 1) * n_out].to_vec()); + } + ops.push(ExportOp::Dense { W: rows, b: vec![b.to_vec()] }); + } + LayerView::Activation(ActKind::Relu) => ops.push(ExportOp::Relu), + LayerView::Activation(ActKind::Sigmoid) => ops.push(ExportOp::Sigmoid), + LayerView::BatchNorm { gamma, beta, eps, running_mean, running_var } => { + let (rm, rv) = match (running_mean, running_var) { + (Some(m), Some(v)) => (m, v), + _ => return Err(ExportError::UntrainedBatchNorm), + }; + ops.push(ExportOp::Bn { + gamma: gamma.to_vec(), + beta: beta.to_vec(), + running_mean: rm.to_vec(), + running_var: rv.to_vec(), + eps, + }); + } + } + } + Ok(ModelExport { latent_dim: ae.latent_dim, input_dim: ae.input_dim, ops }) +} + +/// Export the decoder half (latent -> 32-dim), matching `train_export.py`. +pub fn export_decoder(ae: &Autoencoder) -> Result { + export_range(ae, ae.decoder_range()) +} + +/// Export the encoder half (32-dim -> latent). +pub fn export_encoder(ae: &Autoencoder) -> Result { + export_range(ae, ae.encoder_range()) +} + +/// Build a runnable `Decoder` from an export by round-tripping through the same +/// JSON the baked weights use, so there is a single forward-pass implementation. +pub fn to_decoder(export: &ModelExport) -> Result { + let s = serde_json::to_string(export)?; + Decoder::from_json_str(&s) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Train tiny, export the decoder, rebuild a `Decoder`, and confirm its + /// forward matches the in-memory autoencoder's own decoder forward. This + /// ties the training net (autoencoder.rs) to the inference path (decoder.rs) + /// through the persisted JSON format. + #[test] + fn export_decoder_roundtrips_through_decoder() { + let data = [[0.0f32; 32], [1.0f32; 32], { + let mut v = [0.0f32; 32]; + v[2] = 1.0; + v[16] = 0.5; + v + }]; + let mut ae = Autoencoder::new(7); + ae.fit(&data, 50, 3, 1, |_, _| true); // populate BN running stats + + let export = export_decoder(&ae).expect("decoder should export after fit"); + let dec = to_decoder(&export).expect("export must parse as Decoder"); + + for seed in 0..5u64 { + // Deterministic test latents. + let z = [ + ((seed * 7 + 1) % 11) as f64 / 11.0, + ((seed * 13 + 3) % 11) as f64 / 11.0, + ((seed * 5 + 2) % 11) as f64 / 11.0, + ((seed * 3 + 9) % 11) as f64 / 11.0, + ]; + let a = ae.decode(&z); + let b = dec.forward(&z); + assert_eq!(b.len(), 32); + for i in 0..32 { + assert!((a[i] - b[i]).abs() < 1e-9, "op {i}: {} vs {}", a[i], b[i]); + } + } + } + + /// Encoder export is also a valid runnable op chain (32-dim in -> 4-dim out). + #[test] + fn export_encoder_matches_encode() { + let data = [[0.0f32; 32], [1.0f32; 32]]; + let mut ae = Autoencoder::new(3); + ae.fit(&data, 30, 2, 1, |_, _| true); + + let export = export_encoder(&ae).unwrap(); + let enc = to_decoder(&export).unwrap(); + + let mut x = [0.0f64; 32]; + x[0] = 1.0; + x[16] = 0.5; + let z_ref = ae.encode(&x); + let z_run = enc.forward(&x); + assert_eq!(z_run.len(), 4); + for i in 0..4 { + assert!((z_ref[i] - z_run[i]).abs() < 1e-9, "{} vs {}", z_ref[i], z_run[i]); + } + } + + /// Exporting before any forward pass fails loudly (untrained BN). + #[test] + fn export_untrained_bn_errors() { + let ae = Autoencoder::new(1); + assert!(matches!( + export_decoder(&ae), + Err(ExportError::UntrainedBatchNorm) + )); + } +} diff --git a/deepsteps-plugin/src/params.rs b/deepsteps-plugin/src/params.rs index 95ae059..9f8bc57 100644 --- a/deepsteps-plugin/src/params.rs +++ b/deepsteps-plugin/src/params.rs @@ -6,7 +6,9 @@ use nih_plug::prelude::*; use nih_plug_egui::EguiState; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; + +use crate::model_ops::TrainedModel; /// Default editor window size (compact layout; shrunk from the Stage-1 canvas in /// commit c42a1ff to fit the collapsed-grid pitch panel). @@ -89,6 +91,13 @@ pub struct DeepStepsParams { #[persist = "editor-state"] pub editor_state: Arc, + /// Trained model (decoder + encoder op lists) persisted in DAW state, so a + /// model trained in-session survives save/reload and travels with presets. + /// `None` until the user trains; the baked `decoder.json` is the fallback. + /// Shared (same `Arc`) with `training::TrainShared` for hot-swap. + #[persist = "trained-model"] + pub trained_model: Arc>>, + /// Latent dimension A driving the decoder (later host-MIDI-CC-mappable). #[id = "latentA"] pub latent_a: FloatParam, @@ -136,6 +145,7 @@ impl Default for DeepStepsParams { Self { editor_state: EguiState::from_size(EDITOR_WIDTH, EDITOR_HEIGHT), + trained_model: Arc::new(Mutex::new(None)), latent_a: latent("Latent A"), latent_b: latent("Latent B"), diff --git a/deepsteps-plugin/src/shared.rs b/deepsteps-plugin/src/shared.rs index 0363c77..cf3963f 100644 --- a/deepsteps-plugin/src/shared.rs +++ b/deepsteps-plugin/src/shared.rs @@ -7,7 +7,7 @@ //! values are independent (a display mask and a playhead index) with no //! cross-variable invariant to protect. -use std::sync::atomic::{AtomicU16, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicU16, AtomicU64, AtomicUsize, Ordering}; /// Playhead sentinel meaning "no step is currently playing" (transport stopped). pub const NO_STEP: usize = usize::MAX; @@ -20,6 +20,11 @@ pub struct SharedState { /// Index of the step currently playing, for the editor's playhead highlight, /// or [`NO_STEP`] when the transport is stopped. pub current_step: AtomicUsize, + /// Per-step substep timing offsets (the decoder's raw `[0,1]` outputs), as + /// `f64::to_bits`. Written by the decoder on regeneration; read by the editor + /// so the capture/encode features can rebuild a full 32-dim sample from the + /// live pattern. Display/training only — never read by playback. + pub substeps: [AtomicU64; 16], } impl Default for SharedState { @@ -27,6 +32,7 @@ impl Default for SharedState { Self { steps: AtomicU16::new(0), current_step: AtomicUsize::new(NO_STEP), + substeps: std::array::from_fn(|_| AtomicU64::new(0)), } } } @@ -65,6 +71,18 @@ impl SharedState { pub fn current(&self) -> usize { self.current_step.load(Ordering::Relaxed) } + + /// Publish the decoder's per-step substep offsets (called on regeneration). + pub fn set_substeps(&self, ss: &[f64; 16]) { + for (a, &v) in self.substeps.iter().zip(ss.iter()) { + a.store(v.to_bits(), Ordering::Relaxed); + } + } + + /// Read the per-step substep offsets for pattern capture / encoding. + pub fn substeps(&self) -> [f64; 16] { + std::array::from_fn(|i| f64::from_bits(self.substeps[i].load(Ordering::Relaxed))) + } } /// Pack a 16-element bool pattern into a `u16` mask (bit `i` = `pattern[i]`). diff --git a/deepsteps-plugin/src/training.rs b/deepsteps-plugin/src/training.rs new file mode 100644 index 0000000..9bc6a9b --- /dev/null +++ b/deepsteps-plugin/src/training.rs @@ -0,0 +1,285 @@ +//! Runtime training state + background-thread driver. +//! +//! Training must never run on the audio thread. It runs on nih-plug's background +//! thread (via `Plugin::task_executor` + `AsyncExecutor::execute_background`), +//! reading/writing the `Arc` shared with the audio and GUI threads. +//! +//! The audio thread only ever does a wait-free `model.load()` (an `ArcSwap`) and +//! a few `Relaxed` atomic reads — it never locks a `Mutex`. The dataset and +//! trained-model `Mutex`es are touched solely by the GUI and background threads. + +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicU8, AtomicUsize, Ordering::Relaxed}; +use std::sync::{Arc, Mutex}; + +use arc_swap::ArcSwap; + +use crate::autoencoder::Autoencoder; +use crate::decoder::Decoder; +use crate::model_ops::{self, TrainedModel}; + +/// A background task. Kept `Copy`/heap-free per nih-plug's `BackgroundTask` +/// contract; all data flows through `TrainShared`. +#[derive(Clone, Copy)] +pub enum Task { + /// Train an autoencoder on the current dataset and hot-swap the result. + Train { epochs: usize, batch: usize, seed: u64 }, + /// Decode + onset-detect the files queued in `pending_paths`, appending each + /// to the dataset. + IngestAudio, +} + +/// Current background-operation status, for the GUI. Distinct from whether a +/// trained model exists (that is tracked by `model_generation > 0`). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum TrainStatus { + Idle, + Ingesting, + Running, + Done, + Cancelled, + Error, +} + +impl TrainStatus { + fn to_u8(self) -> u8 { + match self { + TrainStatus::Idle => 0, + TrainStatus::Ingesting => 1, + TrainStatus::Running => 2, + TrainStatus::Done => 3, + TrainStatus::Cancelled => 4, + TrainStatus::Error => 5, + } + } + fn from_u8(v: u8) -> Self { + match v { + 1 => TrainStatus::Ingesting, + 2 => TrainStatus::Running, + 3 => TrainStatus::Done, + 4 => TrainStatus::Cancelled, + 5 => TrainStatus::Error, + _ => TrainStatus::Idle, + } + } +} + +/// State shared between the audio thread, the GUI thread, and the background +/// training thread. +pub struct TrainShared { + /// Accumulated training samples. GUI/background only. + pub dataset: Mutex>, + /// Audio files queued for ingestion (the GUI can't pass a `Vec` + /// through the `Copy` task enum, so it parks them here). GUI/background only. + pub pending_paths: Mutex>, + + status: AtomicU8, + /// Completed epochs (for the progress bar). + pub epoch: AtomicUsize, + pub total_epochs: AtomicUsize, + /// Latest epoch loss, as `f64::to_bits` (display only). + last_loss_bits: AtomicU64, + /// GUI sets this to request the running training stop after the next epoch. + pub cancel: AtomicBool, + /// Bumped on every model swap so the audio thread knows to regenerate. + pub model_generation: AtomicU64, + + /// The live decoder the audio thread runs. Hot-swapped on training finish. + pub model: ArcSwap, + /// The live encoder for the "encode pattern -> latent" feature. `None` until + /// a model is trained/restored (the baked default ships no encoder). + pub encoder: ArcSwap>, + /// Persisted trained model (shared with the `#[persist]` param field). + pub trained_model: Arc>>, +} + +impl TrainShared { + pub fn new( + initial_decoder: Decoder, + trained_model: Arc>>, + ) -> Self { + TrainShared { + dataset: Mutex::new(Vec::new()), + pending_paths: Mutex::new(Vec::new()), + status: AtomicU8::new(TrainStatus::Idle.to_u8()), + epoch: AtomicUsize::new(0), + total_epochs: AtomicUsize::new(0), + last_loss_bits: AtomicU64::new(0), + cancel: AtomicBool::new(false), + model_generation: AtomicU64::new(0), + model: ArcSwap::from_pointee(initial_decoder), + encoder: ArcSwap::from_pointee(None), + trained_model, + } + } + + pub fn status(&self) -> TrainStatus { + TrainStatus::from_u8(self.status.load(Relaxed)) + } + fn set_status(&self, s: TrainStatus) { + self.status.store(s.to_u8(), Relaxed); + } + pub fn last_loss(&self) -> f64 { + f64::from_bits(self.last_loss_bits.load(Relaxed)) + } + pub fn dataset_len(&self) -> usize { + self.dataset.lock().map(|d| d.len()).unwrap_or(0) + } + pub fn has_trained_model(&self) -> bool { + self.model_generation.load(Relaxed) > 0 + } + /// Encode a 32-dim pattern to a latent using the live encoder, if one exists. + pub fn encode(&self, x: &[f64]) -> Option<[f64; 4]> { + let guard = self.encoder.load(); + let enc = guard.as_ref().as_ref()?; + let out = enc.forward(x); + if out.len() < 4 { + return None; + } + Some([out[0], out[1], out[2], out[3]]) + } +} + +/// Build the background task executor closure. Called once by `Plugin::task_executor`. +pub fn executor(train: Arc) -> Box { + Box::new(move |task| match task { + Task::Train { epochs, batch, seed } => run_training(&train, epochs, batch, seed), + Task::IngestAudio => run_ingest(&train), + }) +} + +fn run_training(train: &Arc, epochs: usize, batch: usize, seed: u64) { + train.cancel.store(false, Relaxed); + train.epoch.store(0, Relaxed); + train.total_epochs.store(epochs, Relaxed); + train.set_status(TrainStatus::Running); + + let data = match train.dataset.lock() { + Ok(d) => d.clone(), + Err(_) => { + train.set_status(TrainStatus::Error); + return; + } + }; + if data.is_empty() { + train.set_status(TrainStatus::Error); + return; + } + + let mut ae = Autoencoder::new(seed); + let completed = ae.fit(&data, epochs, batch, seed, |epoch, loss| { + train.epoch.store(epoch + 1, Relaxed); + train.last_loss_bits.store(loss.to_bits(), Relaxed); + !train.cancel.load(Relaxed) + }); + + // Export + swap even if cancelled: a partially trained net is still a valid + // model (BN running stats are populated after the first forward). + let dec_exp = model_ops::export_decoder(&ae); + let enc_exp = model_ops::export_encoder(&ae); + match (dec_exp, enc_exp) { + (Ok(dec_exp), Ok(enc_exp)) => { + if let Ok(dec) = model_ops::to_decoder(&dec_exp) { + train.model.store(Arc::new(dec)); + } + if let Ok(enc) = model_ops::to_decoder(&enc_exp) { + train.encoder.store(Arc::new(Some(enc))); + } + if let Ok(mut slot) = train.trained_model.lock() { + *slot = Some(TrainedModel { decoder: dec_exp, encoder: enc_exp }); + } + train.model_generation.fetch_add(1, Relaxed); + train.set_status(if completed < epochs { + TrainStatus::Cancelled + } else { + TrainStatus::Done + }); + } + _ => train.set_status(TrainStatus::Error), + } +} + +fn run_ingest(train: &Arc) { + let prev = train.status(); + train.set_status(TrainStatus::Ingesting); + let paths: Vec = match train.pending_paths.lock() { + Ok(mut q) => std::mem::take(&mut *q), + Err(_) => Vec::new(), + }; + for p in paths { + match crate::audio::file_to_sample(&p) { + Ok(Some(v)) => { + if let Ok(mut d) = train.dataset.lock() { + d.push(v); + } + } + Ok(None) => nih_plug::nih_log!("DeepSteps: no onsets in {p:?}, skipped"), + Err(e) => nih_plug::nih_log!("DeepSteps: ingest failed for {p:?}: {e}"), + } + } + // Restore the prior status (e.g. keep showing Done) unless training is active. + train.set_status(if prev == TrainStatus::Running { + TrainStatus::Running + } else { + TrainStatus::Idle + }); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dataset::encode_grid; + + fn shared() -> Arc { + let slot = Arc::new(Mutex::new(None)); + Arc::new(TrainShared::new(Decoder::empty(), slot)) + } + + /// End-to-end: dataset -> Train task -> hot-swap + persistence slot filled. + #[test] + fn train_task_swaps_and_persists() { + let train = shared(); + // A few distinct patterns captured from "grids". + { + let mut d = train.dataset.lock().unwrap(); + for i in 0..8u16 { + let mask = (i.wrapping_mul(37) | 1) & 0xFFFF; + let mut ss = [0.0f64; 16]; + ss[(i % 16) as usize] = 0.5; + d.push(encode_grid(mask, &ss)); + } + } + assert_eq!(train.model_generation.load(Relaxed), 0); + assert!(!train.has_trained_model()); + + // Run the executor inline (no real background thread needed for the test). + executor(train.clone())(Task::Train { epochs: 40, batch: 4, seed: 1 }); + + assert_eq!(train.status(), TrainStatus::Done); + assert_eq!(train.model_generation.load(Relaxed), 1); + assert!(train.has_trained_model()); + // Persistence slot is filled with both halves. + assert!(train.trained_model.lock().unwrap().is_some()); + // Encoder is live and produces a 4-dim latent. + let x: Vec = encode_grid(0b1010_1010_1010_1010, &[0.5; 16]) + .iter() + .map(|&v| v as f64) + .collect(); + let z = train.encode(&x).expect("encoder available after training"); + assert!(z.iter().all(|v| v.is_finite())); + // The swapped decoder runs and yields a 32-element pattern. + let dec = train.model.load(); + let out = dec.forward(&[z[0], z[1], z[2], z[3]]); + assert_eq!(out.len(), 32); + } + + /// Training with an empty dataset reports an error and swaps nothing. + #[test] + fn train_empty_dataset_errors() { + let train = shared(); + executor(train.clone())(Task::Train { epochs: 5, batch: 2, seed: 1 }); + assert_eq!(train.status(), TrainStatus::Error); + assert_eq!(train.model_generation.load(Relaxed), 0); + assert!(!train.has_trained_model()); + } +} diff --git a/docs/plans/2026-06-16-runtime-training-design.md b/docs/plans/2026-06-16-runtime-training-design.md new file mode 100644 index 0000000..30dff35 --- /dev/null +++ b/docs/plans/2026-06-16-runtime-training-design.md @@ -0,0 +1,87 @@ +# Runtime Autoencoder Training + In-App Dataset Building (Stage 3) + +*2026-06-16* + +## Context + +Upstream DeepSteps shipped a full ML pipeline — **train + infer**. The Stage 2 Rust port +kept only **inference**: a frozen decoder (`weights/decoder.json`) baked into the plugin +via `include_str!`, run in pure f64 (`src/decoder.rs`). Training stayed offline in Python +(`Deep_Steps_project/`), and only the *decoder* shipped. + +Stage 3 ports the **training half** into the live plugin: a from-scratch autoencoder +(encoder + decoder) that trains in-session on a dataset the user builds inside the plugin, +then hot-swaps the trained decoder into the audio path. The user can record/load material, +train a model live, and have the latent sliders drive *their* model. + +### Decisions +1. **Dataset source = both**: audio files from disk (decode + onset detection) **and** user + patterns captured from the grid. +2. **Fidelity = fix the bugs**: correct Adam bias correction (per-param step counter `t`), + per-epoch batch shuffle. Does not match the Python numerically; converges better. +3. **Encoder exposed at runtime**: "encode current grid pattern → set the 4 latent sliders". +4. **Persist trained model** in DAW state (`#[persist]`); baked `decoder.json` stays as the + default/fallback. + +### Constraints (from the codebase) +- MIDI-only plugin, no host audio (`lib.rs` empty `AUDIO_IO_LAYOUTS`) → audio datasets come + from disk files, not the host stream. +- `process()` is the audio thread; training must never run there, and the audio thread must + never lock a `Mutex`. +- nih-plug `BackgroundTask` must stay heap-free → data flows through shared `Arc`s. +- The existing op-list JSON format is the serialization target, so a trained model loads + into `decoder.rs` unchanged. + +## Architecture + +`decoder.rs` stays the untouched, panic-free inference path. A separate pure training +module produces models; the two communicate only through the JSON op format. Training runs +on nih-plug's background task; the trained decoder hot-swaps via `arc-swap` (wait-free read +on the audio thread). + +### New modules +- **`src/autoencoder.rs`** — pure f64 full AE (encoder 32→16→8→4, decoder 4→8→16→32; + Dense/ReLU/BatchNorm/Sigmoid; manual backprop; Adam). Ported from `AE_init.py` with the + two bug fixes. Tiny inline PCG PRNG for init + shuffle (no `rand`). +- **`src/model_ops.rs`** — `ExportOp`/`ModelExport`/`TrainedModel` (`Serialize+Deserialize`), + byte-compatible with `decoder.json` and `train_export.py`. `export_decoder`/`export_encoder` + + `to_decoder` (round-trips through the same JSON `decoder.rs` consumes). +- **`src/dataset.rs`** — `encode_onsets` (exact port of `corpus_encode.py`, incl. Python + banker's rounding) and `encode_grid` (capture the live pattern). +- **`src/audio.rs`** — `decode_audio` (symphonia → mono f32) and `detect_onsets` + (spectral-flux + adaptive peak-pick via rustfft); `file_to_sample` glues them to + `encode_onsets`. Intentionally not a librosa clone. +- **`src/training.rs`** — `TrainShared` (dataset, pending paths, progress atomics, + `ArcSwap` model + `ArcSwap>` encoder, persisted-model handle), + the background `executor`, and `run_training`/`run_ingest`. + +### Modified +- **`src/lib.rs`** — `BackgroundTask = training::Task`; `task_executor` wired; owned + `decoder` replaced by `train: Arc` whose `ArcSwap` the audio thread + loads; `model_generation` invalidates `last_latent` on swap; `initialize()` restores a + persisted model. +- **`src/params.rs`** — `#[persist = "trained-model"] trained_model: Arc>>`, + shared (same `Arc`) with `TrainShared`. +- **`src/shared.rs`** — per-step `substeps: [AtomicU64; 16]`, written on regeneration, read by + the GUI for capture/encode. +- **`src/editor.rs`** — "Training" panel: dataset size, *Capture pattern* / *Add audio…* / + *Clear*, epochs/batch, *Train* / *Cancel*, progress bar + live loss, model status, and + *Encode pattern → latent* (writes the 4 latents via begin/set/end gestures, clamped to + `[0,1]`). + +### Threading / safety +The audio thread only does a wait-free `model.load()` (an `ArcSwap`) and `Relaxed` atomic +reads — never a `Mutex`. The dataset and trained-model `Mutex`es are GUI/background only. +On training finish the executor exports the model, parses it back into a `Decoder`/encoder, +`ArcSwap::store`s them, bumps `model_generation`, and fills the persisted slot. + +## Verification +- `cargo test --workspace` (34): AE overfit-to-zero (gradient correctness gate), Adam-fix + lock-in, per-epoch shuffle determinism, export↔decoder parity (1e-9), `corpus_encode` + parity with the Python `test_corpus_encode.py` cases, onset-detection sanity, and an + end-to-end train→hot-swap→persist→encode integration test. +- `cargo clippy --all-targets -- -D warnings` clean; CLAP + VST3 bundle builds; both + headless host scale-tests pass (audio path intact through a real host); `pluginval` + (VST3, strictness 8) in CI. +- Manual: train in Carla (VST3), watch the loss bar, confirm the pattern swaps, test + *Encode → latent*, and save/reload the project to verify persistence. From c5209d0a0617f2bc408c82e02793437f07459138 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Tue, 16 Jun 2026 15:38:29 -0300 Subject: [PATCH 2/3] style: satisfy clippy 1.96 lints (needless_range_loop, abs_diff, vec_init, identity_op) CI clippy (-D warnings) failed on 7 style lints that a stale local clippy cache had masked: col_mean and three test loops use enumerate instead of range-index; Autoencoder::new builds via vec![]/extend instead of init-then-push; the onset test uses usize::abs_diff; and a test mask drops a no-op `& 0xFFFF`. No behaviour change; 34 tests still pass. --- deepsteps-plugin/src/audio.rs | 5 +-- deepsteps-plugin/src/autoencoder.rs | 59 ++++++++++++++++------------- deepsteps-plugin/src/training.rs | 2 +- 3 files changed, 34 insertions(+), 32 deletions(-) diff --git a/deepsteps-plugin/src/audio.rs b/deepsteps-plugin/src/audio.rs index f09beb7..16a7491 100644 --- a/deepsteps-plugin/src/audio.rs +++ b/deepsteps-plugin/src/audio.rs @@ -239,10 +239,7 @@ mod tests { // Every real click should have a detected onset within ~1.5 hops. let tol = (HOP as f32 * 1.5) as usize + FRAME; // detection lags by ~a frame for &c in &clicks { - let found = onsets.iter().any(|&o| { - let d = if o > c { o - c } else { c - o }; - d <= tol - }); + let found = onsets.iter().any(|&o| o.abs_diff(c) <= tol); assert!(found, "no onset near click {c}; got {onsets:?}"); } // Should not produce a flood of spurious onsets. diff --git a/deepsteps-plugin/src/autoencoder.rs b/deepsteps-plugin/src/autoencoder.rs index 2cd6efc..5ada745 100644 --- a/deepsteps-plugin/src/autoencoder.rs +++ b/deepsteps-plugin/src/autoencoder.rs @@ -38,8 +38,8 @@ impl Mat { fn col_mean(&self) -> Vec { let mut m = vec![0.0; self.cols]; for r in 0..self.rows { - for c in 0..self.cols { - m[c] += self.at(r, c); + for (c, mc) in m.iter_mut().enumerate() { + *mc += self.at(r, c); } } for v in &mut m { @@ -410,27 +410,32 @@ impl Autoencoder { let input_dim = 32; let latent_dim = 4; let mut rng = Rng::new(seed); - let mut layers = Vec::new(); - - // Encoder 32 -> 16 -> 8 -> 4 - layers.push(Layer::dense(32, 16, LR, B1, B2, &mut rng)); - layers.push(Layer::Activation { kind: ActKind::Relu, in_cache: Mat::zeros(0, 0) }); - layers.push(Layer::bn(16, BN_MOMENTUM, LR, B1, B2)); - layers.push(Layer::dense(16, 8, LR, B1, B2, &mut rng)); - layers.push(Layer::Activation { kind: ActKind::Relu, in_cache: Mat::zeros(0, 0) }); - layers.push(Layer::bn(8, BN_MOMENTUM, LR, B1, B2)); - layers.push(Layer::dense(8, 4, LR, B1, B2, &mut rng)); + let relu = || Layer::Activation { kind: ActKind::Relu, in_cache: Mat::zeros(0, 0) }; + + // Encoder 32 -> 16 -> 8 -> 4. (Each `dense` borrows `rng` only for its + // own call, so the sequential `&mut rng` uses don't overlap.) + let mut layers = vec![ + Layer::dense(32, 16, LR, B1, B2, &mut rng), + relu(), + Layer::bn(16, BN_MOMENTUM, LR, B1, B2), + Layer::dense(16, 8, LR, B1, B2, &mut rng), + relu(), + Layer::bn(8, BN_MOMENTUM, LR, B1, B2), + Layer::dense(8, 4, LR, B1, B2, &mut rng), + ]; let enc_len = layers.len(); - // Decoder 4 -> 8 -> 16 -> 32 - layers.push(Layer::dense(4, 8, LR, B1, B2, &mut rng)); - layers.push(Layer::Activation { kind: ActKind::Relu, in_cache: Mat::zeros(0, 0) }); - layers.push(Layer::bn(8, BN_MOMENTUM, LR, B1, B2)); - layers.push(Layer::dense(8, 16, LR, B1, B2, &mut rng)); - layers.push(Layer::Activation { kind: ActKind::Relu, in_cache: Mat::zeros(0, 0) }); - layers.push(Layer::bn(16, BN_MOMENTUM, LR, B1, B2)); - layers.push(Layer::dense(16, 32, LR, B1, B2, &mut rng)); - layers.push(Layer::Activation { kind: ActKind::Sigmoid, in_cache: Mat::zeros(0, 0) }); + // Decoder 4 -> 8 -> 16 -> 32. + layers.extend([ + Layer::dense(4, 8, LR, B1, B2, &mut rng), + relu(), + Layer::bn(8, BN_MOMENTUM, LR, B1, B2), + Layer::dense(8, 16, LR, B1, B2, &mut rng), + relu(), + Layer::bn(16, BN_MOMENTUM, LR, B1, B2), + Layer::dense(16, 32, LR, B1, B2, &mut rng), + Layer::Activation { kind: ActKind::Sigmoid, in_cache: Mat::zeros(0, 0) }, + ]); Autoencoder { layers, enc_len, input_dim, latent_dim } } @@ -491,8 +496,8 @@ impl Autoencoder { // Build batch matrix (target == input for an autoencoder). let mut x = Mat::zeros(rows, 32); for (r, &idx) in order[start..end].iter().enumerate() { - for c in 0..32 { - x.set(r, c, data[idx][c] as f64); + for (c, &val) in data[idx].iter().enumerate() { + x.set(r, c, val as f64); } } let recon = self.forward_all(&x, true); @@ -612,8 +617,8 @@ mod tests { }, { let mut v = [0.0f32; 32]; - for i in 0..16 { - v[i] = (i % 2) as f32; + for (i, slot) in v.iter_mut().enumerate().take(16) { + *slot = (i % 2) as f32; } v }, @@ -637,8 +642,8 @@ mod tests { // Reconstruction (infer mode) should match each input closely. let mut x = Mat::zeros(data.len(), 32); for (r, row) in data.iter().enumerate() { - for c in 0..32 { - x.set(r, c, row[c] as f64); + for (c, &val) in row.iter().enumerate() { + x.set(r, c, val as f64); } } let recon = ae.forward_all(&x, false); diff --git a/deepsteps-plugin/src/training.rs b/deepsteps-plugin/src/training.rs index 9bc6a9b..1597025 100644 --- a/deepsteps-plugin/src/training.rs +++ b/deepsteps-plugin/src/training.rs @@ -243,7 +243,7 @@ mod tests { { let mut d = train.dataset.lock().unwrap(); for i in 0..8u16 { - let mask = (i.wrapping_mul(37) | 1) & 0xFFFF; + let mask = i.wrapping_mul(37) | 1; let mut ss = [0.0f64; 16]; ss[(i % 16) as usize] = 0.5; d.push(encode_grid(mask, &ss)); From d6e3939c7c8db4ae4d84d8199cd88f652a1f8050 Mon Sep 17 00:00:00 2001 From: Gustavo Date: Tue, 16 Jun 2026 17:34:14 -0300 Subject: [PATCH 3/3] fix(training): retire hot-swapped decoders off the audio thread MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address PR #6 review. M1 (RT-safety): `ArcSwap::load()` is wait-free but not allocation-free — after a `store`, the audio thread could hold the last reference to the old `Arc` and free its heap inside `process()` when the load guard dropped. Hot-swaps now go through `TrainShared::swap_model`, which uses `ArcSwap::swap` and parks the displaced decoder in a generation- tagged graveyard. The audio thread publishes `gen_acked` at the end of `maybe_regen` (after its load guard is dropped, Release-ordered); `collect_garbage` (GUI/background thread, Acquire) then drops only the retired decoders the audio thread has provably moved past, so the heap free never runs on the audio thread. Drained from the editor frame, the background executor, and `initialize`. New unit test covers retire/ack/ collect and the live decoder staying intact. M2: `run_ingest` now restores the captured prior status verbatim (so a prior `Done` survives an audio ingest) instead of collapsing everything non-`Running` to `Idle`; only a transient `Ingesting` falls back to Idle. L1: "Encode pattern → latent" gains a hover note that latents are clamped to 0..1 so the round-trip is approximate. L2: documented that batch size acts as a secondary LR knob in `fit`. L4: documented that `pick_files()` blocks only the editor thread. L6: `initialize` uses `swap_model` (retiring restore) and collects garbage; the redundant-swap concern is now bounded by the graveyard and never touches the audio thread. --- deepsteps-plugin/src/autoencoder.rs | 6 ++ deepsteps-plugin/src/editor.rs | 18 +++++ deepsteps-plugin/src/lib.rs | 17 +++- deepsteps-plugin/src/training.rs | 118 +++++++++++++++++++++++++--- 4 files changed, 146 insertions(+), 13 deletions(-) diff --git a/deepsteps-plugin/src/autoencoder.rs b/deepsteps-plugin/src/autoencoder.rs index 5ada745..f2e2342 100644 --- a/deepsteps-plugin/src/autoencoder.rs +++ b/deepsteps-plugin/src/autoencoder.rs @@ -464,6 +464,12 @@ impl Autoencoder { /// Train on `data` (each row a 32-dim sample). `on_epoch(epoch, avg_loss)` /// is called after every epoch; returning `false` cancels training. /// Returns the number of completed epochs. + /// + /// Note: the backward pass sums (does not average) the per-element loss + /// gradient over a batch's rows, matching the Python original. So the + /// weight-gradient magnitude scales with `batch`, and `batch` acts as a + /// secondary learning-rate knob — Adam's per-parameter normalisation absorbs + /// most of this, but larger batches still train slightly more aggressively. pub fn fit( &mut self, data: &[[f32; 32]], diff --git a/deepsteps-plugin/src/editor.rs b/deepsteps-plugin/src/editor.rs index 1313e74..899fff7 100644 --- a/deepsteps-plugin/src/editor.rs +++ b/deepsteps-plugin/src/editor.rs @@ -48,6 +48,11 @@ pub fn create( }, |_ctx, _state| {}, |ctx, setter, state| { + // Reclaim decoders retired by a hot-swap. This runs on the GUI thread + // (never the audio thread), so the old model's heap is freed here, not + // inside `process()` (see `TrainShared::swap_model`). + state.train.collect_garbage(); + // Keep repainting while the playhead moves or a training run is in // progress (so the progress bar advances). Otherwise let egui idle. if state.shared.current() != NO_STEP @@ -165,6 +170,11 @@ fn training_section(ui: &mut egui::Ui, setter: &ParamSetter, state: &EditorState } } if ui.add_enabled(!busy, egui::Button::new("Add audio…")).clicked() { + // `pick_files()` is a blocking, modal native dialog: it stalls + // this GUI frame until the user dismisses it. That is fine — + // it blocks only the editor thread, never the audio thread — + // and the actual decode/onset work is dispatched to the + // background thread below. if let Some(files) = rfd::FileDialog::new() .add_filter("audio", &["wav", "flac"]) .pick_files() @@ -232,8 +242,16 @@ fn training_section(ui: &mut egui::Ui, setter: &ParamSetter, state: &EditorState "Model: {}", if trained { "Trained" } else { "Default (baked)" } )); + // The encoder's output is unbounded, but the latent params are + // `[0,1]`, so `set_latents` clamps. Encoding then re-decoding a + // pattern is therefore approximate, not a faithful round-trip — + // flag it on hover so the result isn't surprising. if ui .add_enabled(trained, egui::Button::new("Encode pattern → latent")) + .on_hover_text( + "Sets the latent sliders to this pattern's encoded latent.\n\ + Values are clamped to 0..1, so re-decoding is approximate.", + ) .clicked() { let grid = crate::dataset::encode_grid(shared.mask(), &shared.substeps()); diff --git a/deepsteps-plugin/src/lib.rs b/deepsteps-plugin/src/lib.rs index 555c12d..067ab31 100644 --- a/deepsteps-plugin/src/lib.rs +++ b/deepsteps-plugin/src/lib.rs @@ -121,7 +121,12 @@ impl DeepSteps { self.shared.set_substeps(&ss); self.substeps = ss; self.last_latent = z; + // `decoder` guard drops here, at the end of the block. } + // Acknowledge this generation *after* any load guard above is dropped, so + // `collect_garbage` can prove the retired decoder has no live reader and + // free it off the audio thread (see `TrainShared::swap_model`). + self.train.ack_generation(gen); } } @@ -174,18 +179,24 @@ impl Plugin for DeepSteps { // State has already been restored at this point: if the host loaded a // trained model, hot-swap it into the audio + encode paths (overriding - // the baked default). The bump forces `maybe_regen` to use it. + // the baked default). `swap_model` bumps the generation, forcing + // `maybe_regen` to pick it up, and retires the displaced decoder off the + // audio thread. `initialize` can be called repeatedly (e.g. on a + // sample-rate change) and also runs on a project/preset reload, so we + // restore every time rather than guarding on generation — the graveyard + // bounds the cost and the swap never frees on the audio thread. if let Ok(slot) = self.params.trained_model.lock() { if let Some(tm) = slot.as_ref() { if let Ok(dec) = model_ops::to_decoder(&tm.decoder) { - self.train.model.store(Arc::new(dec)); + self.train.swap_model(dec); } if let Ok(enc) = model_ops::to_decoder(&tm.encoder) { self.train.encoder.store(Arc::new(Some(enc))); } - self.train.model_generation.fetch_add(1, Relaxed); } } + // Reclaim anything the restore swap retired (main thread, not audio). + self.train.collect_garbage(); true } diff --git a/deepsteps-plugin/src/training.rs b/deepsteps-plugin/src/training.rs index 1597025..40838eb 100644 --- a/deepsteps-plugin/src/training.rs +++ b/deepsteps-plugin/src/training.rs @@ -7,9 +7,20 @@ //! The audio thread only ever does a wait-free `model.load()` (an `ArcSwap`) and //! a few `Relaxed` atomic reads — it never locks a `Mutex`. The dataset and //! trained-model `Mutex`es are touched solely by the GUI and background threads. +//! +//! Hot-swaps go through [`TrainShared::swap_model`], which retires the previous +//! decoder into a graveyard instead of dropping it. A wait-free `load()` is not +//! allocation-free: if the audio thread held the last reference to a swapped-out +//! `Arc`, dropping its load guard would free that decoder's heap inside +//! `process()`. The graveyard keeps the old decoder alive until the audio thread +//! has published (via `gen_acked`) that it has moved past it, at which point +//! [`TrainShared::collect_garbage`] drops it on the GUI/background thread. use std::path::PathBuf; -use std::sync::atomic::{AtomicBool, AtomicU64, AtomicU8, AtomicUsize, Ordering::Relaxed}; +use std::sync::atomic::{ + AtomicBool, AtomicU64, AtomicU8, AtomicUsize, + Ordering::{Acquire, Relaxed, Release}, +}; use std::sync::{Arc, Mutex}; use arc_swap::ArcSwap; @@ -91,6 +102,16 @@ pub struct TrainShared { pub encoder: ArcSwap>, /// Persisted trained model (shared with the `#[persist]` param field). pub trained_model: Arc>>, + + /// Decoders retired by [`swap_model`](Self::swap_model), tagged with the + /// `model_generation` that replaced them. Kept alive — never dropped on the + /// audio thread — until `gen_acked` shows the audio thread has moved past + /// them, then dropped by [`collect_garbage`](Self::collect_garbage). + graveyard: Mutex)>>, + /// Highest `model_generation` the audio thread has finished a regen for, + /// published at the end of `maybe_regen` *after* its load guard is dropped. + /// Lets `collect_garbage` prove an old decoder has no live audio reader left. + pub gen_acked: AtomicU64, } impl TrainShared { @@ -110,9 +131,44 @@ impl TrainShared { model: ArcSwap::from_pointee(initial_decoder), encoder: ArcSwap::from_pointee(None), trained_model, + graveyard: Mutex::new(Vec::new()), + gen_acked: AtomicU64::new(0), + } + } + + /// Hot-swap the audio-thread decoder, retiring the previous one into the + /// graveyard so its eventual `Drop` (heap free) runs on a non-audio thread, + /// never inside `process()`. Bumps `model_generation` so the audio thread + /// regenerates. Returns the new generation. Call only off the audio thread. + pub fn swap_model(&self, dec: Decoder) -> u64 { + // Publish the new decoder pointer *before* bumping the generation, so the + // audio thread never sees a new generation pointing at the old decoder. + let old = self.model.swap(Arc::new(dec)); + let gen = self.model_generation.fetch_add(1, Relaxed) + 1; + if let Ok(mut g) = self.graveyard.lock() { + g.push((gen, old)); + } + gen + } + + /// Drop every retired decoder the audio thread has provably finished with + /// (`gen_acked >= gen`). Runs on the GUI/background thread; the heap free of + /// each dropped decoder therefore happens here, off the audio thread. The + /// `Acquire` load pairs with the audio thread's `Release` store of + /// `gen_acked`, ordering its load-guard drop before this drop. + pub fn collect_garbage(&self) { + let acked = self.gen_acked.load(Acquire); + if let Ok(mut g) = self.graveyard.lock() { + g.retain(|(gen, _)| acked < *gen); } } + /// Audio thread: record that a `maybe_regen` cycle for `gen` is complete and + /// its load guard dropped. `Release` so `collect_garbage`'s `Acquire` sees it. + pub fn ack_generation(&self, gen: u64) { + self.gen_acked.store(gen, Release); + } + pub fn status(&self) -> TrainStatus { TrainStatus::from_u8(self.status.load(Relaxed)) } @@ -142,9 +198,14 @@ impl TrainShared { /// Build the background task executor closure. Called once by `Plugin::task_executor`. pub fn executor(train: Arc) -> Box { - Box::new(move |task| match task { - Task::Train { epochs, batch, seed } => run_training(&train, epochs, batch, seed), - Task::IngestAudio => run_ingest(&train), + Box::new(move |task| { + // Reclaim retired decoders here too (this is a non-audio thread), so the + // graveyard is bounded even when the editor is closed. + train.collect_garbage(); + match task { + Task::Train { epochs, batch, seed } => run_training(&train, epochs, batch, seed), + Task::IngestAudio => run_ingest(&train), + } }) } @@ -180,15 +241,18 @@ fn run_training(train: &Arc, epochs: usize, batch: usize, seed: u64 match (dec_exp, enc_exp) { (Ok(dec_exp), Ok(enc_exp)) => { if let Ok(dec) = model_ops::to_decoder(&dec_exp) { - train.model.store(Arc::new(dec)); + // Retiring swap (bumps model_generation); old decoder is freed + // off the audio thread by `collect_garbage`. + train.swap_model(dec); } if let Ok(enc) = model_ops::to_decoder(&enc_exp) { + // The encoder is never read on the audio thread, so a plain store + // (which drops the old encoder on this thread) is already safe. train.encoder.store(Arc::new(Some(enc))); } if let Ok(mut slot) = train.trained_model.lock() { *slot = Some(TrainedModel { decoder: dec_exp, encoder: enc_exp }); } - train.model_generation.fetch_add(1, Relaxed); train.set_status(if completed < epochs { TrainStatus::Cancelled } else { @@ -217,11 +281,13 @@ fn run_ingest(train: &Arc) { Err(e) => nih_plug::nih_log!("DeepSteps: ingest failed for {p:?}: {e}"), } } - // Restore the prior status (e.g. keep showing Done) unless training is active. - train.set_status(if prev == TrainStatus::Running { - TrainStatus::Running - } else { + // Restore whatever status was showing before the ingest (e.g. keep showing + // `Done` from a prior training run), but never resurrect a transient + // `Ingesting` — fall back to `Idle` for that one case. + train.set_status(if prev == TrainStatus::Ingesting { TrainStatus::Idle + } else { + prev }); } @@ -273,6 +339,38 @@ mod tests { assert_eq!(out.len(), 32); } + /// M1: retiring swaps park old decoders in the graveyard; `collect_garbage` + /// drops only those the audio thread has acked, and the live model still runs. + #[test] + fn swap_model_retires_and_collects() { + let train = shared(); + + // Three swaps -> three retired decoders, generation advances to 3. + for _ in 0..3 { + train.swap_model(Decoder::empty()); + } + assert_eq!(train.model_generation.load(Relaxed), 3); + assert_eq!(train.graveyard.lock().unwrap().len(), 3); + + // Audio thread hasn't acked anything yet: nothing is safe to drop. + train.collect_garbage(); + assert_eq!(train.graveyard.lock().unwrap().len(), 3); + + // Audio acks through generation 2: gens 1 and 2 are reclaimed, 3 stays. + train.ack_generation(2); + train.collect_garbage(); + assert_eq!(train.graveyard.lock().unwrap().len(), 1); + + // Acking the final generation drains the rest. + train.ack_generation(3); + train.collect_garbage(); + assert!(train.graveyard.lock().unwrap().is_empty()); + + // The live (most recent) decoder is intact and still generates a pattern. + let (steps, _) = train.model.load().generate(&[0.5, 0.5, 0.5, 0.5]); + assert_eq!(steps.len(), 16); + } + /// Training with an empty dataset reports an error and swaps nothing. #[test] fn train_empty_dataset_errors() {