feat/phase3-perf-and-refactor

backend/app/api/routes_library.py

@@ -6,12 +6,16 @@
 # Foundation, either version 3 of the License, or (at your option) any later
 # version. Distributed WITHOUT ANY WARRANTY. See the GNU General Public License
 # <https://www.gnu.org/licenses/> for details.
+import logging
+
 from fastapi import APIRouter, HTTPException
 from pydantic import BaseModel
 
 from app.services.library import save_generation, list_library, exclude_generation
 from app.core.config import EXPORTS_DIR
 
+logger = logging.getLogger(__name__)
+
 router = APIRouter(prefix="/library", tags=["library"])
 
 # How much more a user-kept generation counts than a merely high-scoring one.
@@ -38,6 +42,7 @@ def record_export_keep(gen_id: str) -> bool:
         )
         return True
     except Exception:
+        logger.warning("Failed to record export keep for gen_id=%s", gen_id, exc_info=True)
         return False
 
 

backend/app/services/library.py

@@ -74,7 +74,7 @@ def save_generation(
                 if prev.get("source") in ("thumbs_up", "export") and source == "score":
                     source = prev["source"]   # don't demote an explicit keep
             except Exception:
-                pass
+                _logger.debug("Ignoring unreadable prior library entry %s", path, exc_info=True)
         entry = {
             "gen_id":   gen_id,
             "style_id": style_id,

backend/app/services/priors.py

@@ -14,9 +14,12 @@
 templates. Priors live in ``backend/app/priors/<genre>.json``.
 """
 import json
+import logging
 import random
 from pathlib import Path
 
+logger = logging.getLogger(__name__)
+
 _PRIORS_DIR = Path(__file__).resolve().parent.parent / "priors"
 _GROOVES_DIR = _PRIORS_DIR / "grooves"
 
@@ -47,6 +50,7 @@ def load_groove(name: str) -> dict | None:
         try:
             g = json.loads(path.read_text())
         except Exception:
+            logger.warning("Malformed groove prior %s — ignoring", path, exc_info=True)
             g = None
     _groove_cache[name] = g
     return g
@@ -78,6 +82,7 @@ def load_prior(genre: str) -> dict | None:
         try:
             prior = json.loads(path.read_text())
         except Exception:
+            logger.warning("Malformed genre prior %s — ignoring", path, exc_info=True)
             prior = None
     _cache[genre] = prior
     return prior

backend/requirements-lock.txt

@@ -7,8 +7,10 @@ click==8.3.2
 fastapi==0.139.0
 h11==0.16.0
 httpcore==1.0.9
+httpcore2==2.9.1
 httptools==0.8.0
 httpx==0.28.1
+httpx2==2.9.1
 idna==3.18
 iniconfig==2.3.0
 mido==1.3.3
@@ -27,6 +29,7 @@ PyYAML==6.0.3
 ruff==0.15.22
 setuptools==83.0.0
 starlette==1.3.1
+truststore==0.10.4
 typing-inspection==0.4.2
 typing_extensions==4.15.0
 uvicorn==0.50.0

backend/requirements.txt

@@ -5,7 +5,9 @@ mido>=1.3.2
 python-multipart>=0.0.32
 pytest>=8.2.0
 pytest-asyncio>=0.23.7
-httpx>=0.27.0
+# httpx2 (not httpx): Starlette 1.3+ TestClient prefers httpx2 and warns on plain
+# httpx as deprecated. Test-only dependency (nothing imports httpx at runtime).
+httpx2>=2.0.0
 
 # Security floors for transitive deps (patched versions — see Dependabot alerts)
 starlette>=1.3.1

backend/tests/test_grooves.py

@@ -56,7 +56,7 @@ def test_groove_overlay_and_runtime(tmp_path, monkeypatch):
     """A groove prior should overlay the style's drums and drive live generation."""
     import app.services.priors as priors
     from app.services.priors import groove_fields_for
-    from app.api.routes_generate import _overlay_groove
+    from app.services.generation import _overlay_groove
     from app.models.schemas import GenerateRequest
     from app.api.routes_generate import generate
 

backend/tests/test_musicality.py

@@ -304,7 +304,7 @@ def test_dj_edit_adds_beat_only_bookends():
 
 
 def test_bridge_escape_opens_off_path_and_walks_home():
-    from app.api.routes_song import _bridge_escape_progression
+    from app.services.song_builder import _bridge_escape_progression
 
     # I-heavy major song → bridge opens on vi (unused), ends on a dominant pedal.
     prog, opener = _bridge_escape_progression(["I", "IV", "V", "I"], "major")

backend/tests/test_progression_mode.py

@@ -20,7 +20,8 @@
 import json
 from pathlib import Path
 
-from app.api.routes_generate import _choose_progression, _scale_mode, _template_tonic_mode
+from app.services.generation import _choose_progression, _template_tonic_mode
+from app.theory.scales import scale_mode as _scale_mode
 
 STYLES_DIR = Path(__file__).parent.parent / "app" / "styles"
 
@@ -79,7 +80,7 @@ def test_no_shipped_style_has_mixed_tonic_templates():
 
 
 def test_resolve_avoid_notes():
-    from app.api.routes_generate import _resolve_avoid_notes
+    from app.services.generation import _resolve_avoid_notes
     from app.services.midi_writer import NoteEvent
 
     c_minor_pcs = {0, 2, 3, 5, 7, 8, 10}

backend/tests/test_song.py

@@ -42,7 +42,7 @@ def test_regenerate_song_part_isolates_the_target():
 def test_recurring_sections_reuse_the_theme():
     """Verse 2 reuses Verse's theme (with light variation); drums stay fresh."""
     from app.services.style_loader import load_style
-    from app.api.routes_song import _generate_song_sections
+    from app.services.song_builder import _generate_song_sections
 
     req = BuildSongRequest(style_id="lofi", key="C", scale="major", bpm=90,
                            template="verse_chorus", parts=["chords", "bass", "melody", "drums"],

docs/roadmap.md

@@ -6,9 +6,11 @@ they land; add new findings under the right phase so nothing gets lost.
 **Status legend:** `[ ]` todo · `[~]` in progress · `[x]` done · `[-]` won't do / obsolete ·
 `[→]` moved to another phase
 
-_Last updated: 2026-07-23 — Phase 1 complete. Phase 2 complete except for desktop runtime
-testing of the custom-instruments MVP: limiter, per-style FX, loudness norm, velocity-layer
-engine, license compliance, and the sample re-sourcing pass have all landed._
+_Last updated: 2026-07-28 — Phase 1 & 2 complete (Phase 2 bar the manual custom-instruments
+desktop pass). Phase 3 well underway: the "parallelize generation" item was retired as a
+phantom (generation is ~0.5s; the real wait is the WAV export render, now parallelised),
+deprecation warnings cleared, swallowed exceptions logged, and the two route god-files
+split into `services/`. Remaining Phase 3: split `useMidiPlayer.ts`, grow frontend tests._
 
 ---
 
@@ -131,19 +133,76 @@ engine, license compliance, and the sample re-sourcing pass have all landed._
 
 ## Phase 3 — Performance & refactor
 
-- [ ] **Parallelize full-song section generation** — song builds take ~1–2 min and
-  sections generate serially though they're embarrassingly parallel. SSE progress
-  plumbing (`generate-stream`) already exists.
+- [-] **Parallelize full-song section generation** — _obsolete: the premise was
+  wrong._ Measured 2026-07-28: backend section generation is **0.5–0.8s** end-to-end
+  (`_do_build_song`, every style, up to max complexity / 12-section templates), not the
+  claimed "~1–2 min". Building a song never renders audio — it calls the backend (~0.5s)
+  then plays back **live**, so the "1–2 min" was the song's own playback length (a 56-bar
+  song at 120 BPM *is* ~112s of audio), not generation time. Parallelizing a sub-second
+  step would *regress* latency (process-pool spawn + pickling > the serial work) and, per
+  the dependency audit, isn't even embarrassingly parallel: the section loop threads real
+  cross-section state (`prev_voicing` seam voice-leading, `verse_motif`/`rhythm_cell`
+  motif glue, `type_theme` repeat reuse) and generation uses the process-global `random`
+  module (17 `random.seed()` calls) so threads can't run it deterministically. The only
+  genuinely CPU-heavy path is the **on-demand WAV export** (`offlineRender`, synth-only
+  `OfflineAudioContext`) — tracked as a new item below.
   → `backend/app/api/routes_song.py`
-- [ ] **Extract song/arrangement logic out of route god-files into `services/`** —
-  handlers should be thin. → `routes_song.py` (1498), `routes_generate.py` (1299)
+- [~] **Speed up WAV export render** — _this is the real "1–2 min" wait, not generation._
+  Measured 2026-07-28 (headless Chromium, real Web Audio, 112s song / 2,576 voices, graph
+  matching `offlineRender`): **56.4s** with the full FX chain, **51.8s** with all FX removed.
+  The actual Tone.js app render is ≥ this (per-voice JS wrappers, `standardized-audio-context`
+  shims, `render(true)` yields). **The bottleneck is per-note voice synthesis (~92%), NOT the
+  reverb (~8%)** — so caching the convolution IR is nearly worthless (an earlier guess, now
+  disproven by measurement).
+  **Shipped — parallel per-part render.** Empirically, concurrent `OfflineAudioContext`
+  renders run on separate Chromium threads (probe: 4 contexts, 9.2s serial → 3.7s parallel,
+  **2.48× on 8 cores**), so `offlineRender` now renders each part on its own context via
+  `Promise.all` instead of one big context. The offline graph has no shared reverb — every
+  voice runs its own delay/chorus/filter into a plain gain — so the per-part outputs SUM to
+  exactly the old pre-limiter mix; the master soft-clip limiter is then applied once to the
+  sum in a raw-WaveShaper pass that reproduces `makeMasterLimiter` bit-for-bit (curve =
+  `softClipCurve` sampled Tone's way, `oversample='4x'`). Result is mathematically identical
+  to the old single-pass render (bar float summation order), just parallelised. A single-part
+  stem export keeps the limiter inline (nothing to parallelise). Validated: pure mix logic
+  unit-tested (`offlineMix.test.ts`); full end-to-end run in real Chromium produces valid
+  non-silent audio via the parallel path; `vue-tsc` + eslint + 96 vitest green.
+  **Still open:** a representative desktop wall-time before/after — headless SwiftShader
+  renders the Tone graph ~10× slower than realtime, so it can't measure the real speedup;
+  needs a WAV export in the actual desktop Electron app (GPU Chromium) to confirm the number.
+  Bench harness: `frontend/scripts/bench_render.mjs` (drives cached Chromium over CDP; no display).
+  → `frontend/src/composables/useMidiPlayer.ts` (`offlineRender`, `renderOfflineFast`,
+  `sumPartBuffers`, `limitMix`)
+- [~] **Extract song/arrangement logic out of route god-files into `services/`** —
+  handlers should be thin. Done for the two biggest logic blobs, each verified by the 135
+  backend tests + ruff: the **generation core** (`_run_attempt`, progression choice, style
+  blend/groove overlay, voice-leading + quality helpers) → **`services/generation.py`**, and
+  the **song arrangement engine** (`_generate_song_sections` — the 465-line section loop —
+  plus its motif/voice-leading helpers, the combined-MIDI writer, and bridge-escape) →
+  **`services/song_builder.py`**. Both route files (and the new services) now depend on
+  `services/generation.py` instead of importing generation logic out of `routes_generate.py`.
+  `routes_generate.py` **1299 → 636**, `routes_song.py` **1498 → 889**.
+  _Remaining follow-up:_ the song-build coordinator `_do_build_song` and a few
+  regenerate/stem/version helpers still sit in `routes_song.py` between the endpoints; they
+  could move into `song_builder.py` too, but the endpoints that call them are already thin.
 - [ ] **Split `useMidiPlayer.ts`** (1018 lines) into focused composables.
-- [ ] **Log swallowed exceptions** — 27 broad/bare `except`; several return silently
-  (e.g. `record_export_keep`). Add logging.
+- [x] **Log swallowed exceptions** — audited all 27 broad `except` blocks. 12 already
+  logged or re-raised; 6 genuine silent faults now log — `record_export_keep`
+  (`routes_library.py`, warning), malformed groove/genre priors (`priors.py`, warning),
+  library keep-merge on a corrupt prior entry (`library.py`, debug), style-load for track
+  names (`routes_song.py` ×2, warning), and the arpeggio chord-tone derivation
+  (`routes_generate.py`, debug). The rest were left deliberately: `mido_key_signature`'s
+  `except` is by-design control flow (exotic modes have no MIDI key sig, documented), and
+  the generator inner-loop `except`s (bass/melody/quality/answer/corpus) are intentional
+  musical graceful-degradation, not swallowed faults — logging each would be pure noise.
 - [ ] **Grow frontend test coverage** — 12 tests vs 132 backend; the audio scheduler and
-  WAV-render path (most fragile code) are untested.
-- [ ] **Clear deprecation warnings** — Starlette `TestClient` httpx warning; Electron
-  numeric-level `console-message` signature. → `frontend/electron/main.ts:206`
+  WAV-render path (most fragile code) are untested. _(Up to 96 frontend tests now — the
+  WAV-render mix path gained `offlineMix.test.ts` — but the scheduler is still untested.)_
+- [x] **Clear deprecation warnings** — **Starlette `TestClient`**: moved the test dep from
+  `httpx` to `httpx2` (`requirements.txt`); TestClient prefers httpx2 and the
+  `StarletteDeprecationWarning` is gone, 135 backend tests still green. **Electron
+  `console-message`**: rewrote the handler for the Electron 36+ single-`details` signature
+  (string `level` instead of the deprecated positional numeric level — which the old code
+  also mapped wrong). → `frontend/electron/main.ts`
 
 ## Phase 4 — Features
 
@@ -160,6 +219,28 @@ engine, license compliance, and the sample re-sourcing pass have all landed._
 
 ## Done log
 
+- **2026-07-28 — Phase 3 performance & refactor (first pass):**
+  - **Retired "parallelize section generation"** — measured generation at **0.5–0.8s**
+    (not the claimed 1–2 min), so parallelising it would only add process overhead. The real
+    "1–2 min" is the on-demand **WAV export render** (~56s, headless-Chromium bench).
+  - **Parallel per-part WAV export** — `offlineRender` now renders each part on its own
+    `OfflineAudioContext` concurrently (Chromium runs them on separate threads — measured
+    2.48× on 8 cores), sums them, and applies the master limiter once in a raw-WaveShaper
+    pass that reproduces `makeMasterLimiter` bit-for-bit. Mathematically identical to the old
+    single-pass mix. Pure mix logic unit-tested (`offlineMix.test.ts`); real-browser e2e
+    produced valid audio. Desktop wall-time number still to confirm.
+  - **Auto-pause playback on WAV export** — the render walks the timeline on the main thread
+    and starved the live Transport scheduler (playback froze mid-export); `offlineRender` now
+    pauses playback for the duration (MIDI export was never affected).
+  - **Route god-files split into `services/`** — generation core → `services/generation.py`,
+    song arrangement engine → `services/song_builder.py`. `routes_generate.py` 1299→636,
+    `routes_song.py` 1498→889; killed the route→route generation import.
+  - **Deprecation warnings cleared** — test dep `httpx`→`httpx2` (Starlette TestClient);
+    Electron `console-message` rewritten for the 36+ single-`details` signature.
+  - **Swallowed exceptions logged** — 6 genuine silent faults now log (`record_export_keep`,
+    malformed priors, style-load for track names, …); by-design fallbacks left alone.
+  - Tests: backend 135 green + ruff clean; frontend 92→96 green + `vue-tsc`/eslint clean.
+
 - **2026-07-23 — Phase 1 security hardening** (uncommitted working changes):
   - Export downloads hardened against path traversal (`_safe_export_dir` containment
     check on `/exports/{gen_id}/bundle.zip`, `/sections.zip`, and `/{filename}`).

frontend/electron/main.ts

@@ -230,9 +230,13 @@ async function createWindow(): Promise<void> {
 
   // ── DEBUG: mirror the whole renderer console into the main-process stdout, so the full
   // renderer log (audio setup, errors, everything) shows in the terminal without DevTools.
-  win.webContents.on('console-message', (_e, level, message) => {
-    const tag = ['LOG', 'WARN', 'ERR', 'INFO'][level] ?? 'LOG'
-    console.log(`[renderer:${tag}] ${message}`)
+  // Electron 36+ replaced the positional (event, level:number, message, …) signature with a
+  // single details object whose `level` is a string — the old form is deprecated. Map the
+  // string severity to a short tag (the old numeric mapping was also wrong: 0-3 is
+  // verbose/info/warning/error, not the log/warn/err/info it assumed).
+  win.webContents.on('console-message', (details) => {
+    const tag = ({ info: 'INFO', warning: 'WARN', error: 'ERR', debug: 'LOG' } as const)[details.level] ?? 'LOG'
+    console.log(`[renderer:${tag}] ${details.message}`)
   })
   win.webContents.on('did-fail-load', (_e, code, desc, url) => {
     console.error(`[main] renderer FAILED to load: code=${code} "${desc}" url=${url}`)