From 2177bb655abfded0d3f3b11a53d17774d8bcdc24 Mon Sep 17 00:00:00 2001 From: Thales Pereira <31625914+thcp@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:18:59 +0100 Subject: [PATCH 1/4] fix(desktop): NVIDIA build silently falling back to CPU (#247) Three independent defects each land the NVIDIA build on CPU with no visible error and no recovery path: 1. The cpu-only marker was trusted in the shared per-user data dir, not just the app root. The CPU build wrote/migrated that marker there, so anyone who ever ran the CPU build got the NVIDIA build permanently pinned to CPU -- GPU detection never even ran. is_cpu_only_package now checks the app root only; a stale data-dir marker is auto-deleted and logged. 2. A CPU result from a transient failure (no GPU detected, CUDA verify failed) was persisted the same as a real CPU-only package, and the setup gate treated any truthy torchDevice as "done" -- one bad first run pinned CPU forever. Device selection now persists a reason (torchDeviceReason), and the setup gate only treats cuda/mps or a genuine cpu-only package as settled; a failure-born CPU or a legacy install with no reason re-probes the GPU on the next launch. Existing affected installs self-heal on relaunch, no user action needed. 3. nvidia-smi discovery only checked System32 and PATH; some DCH driver installs place it only under DriverStore\FileRepository\nv*\. Added that scan (newest package wins) and raised the first probe's timeout to 30s for Optimus laptops waking a sleeping dGPU. Every detection decision is now logged to setup.log. Also drops the Windows CPU-only portable package's data\cpu-only staging (scripts/windows/make-portable.ps1), which was the source of the poisoned marker. 5 new Rust unit tests cover marker precedence, the self-heal + log line, CPU builds not churning their own marker, and the DriverStore newest-wins scan. --- desktop/src-tauri/src/main.rs | 256 ++++++++++++++++++++++++++---- desktop/ui/setup.js | 13 +- scripts/windows/make-portable.ps1 | 4 +- 3 files changed, 244 insertions(+), 29 deletions(-) diff --git a/desktop/src-tauri/src/main.rs b/desktop/src-tauri/src/main.rs index bcd2fd30..294600dc 100644 --- a/desktop/src-tauri/src/main.rs +++ b/desktop/src-tauri/src/main.rs @@ -102,6 +102,11 @@ struct RuntimeProbe { ffmpeg_ready: bool, /// Persisted from previous setup run; None means GPU step hasn't run yet. torch_device: Option, + /// Why the persisted device was chosen (e.g. "verified", "no-gpu-detected", + /// "cuda-verify-failed", "cpu-only-package"). None on installs that predate + /// reason tracking -- the setup gate treats those as unsettled so a wrongly + /// pinned CPU heals itself on the next launch (#247). + torch_device_reason: Option, } #[derive(Deserialize, Serialize, Clone)] @@ -166,6 +171,8 @@ struct GpuSetup { cuda_version: Option, torch_device: String, cuda_verified: bool, + /// Why this device was chosen; mirrors the persisted torchDeviceReason. + reason: String, } fn main() { @@ -330,6 +337,7 @@ fn probe_runtime() -> Result { } let ffmpeg = resolve_existing_ffmpeg(&data_dir); let torch_device = read_config_str(&data_dir, "torchDevice"); + let torch_device_reason = read_config_str(&data_dir, "torchDeviceReason"); Ok(RuntimeProbe { app_root: root.display().to_string(), data_dir: data_dir.display().to_string(), @@ -338,6 +346,7 @@ fn probe_runtime() -> Result { ffmpeg_ready: ffmpeg.is_some(), ffmpeg_path: ffmpeg.map(|p| p.display().to_string()), torch_device, + torch_device_reason, }) } @@ -505,10 +514,6 @@ fn ensure_workspace() -> Result<(), String> { fs::create_dir_all(data.join(dir)) .map_err(|e| format!("failed to create data/{dir}: {e}"))?; } - if is_cpu_only_package(&root, &data) { - fs::write(data.join("cpu-only"), "") - .map_err(|e| format!("failed to write data/cpu-only: {e}"))?; - } let config = data.join("config.json"); if !config.exists() { fs::write( @@ -683,15 +688,20 @@ fn ensure_torch_device(state: tauri::State) -> Result) -> Result) -> Result { let index_url = cuda_index_url(compute_cap.as_deref(), &cuda_version); install_cuda_torch(&python, &index_url, &state)?; let cuda_verified = verify_cuda_torch(&python); + let reason = if cuda_verified { + "verified" + } else { + "cuda-verify-failed" + }; GpuSetup { gpu_detected: true, gpu_name: Some(gpu_name), cuda_version: Some(cuda_version), torch_device: if cuda_verified { "cuda" } else { "cpu" }.to_string(), cuda_verified, + reason: reason.to_string(), } } None => GpuSetup { @@ -739,22 +760,64 @@ fn ensure_torch_device(state: tauri::State) -> Result bool { - root.join("cpu-only").is_file() || data_dir.join("cpu-only").is_file() +/// The `cpu-only` marker is trusted ONLY in the app root: it ships inside the +/// package, so it is always correct for the running build. The per-user data +/// dir is shared across installs -- honoring a marker there let a previously +/// installed CPU build permanently force the NVIDIA build onto CPU (#247). +fn is_cpu_only_package(root: &Path) -> bool { + root.join("cpu-only").is_file() } -fn persist_torch_device(data_dir: &std::path::Path, device: &str) { +/// Deletes a stale `cpu-only` marker left in the shared data dir by an older +/// CPU-build install (which used to write/migrate it there). Without this, the +/// NVIDIA build would keep re-reading it forever on builds that trusted the +/// data-dir copy. Best-effort; logs so setup.log tells the story (#247). +fn clear_stale_cpu_marker(root: &Path, data_dir: &Path) { + let stale = data_dir.join("cpu-only"); + if !is_cpu_only_package(root) && stale.is_file() { + match fs::remove_file(&stale) { + Ok(()) => append_to_setup_log( + data_dir, + "removed stale cpu-only marker left by a previous CPU-build install; \ + GPU detection will run", + ), + Err(e) => append_to_setup_log( + data_dir, + &format!("could not remove stale cpu-only marker: {e}"), + ), + } + } +} + +fn persist_torch_device(data_dir: &std::path::Path, device: &str, reason: &str) { let _ = update_setup_config( data_dir, - [("torchDevice", serde_json::Value::String(device.to_string()))], + [ + ("torchDevice", serde_json::Value::String(device.to_string())), + ( + "torchDeviceReason", + serde_json::Value::String(reason.to_string()), + ), + ], ); } @@ -772,39 +835,93 @@ fn verify_mps_torch(python: &Path) -> bool { .unwrap_or(false) } +/// Finds nvidia-smi.exe under a DriverStore FileRepository directory. Modern +/// NVIDIA DCH drivers sometimes ship it ONLY there (no System32 copy), e.g. +/// `...\FileRepository\nv_dispi.inf_amd64_\nvidia-smi.exe`. Scans the +/// `nv*`-prefixed package dirs and returns the most recently modified hit, so +/// after a driver update the current package wins (#247). +#[cfg(any(windows, test))] +fn find_driver_store_nvidia_smi(file_repository: &Path) -> Option { + let entries = fs::read_dir(file_repository).ok()?; + let mut best: Option<(std::time::SystemTime, PathBuf)> = None; + for entry in entries.flatten() { + let name = entry.file_name(); + let name = name.to_string_lossy().to_ascii_lowercase(); + if !name.starts_with("nv") { + continue; + } + let candidate = entry.path().join("nvidia-smi.exe"); + if !candidate.is_file() { + continue; + } + let modified = candidate + .metadata() + .and_then(|m| m.modified()) + .unwrap_or(std::time::SystemTime::UNIX_EPOCH); + if best.as_ref().is_none_or(|(t, _)| modified > *t) { + best = Some((modified, candidate)); + } + } + best.map(|(_, path)| path) +} + #[cfg(not(target_os = "macos"))] -fn nvidia_smi_exe() -> &'static str { +fn nvidia_smi_exe() -> String { // nvidia-smi.exe lives in System32 on Windows but Tauri child processes - // inherit a stripped PATH that may not include it. + // inherit a stripped PATH that may not include it. Some DCH driver installs + // only place it in the DriverStore, so scan there before falling back to + // PATH (#247). #[cfg(windows)] { const SYSTEM32: &str = r"C:\Windows\System32\nvidia-smi.exe"; if std::path::Path::new(SYSTEM32).is_file() { - return SYSTEM32; + return SYSTEM32.to_string(); + } + let file_repository = + std::path::Path::new(r"C:\Windows\System32\DriverStore\FileRepository"); + if let Some(found) = find_driver_store_nvidia_smi(file_repository) { + return found.display().to_string(); } } - "nvidia-smi" + "nvidia-smi".to_string() } #[cfg(not(target_os = "macos"))] -fn detect_nvidia_gpu() -> Option<(String, String, Option)> { +fn detect_nvidia_gpu(data_dir: &Path) -> Option<(String, String, Option)> { let smi = nvidia_smi_exe(); - let mut cmd = Command::new(smi); + append_to_setup_log(data_dir, &format!("GPU detection using: {smi}")); + let mut cmd = Command::new(&smi); cmd.args(["--query-gpu=name", "--format=csv,noheader"]) .stdout(Stdio::piped()) .stderr(Stdio::null()); hide_console_window(&mut cmd); - let name_out = command_output_with_timeout(cmd, Duration::from_secs(10), "nvidia-smi").ok()?; + // 30s (vs 10s elsewhere): the first nvidia-smi call can be slow on Optimus + // laptops that have to wake a sleeping dGPU (#247). + let name_out = match command_output_with_timeout(cmd, Duration::from_secs(30), "nvidia-smi") { + Ok(out) => out, + Err(e) => { + append_to_setup_log(data_dir, &format!("nvidia-smi failed to run: {e}")); + return None; + } + }; if !name_out.status.success() { + append_to_setup_log( + data_dir, + &format!( + "nvidia-smi exited with {}; treating as no GPU", + name_out.status + ), + ); return None; } let gpu_name = String::from_utf8_lossy(&name_out.stdout).trim().to_string(); if gpu_name.is_empty() { + append_to_setup_log(data_dir, "nvidia-smi reported no GPU name"); return None; } // Read CUDA version from the standard nvidia-smi header. - let mut smi_cmd = Command::new(smi); + let mut smi_cmd = Command::new(&smi); smi_cmd.stdout(Stdio::piped()).stderr(Stdio::null()); hide_console_window(&mut smi_cmd); let smi_out = @@ -816,7 +933,7 @@ fn detect_nvidia_gpu() -> Option<(String, String, Option)> { // "8.9" for Ada). Drives the wheel choice: stock torch 2.6 cu12x wheels // have no sm_120 kernels, so Blackwell needs a cu128 / torch 2.7 build. // Failure here is non-fatal — we fall back to the CUDA-version heuristic. - let compute_cap = detect_compute_cap(smi); + let compute_cap = detect_compute_cap(&smi); Some((gpu_name, cuda_version, compute_cap)) } @@ -1416,10 +1533,13 @@ fn migrate_legacy_data(root: &Path, data_dir: &Path) { let _ = fs::rename(&src, data_dir.join(name)); } } - for name in ["config.json", "cpu-only"] { - let src = old.join(name); + // NOTE: deliberately does NOT migrate `cpu-only` -- the marker is only + // trusted in the app root (see is_cpu_only_package); carrying it into the + // shared data dir poisoned later NVIDIA installs (#247). + { + let src = old.join("config.json"); if src.exists() { - let _ = fs::copy(&src, data_dir.join(name)); + let _ = fs::copy(&src, data_dir.join("config.json")); } } } @@ -2751,4 +2871,86 @@ b6052160df96b31c9b1e33854a4dcda3d4b57641b880270f31736fb9f445d384 ffmpeg-n7.1-la assert_eq!(super::wheel_tag(None, "12.1"), "cu121"); assert_eq!(super::wheel_tag(Some("N/A"), "12.4"), "cu124"); } + + // --- cpu-only marker precedence + self-heal (#247) --- + + #[test] + fn cpu_only_marker_trusted_in_root_only() { + let root = make_tmp(); + let data = make_tmp(); + // Marker only in the data dir (a previous CPU-build install) must NOT + // mark this package CPU-only. + fs::write(data.path().join("cpu-only"), "").unwrap(); + assert!(!super::is_cpu_only_package(root.path())); + // Marker in the app root (ships with the package) does. + fs::write(root.path().join("cpu-only"), "").unwrap(); + assert!(super::is_cpu_only_package(root.path())); + } + + #[test] + fn stale_data_dir_marker_is_removed_for_gpu_builds() { + let root = make_tmp(); + let data = make_tmp(); + let stale = data.path().join("cpu-only"); + fs::write(&stale, "").unwrap(); + // GPU build (no root marker): the stale data-dir marker is deleted. + super::clear_stale_cpu_marker(root.path(), data.path()); + assert!(!stale.exists()); + // And the cleanup is recorded in setup.log. + let log = fs::read_to_string(data.path().join("logs").join("setup.log")).unwrap(); + assert!(log.contains("stale cpu-only marker")); + } + + #[test] + fn cpu_build_keeps_its_data_dir_marker() { + let root = make_tmp(); + let data = make_tmp(); + fs::write(root.path().join("cpu-only"), "").unwrap(); + let legacy = data.path().join("cpu-only"); + fs::write(&legacy, "").unwrap(); + // CPU build (root marker present): nothing to heal, no churn. + super::clear_stale_cpu_marker(root.path(), data.path()); + assert!(legacy.exists()); + assert!(!data.path().join("logs").join("setup.log").exists()); + } + + // --- nvidia-smi DriverStore discovery (#247) --- + + #[test] + fn driver_store_scan_finds_newest_nv_package() { + let repo = make_tmp(); + // Non-NVIDIA package dirs are ignored. + fs::create_dir_all(repo.path().join("intelgpu.inf_amd64_aaa")).unwrap(); + fs::write( + repo.path() + .join("intelgpu.inf_amd64_aaa") + .join("nvidia-smi.exe"), + b"x", + ) + .unwrap(); + // Older NVIDIA package. + let old_pkg = repo.path().join("nv_dispi.inf_amd64_old"); + fs::create_dir_all(&old_pkg).unwrap(); + fs::write(old_pkg.join("nvidia-smi.exe"), b"x").unwrap(); + // Newer NVIDIA package (later mtime via explicit set). + let new_pkg = repo.path().join("nvmdi.inf_amd64_new"); + fs::create_dir_all(&new_pkg).unwrap(); + let new_exe = new_pkg.join("nvidia-smi.exe"); + fs::write(&new_exe, b"x").unwrap(); + let later = std::time::SystemTime::now() + std::time::Duration::from_secs(60); + let f = fs::OpenOptions::new().write(true).open(&new_exe).unwrap(); + f.set_modified(later).unwrap(); + + let found = super::find_driver_store_nvidia_smi(repo.path()); + assert_eq!(found, Some(new_exe)); + } + + #[test] + fn driver_store_scan_handles_missing_dir() { + let repo = make_tmp(); + let missing = repo.path().join("does-not-exist"); + assert_eq!(super::find_driver_store_nvidia_smi(&missing), None); + // Present but empty: also None. + assert_eq!(super::find_driver_store_nvidia_smi(repo.path()), None); + } } diff --git a/desktop/ui/setup.js b/desktop/ui/setup.js index 8e13e975..a60bf3f0 100644 --- a/desktop/ui/setup.js +++ b/desktop/ui/setup.js @@ -237,7 +237,18 @@ async function runSetup() { // refresh the install records the version and subsequent launches match. const versionMismatch = Boolean(expectedVersion) && installedVersion !== expectedVersion; - if (runtime.pythonReady && runtime.ffmpegReady && runtime.torchDevice && !versionMismatch) { + // A persisted torch device only counts as settled when it is a positive + // result ("cuda"/"mps") or the package itself is CPU-only. A CPU device + // born from a failure (no GPU found, CUDA verify failed) -- or from a build + // that predates reason tracking -- re-runs the GPU step on this launch, so + // a single bad first run can't pin the install to CPU forever (#247). + // Cost when nothing changed: one fast nvidia-smi probe. + const torchDeviceSettled = + runtime.torchDevice === "cuda" || + runtime.torchDevice === "mps" || + (runtime.torchDevice === "cpu" && runtime.torchDeviceReason === "cpu-only-package"); + + if (runtime.pythonReady && runtime.ffmpegReady && torchDeviceSettled && !versionMismatch) { for (const step of steps) { step.classList.remove("active", "error"); if (step.dataset.step === "backend") { diff --git a/scripts/windows/make-portable.ps1 b/scripts/windows/make-portable.ps1 index 9e7f0f0b..54eaf2f0 100644 --- a/scripts/windows/make-portable.ps1 +++ b/scripts/windows/make-portable.ps1 @@ -169,8 +169,10 @@ foreach ($Dir in @("cache", "downloads", "ffmpeg", "jobs", "logs", "models")) { New-Item -ItemType Directory -Force (Join-Path $Stage "data\$Dir") | Out-Null } if ($CpuOnly) { + # Root marker only: the app trusts cpu-only solely in the app root (#247). + # A data\cpu-only copy used to leak into the shared per-user data dir and + # silently forced later NVIDIA installs onto CPU. New-Item -ItemType File -Force (Join-Path $Stage "cpu-only") | Out-Null - New-Item -ItemType File -Force (Join-Path $Stage "data\cpu-only") | Out-Null } Copy-Tree (Join-Path $Root "app") (Join-Path $BackendDir "app") From 4f0b07c85a6fff5ad5dcdc19ebe09fdf395a2f74 Mon Sep 17 00:00:00 2001 From: Thales Pereira <31625914+thcp@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:26:37 +0100 Subject: [PATCH 2/4] feat(settings): compute device selector for the self-hosted server Companion to the desktop #247 fix, for the server/Docker/Unraid path: device selection was a frozen constant (DEMUCS_DEVICE, computed once at import), so the only override was the STEMDECK_DEMUCS_DEVICE env var plus a restart -- invisible to Docker/Unraid users without container access. - app/core/settings.py: demucs_device setting (auto | cuda | mps | cpu, default auto = hardware probe). Forcing cuda/mps verifies availability BEFORE persisting and rejects with a clear error otherwise -- never persist a device that would silently fall back later (the #247 lesson applied here). STEMDECK_DEMUCS_DEVICE seeds the default so existing env-based deployments keep their forced device. - app/core/config.py: _detect_device -> detect_torch_device (pure hardware probe; env handling moved to the settings seed); DEMUCS_DEVICE constant removed. - app/pipeline/separate.py: reads the device fresh per job -- a Settings change applies to the next separation, no restart. - app/main.py: /api/settings gains demucs_device (choice) and demucs_device_resolved (what jobs will run on); POST validates via the setter (422 with the reason). Startup log and /api/health read live. - static/js/catalog.js: "Compute device" select in Settings -> Advanced, showing the resolved device; a rejected force surfaces the server's reason via showError and reverts the select. Also aligns the port-input fallback with the 8000 default from the earlier port unification. - .docs/improvements/self-hosted-compute-device-setting.md: design doc. 5 new tests: auto-resolution, env seeding, verify-before-persist rejection, unknown-choice rejection, and the API round trip incl. 422 paths. --- app/core/config.py | 19 ++++++----- app/core/settings.py | 48 ++++++++++++++++++++++++++- app/main.py | 21 ++++++++++-- app/pipeline/separate.py | 9 ++++-- static/js/catalog.js | 60 ++++++++++++++++++++++++++++++++-- tests/test_network_gate.py | 66 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 205 insertions(+), 18 deletions(-) diff --git a/app/core/config.py b/app/core/config.py index 7045309f..08186702 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -17,15 +17,15 @@ def _env_path(name: str, default: Path) -> Path: return Path(raw).expanduser().resolve() if raw else default -def _detect_device() -> str: - """Pick best available Torch device for Demucs. Override via - STEMDECK_DEMUCS_DEVICE env var ('cuda' | 'mps' | 'cpu'). Apple Silicon - silently falls back to CPU otherwise -- demucs's CLI default is - "cuda if available else cpu" and macOS has no CUDA, leaving the - integrated GPU idle and processing 3-5x slower than necessary.""" - forced = os.environ.get("STEMDECK_DEMUCS_DEVICE", "").strip().lower() - if forced in ("cuda", "mps", "cpu"): - return forced +def detect_torch_device() -> str: + """Best available Torch device for Demucs by hardware probe: cuda > mps > + cpu. Apple Silicon needs the explicit MPS check -- demucs's CLI default is + "cuda if available else cpu" and macOS has no CUDA, leaving the integrated + GPU idle and processing 3-5x slower than necessary. + + User-facing device selection lives in app.core.settings (demucs_device, + default "auto" -> this probe); the STEMDECK_DEMUCS_DEVICE env var seeds + that setting's default so env-based deployments keep working.""" try: import torch @@ -67,7 +67,6 @@ def _detect_device() -> str: FFMPEG_DIR / ("ffprobe.exe" if sys.platform.startswith("win") else "ffprobe"), ) DEMUCS_MODEL = os.environ.get("STEMDECK_DEMUCS_MODEL", "htdemucs_6s").strip() or "htdemucs_6s" -DEMUCS_DEVICE = _detect_device() MAX_DURATION_SEC = max(60, _env_int("STEMDECK_MAX_DURATION_SEC", 1200)) # 20 min default JOB_TTL_SECONDS = max(300, _env_int("STEMDECK_JOB_TTL_SECONDS", 24 * 3600)) # 24 h default MAX_PENDING_JOBS = max(1, min(50, _env_int("STEMDECK_MAX_PENDING_JOBS", 3))) diff --git a/app/core/settings.py b/app/core/settings.py index 1b7e9f7e..a63fcd3c 100644 --- a/app/core/settings.py +++ b/app/core/settings.py @@ -6,6 +6,7 @@ - `allow_network` — whether StemDeck answers requests from other devices. - `max_duration_sec` — longest track accepted for processing. - `video_max_height` — max video resolution for MP4 export / YouTube pulls. +- `demucs_device` — compute device for separation: auto | cuda | mps | cpu. Defaults fall back to the config.py constants (which honor their env vars), so nothing changes until the user overrides a value. @@ -18,7 +19,7 @@ import os import threading -from app.core.config import DATA_DIR, MAX_DURATION_SEC, VIDEO_MAX_HEIGHT +from app.core.config import DATA_DIR, MAX_DURATION_SEC, VIDEO_MAX_HEIGHT, detect_torch_device _log = logging.getLogger("stemdeck.settings") @@ -138,3 +139,48 @@ def set_port(value: int) -> int: _ensure()["port"] = clamped _save() return clamped + + +# ── demucs_device ── +# Compute device for stem separation. "auto" (default) resolves to the best +# available device via a hardware probe at job time; "cuda"/"mps"/"cpu" force +# it. Read live per job (app/pipeline/separate.py), so changes apply to the +# NEXT separation without a restart. STEMDECK_DEMUCS_DEVICE seeds the default +# so existing env-based deployments keep their forced device. +_DEVICE_CHOICES = ("auto", "cuda", "mps", "cpu") + + +def _default_demucs_device() -> str: + env = os.environ.get("STEMDECK_DEMUCS_DEVICE", "").strip().lower() + return env if env in ("cuda", "mps", "cpu") else "auto" + + +def get_demucs_device_choice() -> str: + """The persisted user choice ("auto" | "cuda" | "mps" | "cpu") -- what the + Settings UI displays, as opposed to what jobs run on (see below).""" + with _LOCK: + v = _ensure().get("demucs_device") + return v if isinstance(v, str) and v in _DEVICE_CHOICES else _default_demucs_device() + + +def get_demucs_device() -> str: + """The device the next separation job will actually use: the forced choice, + or a fresh hardware probe when the choice is "auto".""" + choice = get_demucs_device_choice() + return detect_torch_device() if choice == "auto" else choice + + +def set_demucs_device(value: str) -> str: + """Persist a device choice. Forcing "cuda"/"mps" verifies the device is + actually available first and raises ValueError if not -- rejecting the + write loudly beats persisting a device that would silently fall back or + crash the next job (the #247 lesson, applied to the server path).""" + choice = (value or "").strip().lower() + if choice not in _DEVICE_CHOICES: + raise ValueError("demucs_device must be one of: " + ", ".join(_DEVICE_CHOICES)) + if choice in ("cuda", "mps") and detect_torch_device() != choice: + raise ValueError(f"{choice} is not available on this machine") + with _LOCK: + _ensure()["demucs_device"] = choice + _save() + return choice diff --git a/app/main.py b/app/main.py index 483baffa..aed01f90 100644 --- a/app/main.py +++ b/app/main.py @@ -19,7 +19,6 @@ from app.api.router import router from app.core.config import ( - DEMUCS_DEVICE, DEMUCS_MODEL, FFMPEG_BIN, JOBS_DIR, @@ -30,10 +29,13 @@ from app.core.registry import restore as restore_registry from app.core.settings import ( get_allow_network, + get_demucs_device, + get_demucs_device_choice, get_max_duration_sec, get_port, get_video_max_height, set_allow_network, + set_demucs_device, set_max_duration_sec, set_port, set_video_max_height, @@ -45,7 +47,9 @@ # logger.info(...) call across the app, including the analyze # diagnostics ("chroma:", "key candidates:"). logging.getLogger("stemdeck").setLevel(logging.INFO) -logging.getLogger("stemdeck").info("demucs config: model=%s device=%s", DEMUCS_MODEL, DEMUCS_DEVICE) +logging.getLogger("stemdeck").info( + "demucs config: model=%s device=%s", DEMUCS_MODEL, get_demucs_device() +) configure_portable_environment() @@ -211,7 +215,7 @@ def health() -> dict[str, object]: "version": app_version(), "ffmpeg_configured": FFMPEG_BIN.is_file(), "demucs_model": DEMUCS_MODEL, - "demucs_device": DEMUCS_DEVICE, + "demucs_device": get_demucs_device(), } @@ -233,6 +237,10 @@ def _settings_payload() -> dict[str, object]: "max_duration_sec": get_max_duration_sec(), "video_max_height": get_video_max_height(), "port": get_port(), + # The user's choice ("auto" | "cuda" | "mps" | "cpu") drives the UI + # select; the resolved value shows what jobs will actually run on. + "demucs_device": get_demucs_device_choice(), + "demucs_device_resolved": get_demucs_device(), } @@ -266,6 +274,13 @@ async def update_settings(request: Request) -> dict[str, object]: setter(int(body[key])) except (TypeError, ValueError): raise HTTPException(status_code=422, detail=f"{key} must be an integer") from None + if "demucs_device" in body: + try: + set_demucs_device(str(body["demucs_device"])) + except ValueError as e: + # set_demucs_device's messages are safe, user-actionable strings + # (invalid choice / device not available on this machine). + raise HTTPException(status_code=422, detail=str(e)) from None return _settings_payload() diff --git a/app/pipeline/separate.py b/app/pipeline/separate.py index cf26dae2..a54c9371 100644 --- a/app/pipeline/separate.py +++ b/app/pipeline/separate.py @@ -9,9 +9,10 @@ import time from pathlib import Path -from app.core.config import DEMUCS_DEVICE, DEMUCS_MODEL, TIMEOUT_DEMUCS_STALL +from app.core.config import DEMUCS_MODEL, TIMEOUT_DEMUCS_STALL from app.core.models import Job, JobCancelled, _set from app.core.registry import set_proc +from app.core.settings import get_demucs_device logger = logging.getLogger("stemdeck.pipeline") @@ -24,6 +25,10 @@ def separate(job: Job, source: Path, job_dir: Path) -> Path: _set(job, status="separating", progress=0.0, stage="Separating stems...") + # Read the device fresh per job (not a frozen import) so a Settings change + # applies to the next separation without a restart. + device = get_demucs_device() + logger.info("[%s] separating on device=%s", job.id, device) cmd = [ sys.executable, "-m", @@ -31,7 +36,7 @@ def separate(job: Job, source: Path, job_dir: Path) -> Path: "-n", DEMUCS_MODEL, "-d", - DEMUCS_DEVICE, + device, "-o", str(job_dir), str(source), diff --git a/static/js/catalog.js b/static/js/catalog.js index f28d229b..8db883b7 100644 --- a/static/js/catalog.js +++ b/static/js/catalog.js @@ -1853,12 +1853,27 @@ async function wireGeneralSettings(overlay) { const durInput = overlay.querySelector(".set-max-duration"); const heightSel = overlay.querySelector(".set-video-height"); const portInput = overlay.querySelector(".set-port"); - if (!durInput && !heightSel && !portInput) return; + const deviceSel = overlay.querySelector(".set-demucs-device"); + const deviceResolved = overlay.querySelector(".set-demucs-resolved"); + if (!durInput && !heightSel && !portInput && !deviceSel) return; + + // Last server-confirmed device choice, to revert the select when the server + // rejects a forced device (e.g. CUDA not available on this machine). + let lastDevice = "auto"; const apply = (d) => { if (durInput && d.max_duration_sec) durInput.value = String(Math.round(d.max_duration_sec / 60)); if (heightSel && d.video_max_height) heightSel.value = String(d.video_max_height); if (portInput && d.port) portInput.value = String(d.port); + if (deviceSel && d.demucs_device) { + deviceSel.value = d.demucs_device; + lastDevice = d.demucs_device; + } + if (deviceResolved) { + deviceResolved.textContent = d.demucs_device_resolved + ? ` (currently: ${d.demucs_device_resolved})` + : ""; + } }; // Keep the text inputs digit-only as the user types (maxlength caps the rest). @@ -1893,9 +1908,36 @@ async function wireGeneralSettings(overlay) { post({ video_max_height: parseInt(heightSel.value, 10) }); }); portInput?.addEventListener("change", () => { - const port = Math.max(1024, Math.min(65535, parseInt(portInput.value, 10) || 8080)); + const port = Math.max(1024, Math.min(65535, parseInt(portInput.value, 10) || 8000)); post({ port }); }); + // Compute device needs its own POST path: unlike the clamped numeric + // settings, the server can REJECT a forced device (422 with a reason, e.g. + // "cuda is not available on this machine") -- surface that and revert. + deviceSel?.addEventListener("change", async () => { + try { + const r = await fetch("/api/settings", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ demucs_device: deviceSel.value }), + }); + if (r.ok) { + apply(await r.json()); + return; + } + let detail = "Could not change the compute device."; + try { + detail = (await r.json()).detail || detail; + } catch (err) { + console.warn("settings error body parse failed:", err); + } + showError(detail); + deviceSel.value = lastDevice; + } catch (err) { + console.warn("compute device update failed:", err); + deviceSel.value = lastDevice; + } + }); } async function wireNetworkSetting(overlay) { @@ -2033,6 +2075,20 @@ function openLibraryEditor() { +
+
+
+
Compute device
+
Device used for stem separation. Applies to the next track.
+
+ +
+
Out of sync tracks
diff --git a/tests/test_network_gate.py b/tests/test_network_gate.py index 033745f2..86a5f857 100644 --- a/tests/test_network_gate.py +++ b/tests/test_network_gate.py @@ -84,6 +84,72 @@ def test_settings_reject_non_integer(): assert c.post("/api/settings", json={"max_duration_sec": "abc"}).status_code == 422 +# ── demucs_device (compute device) ── + + +@pytest.fixture() +def _isolated_settings(monkeypatch, tmp_path): + """Point the settings store at a temp file with a fresh in-memory state, so + device tests neither read nor pollute the developer's real settings.json.""" + monkeypatch.setattr(settings_mod, "_SETTINGS_PATH", tmp_path / "settings.json") + monkeypatch.setattr(settings_mod, "_state", None) + monkeypatch.delenv("STEMDECK_DEMUCS_DEVICE", raising=False) + + +def test_demucs_device_defaults_to_auto_and_resolves(monkeypatch, _isolated_settings): + monkeypatch.setattr(settings_mod, "detect_torch_device", lambda: "cpu") + assert settings_mod.get_demucs_device_choice() == "auto" + assert settings_mod.get_demucs_device() == "cpu" # auto -> hardware probe + # A different probe result flows through without any persisted change. + monkeypatch.setattr(settings_mod, "detect_torch_device", lambda: "cuda") + assert settings_mod.get_demucs_device() == "cuda" + + +def test_demucs_device_env_seeds_default(monkeypatch, _isolated_settings): + # Existing env-based deployments keep their forced device as the default. + monkeypatch.setenv("STEMDECK_DEMUCS_DEVICE", "cuda") + assert settings_mod.get_demucs_device_choice() == "cuda" + assert settings_mod.get_demucs_device() == "cuda" # forced, no probe + + +def test_demucs_device_force_verified_before_persist(monkeypatch, _isolated_settings): + # Forcing a device that isn't available must be rejected loudly, not + # persisted to silently fail later (#247 lesson applied to the server). + monkeypatch.setattr(settings_mod, "detect_torch_device", lambda: "cpu") + with pytest.raises(ValueError): + settings_mod.set_demucs_device("cuda") + assert settings_mod.get_demucs_device_choice() == "auto" # nothing persisted + # Forcing CPU is always allowed; forcing an available GPU is allowed. + assert settings_mod.set_demucs_device("cpu") == "cpu" + monkeypatch.setattr(settings_mod, "detect_torch_device", lambda: "cuda") + assert settings_mod.set_demucs_device("cuda") == "cuda" + assert settings_mod.get_demucs_device() == "cuda" + + +def test_demucs_device_rejects_unknown_choice(_isolated_settings): + with pytest.raises(ValueError): + settings_mod.set_demucs_device("bogus") + + +def test_demucs_device_api_round_trip_and_422(monkeypatch, _isolated_settings): + monkeypatch.setattr(settings_mod, "detect_torch_device", lambda: "cpu") + with TestClient(app) as c: + body = c.get("/api/settings").json() + assert body["demucs_device"] == "auto" + assert body["demucs_device_resolved"] == "cpu" + # Valid change round-trips. + r = c.post("/api/settings", json={"demucs_device": "cpu"}) + assert r.status_code == 200 + assert r.json()["demucs_device"] == "cpu" + # Unavailable device -> 422 with a user-actionable detail, not persisted. + r = c.post("/api/settings", json={"demucs_device": "cuda"}) + assert r.status_code == 422 + assert "not available" in r.json()["detail"] + assert c.get("/api/settings").json()["demucs_device"] == "cpu" + # Unknown value -> 422. + assert c.post("/api/settings", json={"demucs_device": "bogus"}).status_code == 422 + + def test_gate_blocks_non_loopback_when_off(): settings_mod.set_allow_network(False) # TestClient's client host ("testclient") is treated as non-loopback. From 22206b3c9e9f04f780d0b2acaf80b29b3cd44e71 Mon Sep 17 00:00:00 2001 From: Thales Pereira <31625914+thcp@users.noreply.github.com> Date: Tue, 14 Jul 2026 18:26:01 +0100 Subject: [PATCH 3/4] feat(settings): gray out compute devices this machine can't use MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Compute device dropdown now disables options that aren't available or detected (Auto and CPU are always selectable; CUDA/MPS depend on the hardware + torch build), labeling them "— not available" so it's clear why. - config.py: available_torch_devices() returns the usable devices best-first; detect_torch_device() is now its first element (no duplicated torch probe). - settings.py: set_demucs_device verifies against membership in available_torch_devices() rather than only the top pick. - /api/settings: new demucs_devices_available list for the UI. - catalog.js: disable + relabel unavailable