Fix reporting flow, stem-collection availability, and YouTube import gaps

app/api/jobs.py

@@ -108,6 +108,30 @@ def _rmtree_job(job_id: str) -> None:
         logger.warning("failed to remove job dir %s", job_dir, exc_info=True)
 
 
+def _job_files_missing(job: Job) -> bool:
+    """True when a "done" job's stem files are gone from disk: the folder was
+    deleted or moved outside the app, not just an in-flight relocation (#354),
+    which is a known, temporary absence and must not flap the library."""
+    if is_relocating():
+        return False
+    stems_dir = (JOBS_DIR / job.id / "stems").resolve()
+    if not stems_dir.is_relative_to(JOBS_DIR.resolve()):
+        return True
+    return not stems_dir.is_dir() or not any(stems_dir.iterdir())
+
+
+def _job_state(job: Job) -> dict:
+    """job.to_state() with "done" downgraded to "unavailable" when the stem
+    files are missing from disk - ground truth for the client, replacing the
+    old approach of the frontend guessing from a 404 or a disappearance from
+    the job list, neither of which caught a job whose registry entry survived
+    but whose stems folder did not."""
+    state = job.to_state()
+    if job.status == "done" and _job_files_missing(job):
+        state["status"] = "unavailable"
+    return state
+
+
 class JobRequest(BaseModel):
     url: str
     # Subset of stems to include in the post-processing "selected mix"
@@ -259,7 +283,7 @@ async def _create_local_job(request: Request) -> dict[str, str]:
 def list_jobs() -> list[dict]:
     """List all completed jobs in the library, sorted by creation time."""
     return [
-        job.to_state()
+        _job_state(job)
         for job in sorted(registry_all_jobs().values(), key=lambda j: j.created_at)
         if job.status == "done"
     ]
@@ -271,7 +295,7 @@ def get_job(job_id: str) -> dict:
     job = registry_get(job_id)
     if job is None:
         raise HTTPException(status_code=404, detail="job not found")
-    return job.to_state()
+    return _job_state(job)
 
 
 @router.post("/{job_id}/cancel")
@@ -440,9 +464,10 @@ def get_failure(job_id: str) -> dict:
 
     _quarantine_failed_job writes jobs/failed/<id>/error.txt on every pipeline
     failure (#277) and until now nothing ever read it back: the UI had only the
-    one-line `error_detail`, so a bug report could not carry the stderr tail
-    that says *why* demucs died. Read-only, and never serves the whole file --
-    only the technical keys above, plus the tail.
+    one-line `error_detail`, so a bug report could not carry the stderr tail or
+    the full traceback that say *why* demucs died. Read-only, and never serves
+    the whole file -- only the technical keys above, plus the tail and
+    traceback (both already home-directory-redacted by the writer).
     """
     if not JOB_ID_RE.match(job_id):
         raise HTTPException(status_code=404, detail="job not found")
@@ -464,19 +489,32 @@ def get_failure(job_id: str) -> dict:
 
     fields: dict[str, str] = {}
     tail: list[str] = []
-    in_tail = False
+    tb: list[str] = []
+    section = "fields"
     for line in text.splitlines():
-        if line.strip() == "--- stderr tail ---":
-            in_tail = True
+        stripped = line.strip()
+        if stripped == "--- stderr tail ---":
+            section = "tail"
             continue
-        if in_tail:
+        if stripped == "--- traceback ---":
+            # The writer separates sections with a blank line for readability
+            # in the raw file; drop it here rather than let it show up as a
+            # trailing empty entry in `tail`.
+            if tail and tail[-1] == "":
+                tail.pop()
+            section = "traceback"
+            continue
+        if section == "tail":
             tail.append(line)
             continue
+        if section == "traceback":
+            tb.append(line)
+            continue
         key, sep, value = line.partition(":")
         if sep and key in _FAILURE_PUBLIC_KEYS:
             fields[key] = value.strip()
 
-    return {"job_id": job_id, **fields, "tail": tail}
+    return {"job_id": job_id, **fields, "tail": tail, "traceback": tb}
 
 
 @router.get("/{job_id}/beats")

app/core/redact.py

@@ -0,0 +1,58 @@
+"""Stripping locally- or personally-identifying text before it can reach a
+public report.
+
+The notification centre's "Report on GitHub"/"Report on Discord" flow and its
+opt-in "include recent logs" button both end with this text pasted somewhere
+public. Three things in particular have no business being there:
+
+  - The reporter's home directory. On Windows the Python install path alone
+    (`C:\\Users\\<name>\\...`) carries the OS username.
+  - A track's source URL. `download.py` logs it for every job
+    ("download starting: <url>"), so a raw log tail carries the YouTube/
+    SoundCloud link for everything the reporter has ever imported in the
+    fetched window -- not just the one that failed. The per-job failure
+    report has always kept this out deliberately (see
+    app/api/jobs.py's _FAILURE_PUBLIC_KEYS); a raw log has no such allowlist
+    of its own, so it is enforced here instead.
+  - An IP address. The mobile UI talks to this backend over the LAN when
+    network access is on, so uvicorn's access log (captured into
+    backend.log on desktop) can carry another device's address on the
+    reporter's home network, not just their own.
+
+Every place that can hand text to that report flow -- the per-job failure
+evidence (runner.py) and the Settings -> Logs viewer (main.py) -- redacts
+through this one function rather than each carrying its own copy.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+# Mirrors app/pipeline/download.py's _YOUTUBE_HOSTS | _SOUNDCLOUD_HOSTS. Kept
+# as a literal copy rather than importing it: this module is used from
+# main.py's request path, and pipeline.download pulls in yt_dlp, which is not
+# something a log-tail redaction pass should need to import.
+_SOURCE_URL_RE = re.compile(
+    r"https?://(?:[\w-]+\.)*"
+    r"(?:youtube\.com|youtu\.be|youtube-nocookie\.com|soundcloud\.com)\S*",
+    re.IGNORECASE,
+)
+
+# IPv4 only. The app's own LAN detection (app/main.py's _is_lan_ipv4) is
+# IPv4-only too, and a loose IPv6 pattern is a real hazard here: something
+# like "(?:[0-9a-f]{1,4}:){2,7}[0-9a-f]{1,4}" also matches an HH:MM:SS
+# timestamp (every log line has one), which would mangle every line's time
+# instead of catching the rare home IPv6 address.
+_IPV4_RE = re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b")
+
+
+def redact(text: str) -> str:
+    """Strip the reporter's home directory, any YouTube/SoundCloud source
+    URL, and any IPv4 address from `text`."""
+    home = str(Path.home())
+    if home and home not in (".", "/"):
+        text = text.replace(home, "<home>")
+    text = _SOURCE_URL_RE.sub("<source-url-redacted>", text)
+    text = _IPV4_RE.sub("<ip>", text)
+    return text

app/core/settings.py

@@ -39,8 +39,8 @@
 _LOCK = threading.RLock()
 _state: dict | None = None  # whole settings dict, loaded lazily
 
-# Clamp bounds. Max track length is capped at 20 min (the product ceiling).
-_DURATION_MIN, _DURATION_MAX = 60, 1200  # 1 min .. 20 min
+# Clamp bounds. Max track length is capped at 60 min (the product ceiling).
+_DURATION_MIN, _DURATION_MAX = 60, 3600  # 1 min .. 60 min
 _HEIGHT_MIN, _HEIGHT_MAX = 144, 2160
 _PLAYLIST_MIN, _PLAYLIST_MAX = 1, 200
 _PORT_MIN, _PORT_MAX = 1024, 65535

app/main.py

@@ -33,6 +33,7 @@
 )
 from app.core.logging_setup import configure_logging
 from app.core.process import process_exists as _process_exists
+from app.core.redact import redact
 from app.core.registry import all_jobs as registry_all_jobs
 from app.core.registry import registry_path, take_pending_resume
 from app.core.registry import reset_all as reset_registry
@@ -666,9 +667,16 @@ def get_log_tail(view: str, minutes: int = 60) -> PlainTextResponse:
     if len(kept) > _LOG_TAIL_LINES:
         truncated = f"[... {len(kept) - _LOG_TAIL_LINES} earlier lines not shown ...]\n"
         kept = kept[-_LOG_TAIL_LINES:]
-    return PlainTextResponse(
-        truncated + "\n".join(kept) + "\n", media_type="text/plain; charset=utf-8"
-    )
+    # Redacted unconditionally, not just for the report flow's callers: a log
+    # line is a log line regardless of who asks for it, and the notification
+    # centre's "include recent logs" button hands this straight to a public
+    # GitHub issue or Discord message without a second filtering step. Strips
+    # the reporter's home directory, any YouTube/SoundCloud source URL (every
+    # job's download start is logged at info level -- not just the failing
+    # one), and any IPv4 address (the mobile UI talks to this backend over
+    # the LAN, so uvicorn's access log can carry another device's address).
+    body = redact(truncated + "\n".join(kept) + "\n")
+    return PlainTextResponse(body, media_type="text/plain; charset=utf-8")
 
 
 @app.get("/api/logs.zip", tags=["settings"])

app/pipeline/download.py

@@ -94,6 +94,10 @@ def _with_retries(job: Job, fn, *, what: str):
         "m.youtube.com",
         "music.youtube.com",
         "youtu.be",
+        # The "privacy-enhanced mode" embed domain -- same site, same extractor,
+        # shows up in copy-pasted embed/share code rather than the address bar.
+        "youtube-nocookie.com",
+        "www.youtube-nocookie.com",
     )
 )
 # Note: on.soundcloud.com (the share shortener) is intentionally excluded — it
@@ -169,12 +173,18 @@ def normalize_youtube_url(url: str) -> str:
     the playlist extractor. Pass non-YouTube URLs through unchanged.
 
     Cases handled:
-      * `watch?v=X&list=...` -> `watch?v=X` (drop the playlist context)
+      * `watch?v=X&list=...` -> `watch?v=X` (drop the playlist context,
+        regardless of what other tracking/context params ride along --
+        `si=`, `t=`, `app=desktop`, etc.)
       * `?list=RD<videoId>&start_radio=1` -> `watch?v=<videoId>` (Radio
         playlists embed the seed in the list ID; YouTube refuses to view the
         playlist directly with "This playlist type is unviewable.")
       * `youtu.be/<videoId>` -> `watch?v=<videoId>`
       * `youtube.com/shorts/<videoId>` -> `watch?v=<videoId>`
+      * `youtube.com/live/<videoId>` -> `watch?v=<videoId>` (premieres and
+        creator livestreams keep this URL once they end and become a normal
+        VOD -- common for concert/DJ-set recordings)
+      * `youtube-nocookie.com/...` -> the same forms on `youtube.com`
     Everything else (PL/OL/algorithmic playlists with no derivable seed) is
     left alone -- yt-dlp will surface its own error.
     """
@@ -187,7 +197,7 @@ def normalize_youtube_url(url: str) -> str:
         if host.startswith(prefix):
             host = host[len(prefix) :]
             break
-    if host not in ("youtube.com", "youtu.be"):
+    if host not in ("youtube.com", "youtu.be", "youtube-nocookie.com"):
         return url
 
     qs = urllib.parse.parse_qs(parsed.query)
@@ -206,10 +216,13 @@ def normalize_youtube_url(url: str) -> str:
         if _VIDEO_ID_RE.match(vid):
             return f"https://www.youtube.com/watch?v={vid}"
 
-    if host == "youtube.com" and parsed.path.startswith("/shorts/"):
-        vid = parsed.path[len("/shorts/") :].lstrip("/").split("/")[0]
-        if _VIDEO_ID_RE.match(vid):
-            return f"https://www.youtube.com/watch?v={vid}"
+    if host in ("youtube.com", "youtube-nocookie.com"):
+        for path_prefix in ("/shorts/", "/live/", "/embed/"):
+            if parsed.path.startswith(path_prefix):
+                vid = parsed.path[len(path_prefix) :].lstrip("/").split("/")[0]
+                if _VIDEO_ID_RE.match(vid):
+                    return f"https://www.youtube.com/watch?v={vid}"
+                break
 
     return url
 

app/pipeline/runner.py

@@ -6,11 +6,13 @@
 import shutil
 import subprocess
 import time
+import traceback
 from datetime import datetime, timezone
 from pathlib import Path
 
 from app.core.config import DEMUCS_MODEL, TIMEOUT_FFMPEG
 from app.core.models import Job, JobCancelled, _set
+from app.core.redact import redact
 from app.core.registry import persist as persist_registry
 from app.pipeline.analyze import analyze
 from app.pipeline.beatgrid import compute_beat_grid
@@ -256,18 +258,30 @@ def _quarantine_failed_job(job: Job, job_dir: Path, jobs_dir: Path, exc: Excepti
     """Preserve failure evidence instead of destroying it (#277).
 
     Writes error.txt (stage, device, model, timings, classified cause, stderr
-    tail), strips the heavy audio payloads, and moves the dir to
-    jobs/failed/<id> where sweep_failed_jobs expires it after FAILED_TTL.
+    tail, full traceback), strips the heavy audio payloads, and moves the dir
+    to jobs/failed/<id> where sweep_failed_jobs expires it after FAILED_TTL.
     Best-effort throughout: any step failing falls back to plain removal so a
     pathological error can never leak disk."""
     tail: list[str] = getattr(exc, "tail", None) or []
     cause = classify_failure("\n".join([*tail, repr(exc)]))
     detail = cause
     if tail:
-        detail += f" — {tail[-1][:200]}"
+        # error_detail reaches the client directly (job state, notification
+        # card, and the report URL's "what" field) -- redact before the [:200]
+        # truncation, not after, so a redaction placeholder never gets cut in
+        # half.
+        detail += f" — {redact(tail[-1])[:200]}"
     job.error_detail = detail
 
     try:
+        # title/source stay unredacted: they never leave this file (the
+        # /failure API's allowlist excludes both, see app/api/jobs.py), so
+        # this is purely local diagnostic value for the person looking at
+        # their own disk. Everything below IS served to the client and is
+        # redacted accordingly -- exc!r can embed a source URL (yt-dlp errors
+        # often do), and the stderr tail/traceback can carry either a source
+        # URL or the reporter's home directory.
+        redacted_tail = [redact(line) for line in tail]
         lines = [
             f"time: {datetime.now(timezone.utc).isoformat(timespec='seconds')}",
             f"job: {job.id}",
@@ -278,10 +292,13 @@ def _quarantine_failed_job(job: Job, job_dir: Path, jobs_dir: Path, exc: Excepti
             f"model: {DEMUCS_MODEL}",
             f"cause: {cause}",
             f"timings: {json.dumps(job.stage_timings) if job.stage_timings else '(none)'}",
-            f"exception: {exc!r}",
+            f"exception: {redact(repr(exc))}",
         ]
-        if tail:
-            lines += ["", "--- stderr tail ---", *tail]
+        if redacted_tail:
+            lines += ["", "--- stderr tail ---", *redacted_tail]
+        tb = redact("".join(traceback.format_exception(type(exc), exc, exc.__traceback__))).rstrip()
+        if tb:
+            lines += ["", "--- traceback ---", tb]
         (job_dir / "error.txt").write_text("\n".join(lines) + "\n", encoding="utf-8")
 
         # Strip heavy payloads: the quarantine keeps diagnostics, not audio.

static/css/daw.css

@@ -670,8 +670,9 @@ input, textarea { font-family: inherit; }
 .cat-status { width: 6px; height: 6px; border-radius: 50%; background: #5fbc56; flex-shrink: 0; }
 .cat-status.processing { background: var(--accent); animation: cat-pulse 1.4s infinite; }
 .cat-status.unavailable { background: #666; }
-.cat-item.unavailable { cursor: not-allowed; }
-.cat-item.unavailable .cat-meta { opacity: 0.45; }
+.cat-item.unavailable { cursor: pointer; }
+.cat-item.unavailable .cat-title { opacity: 0.55; }
+.cat-unavailable-warning { color: var(--accent); font-weight: 600; opacity: 1; }
 @keyframes cat-pulse { 0%,100%{opacity:1;} 50%{opacity:0.3;} }
 
 /* Import queue: a waiting row and a running row must be told apart at a
@@ -2791,6 +2792,18 @@ input, textarea { font-family: inherit; }
   font-family: var(--font-mono); font-size: 11px; line-height: 1.55;
   color: var(--fg-2); white-space: pre-wrap; overflow-wrap: anywhere;
 }
+/* Opt-in, not automatic: this pulls extra log content into what's about to be
+   copied into a public report, so it gets its own deliberate click rather
+   than firing every time the dialog opens (matches .library-editor-sync's
+   amber "extra action" treatment). */
+.failure-logs-btn {
+  align-self: flex-start;
+  min-height: 28px; margin-top: 8px; padding: 0 12px;
+  border-radius: 7px; border: 1px solid rgba(244, 183, 64, 0.3);
+  background: rgba(244, 183, 64, 0.16); color: var(--accent);
+  font-family: var(--font-mono); font-size: 11px; cursor: pointer;
+}
+.failure-logs-btn:disabled { opacity: 0.55; cursor: default; }
 /* Reporting is the reason this dialog exists, so it gets the app's primary
    button treatment (as Export Mix does) rather than the muted link style the
    About/release dialogs use for their several equal-weight links. */

static/index.html

@@ -886,10 +886,12 @@ <h2 id="releaseTitle">New release available</h2>
         <h2 id="failureTitle">Something failed</h2>
         <span class="about-version-badge failure-when" id="failureWhen"></span>
         <p class="failure-message" id="failureMessage"></p>
-        <p class="failure-hint">These details go in the report. Your track title and source link are not included — add them yourself if they help.</p>
+        <p class="failure-hint">These details go in the report. Your track title and source link are not included - add them yourself if they help. Clicking a button below copies this to your clipboard.</p>
         <pre class="failure-tech"><code id="failureTech"></code></pre>
+        <button class="failure-logs-btn" id="failureIncludeLogs" type="button">Include recent logs</button>
         <div class="about-primary-links">
           <a class="about-link about-link-primary" id="failureReport" target="_blank" rel="noopener noreferrer">Report on GitHub</a>
+          <a class="about-link about-link-secondary" id="failureReportDiscord" target="_blank" rel="noopener noreferrer">Report on Discord</a>
         </div>
       </div>
     </div>