From fbbf9e5c3a1af03a6d10e76b3e320e5986187861 Mon Sep 17 00:00:00 2001 From: Jaasthi Santosh Naidu Lakshman Senna Date: Thu, 10 Sep 2026 02:21:23 -0700 Subject: [PATCH] Fix playlist import: pin YouTubeKit 0.4.9 (ANDROID_VR stream URLs 403) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every imported track resolved, then every ranged download got HTTP 403 → IngestError.streamURLExpired → re-resolve → 403 again → scheduleRetry kept the row .pending through 5 whole-track attempts (minutes of spinner) → .failed (orange retry badge). Reproduced on the iOS Simulator with a live probe: 0/8 tracks ready, every attempt "prep failed … streamURLExpired". Root cause: Packages/ContinuityKit/Package.resolved pinned YouTubeKit at 7cc8190 (2026-07-05), whose stream URLs come from the ANDROID_VR InnerTube client. Since mid-August 2026 YouTube serves only the first ~1 MB of those URLs and 403s the rest, so the app's 1 MiB ranged downloader dies on chunk #2 every time, with every User-Agent. Upstream YouTubeKit 0.4.9 ("Update YouTube Changes (August 2026)") switches to the visionOS/web clients plus an embed fallback; its URLs return 206 for every chunk. - Package.swift: depend on YouTubeKit `exact: "0.4.9"` instead of floating on `branch: "main"` (which resolved differently per machine and let the app silently fall behind). - Package.resolved: 0.4.9 / e5b7d03. - Tests/IngestTests/LiveIngestProbeTests.swift: opt-in live-network probe (skipped unless CONTINUITY_LIVE_PROBE=1) that drives the real PreparationQueue and prints the failing stage/error per track — the tool that found this. With 0.4.9: 8/8 and 6/6 tracks ready on the simulator. - AGENTS.md / CLAUDE.md: record the gotcha and the diagnosis recipe. Co-Authored-By: Claude Fable 5.1 --- AGENTS.md | 11 ++ CLAUDE.md | 10 ++ Packages/ContinuityKit/Package.resolved | 6 +- Packages/ContinuityKit/Package.swift | 2 +- .../IngestTests/LiveIngestProbeTests.swift | 116 ++++++++++++++++++ 5 files changed, 141 insertions(+), 4 deletions(-) create mode 100644 Packages/ContinuityKit/Tests/IngestTests/LiveIngestProbeTests.swift diff --git a/AGENTS.md b/AGENTS.md index 11216ee..f196336 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -104,6 +104,17 @@ or commit it. Bundle id `com.sanylax.continuity` (share extension - **Stem separation on the Simulator** is also CPU-only (and slow). Do not "fix" perceived hangs by enabling CoreML on sim — sim CoreML has no ANE/GPU and routes through a ~100× slower serial CPU queue. +- **YouTubeKit is pinned `exact:` on purpose — bump it deliberately, never float on `main`.** + Stream URLs come from whichever InnerTube client the pinned YouTubeKit asks; YouTube retires + clients without notice (Aug 2026: ANDROID_VR URLs started 403-ing after the first ~1 MB, so + every 1 MiB ranged download died on chunk #2 → `streamURLExpired` on every track → minutes of + spinner, then the orange retry badge). The signature of "YouTube changed again" is *every* + imported track failing with `prep failed for …: streamURLExpired` (or `.network`) in Console; + playlist/search scrapes still succeed. First response: check upstream YouTubeKit for a newer + tag, bump `exact:` in `Packages/ContinuityKit/Package.swift`, and re-run the opt-in probe + `CONTINUITY_LIVE_PROBE=1 … -only-testing:IngestTests/LiveIngestProbeTests` (simulator) — + it prints the failing stage and exact error per track. Beware: a scratch SwiftPM harness + that depends on `branch: "main"` silently resolves upstream HEAD, not the app's pin. - **Scrapers are fragile by design.** YouTube/Spotify change their embedded JSON shapes without notice (YouTube moved playlists to `lockupViewModel` mid-project). Parsers handle multiple shapes and are pinned by tests against real fixtures. Resolvers retry transient failures diff --git a/CLAUDE.md b/CLAUDE.md index 71eb7a5..29f7fe1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -68,6 +68,16 @@ anchor at `mach_absolute_time()+0.03s`, else fall back to plain `play()` calls. - `Player.prepare`/`restore` stay **metadata-only** (no engine build, no `notifyUpcoming()`) — see AGENTS.md jetsam gotcha. +### Playlist import "spins forever, then orange retry" (Sept 2026, resolved) +Every imported track resolved fine, then every ranged download got HTTP 403 → mapped to +`streamURLExpired` → re-resolve → 403 again → `scheduleRetry` kept the row `.pending` through +5 whole-track attempts (minutes of spinner) → `.failed`. Root cause: the pinned YouTubeKit +(7cc8190, July) fetched stream URLs via the ANDROID_VR InnerTube client, which YouTube stopped +serving past the first chunk in mid-August 2026. Fix: pin YouTubeKit `exact: "0.4.9"` +(visionOS/web clients + embed fallback). Verified with the opt-in live probe test on the +simulator: 0/8 tracks ready before, 8/8 after (full files, BPM analysed). See the AGENTS.md +gotcha for the diagnosis recipe. + ### Catalog search (PR #108) iTunes Search API (no key) for songs/albums; custom in-app keyboard with `CatalogAutocorrect` (ContinuityCore, Linux-tested) learning vocabulary from results + the diff --git a/Packages/ContinuityKit/Package.resolved b/Packages/ContinuityKit/Package.resolved index de76faf..cd33aa7 100644 --- a/Packages/ContinuityKit/Package.resolved +++ b/Packages/ContinuityKit/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "7274ab7883ed66e6373fa5e86d553a5f47b1b676558d15005aefa58cdd4c9260", + "originHash" : "9510f39439c4eba8d14bf450fcc3c836fb7203d1498224e004303a913e7ccc15", "pins" : [ { "identity" : "onnxruntime-swift-package-manager", @@ -15,8 +15,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/alexeichhorn/YouTubeKit", "state" : { - "branch" : "main", - "revision" : "7cc81901f9bbf2f9d2766bd8a54e128b974e4c94" + "revision" : "e5b7d0396ce12bf3444f0d209e8436c83373b7af", + "version" : "0.4.9" } } ], diff --git a/Packages/ContinuityKit/Package.swift b/Packages/ContinuityKit/Package.swift index acd1741..4662772 100644 --- a/Packages/ContinuityKit/Package.swift +++ b/Packages/ContinuityKit/Package.swift @@ -16,7 +16,7 @@ let package = Package( ], dependencies: [ .package(path: "../ContinuityCore"), - .package(url: "https://github.com/alexeichhorn/YouTubeKit", branch: "main"), + .package(url: "https://github.com/alexeichhorn/YouTubeKit", exact: "0.4.9"), .package(url: "https://github.com/microsoft/onnxruntime-swift-package-manager", exact: "1.20.0"), ], targets: [ diff --git a/Packages/ContinuityKit/Tests/IngestTests/LiveIngestProbeTests.swift b/Packages/ContinuityKit/Tests/IngestTests/LiveIngestProbeTests.swift new file mode 100644 index 0000000..d80ade2 --- /dev/null +++ b/Packages/ContinuityKit/Tests/IngestTests/LiveIngestProbeTests.swift @@ -0,0 +1,116 @@ +import XCTest +import SwiftData +import OSLog +import Domain +@testable import Ingest + +/// Opt-in, live-network diagnostic for the ingest pipeline. Skipped unless +/// `CONTINUITY_LIVE_PROBE=1` is in the environment, so normal test runs stay hermetic. +/// +/// Drives the real `PreparationQueue` (YouTubeKit resolve → ranged download → analysis) for a +/// handful of well-known videos plus two search-query tracks, then prints every +/// `com.continuity.app` log line the run produced. When YouTube changes something and every +/// imported track starts spinning and then failing, this shows the failing stage and the exact +/// error in one run — no device, no debugger. Run it against an iOS Simulator destination: +/// +/// cd Packages/ContinuityKit && CONTINUITY_LIVE_PROBE=1 \ +/// DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer xcodebuild test \ +/// -scheme ContinuityKit-Package -destination 'platform=iOS Simulator,id=' \ +/// -only-testing:IngestTests/LiveIngestProbeTests +/// +/// (The env var must reach the test process: xcodebuild forwards `TEST_RUNNER_`-prefixed +/// variables, so `TEST_RUNNER_CONTINUITY_LIVE_PROBE=1` also works.) +/// +/// History: on 2026-09-10 this reproduced "every track spins, then shows the orange retry +/// badge" — every ranged download 403'd (`streamURLExpired`) because the pinned YouTubeKit +/// still asked the ANDROID_VR client for stream URLs, which YouTube stopped serving beyond the +/// first chunk in mid-August 2026. YouTubeKit 0.4.9 (visionOS/web clients) fixed it: 8/8 ready. +@MainActor +final class LiveIngestProbeTests: XCTestCase { + + private static let videoIDs = ["dQw4w9WgXcQ", "9bZkp7q19f0", "kJQP7kiw5Fk", "JGwWNGJdvx8"] + private static let queries = ["Blinding Lights The Weeknd", "bad guy Billie Eilish"] + + override func setUp() async throws { + let env = ProcessInfo.processInfo.environment + try XCTSkipUnless(env["CONTINUITY_LIVE_PROBE"] == "1" || env["TEST_RUNNER_CONTINUITY_LIVE_PROBE"] == "1", + "live-network probe; set CONTINUITY_LIVE_PROBE=1 to run") + } + + func testLiveIngest() async throws { + let start = Date() + let budget: TimeInterval = Double(ProcessInfo.processInfo.environment["CONTINUITY_LIVE_PROBE_SECONDS"] ?? "") ?? 300 + print("PROBE start os=\(ProcessInfo.processInfo.operatingSystemVersionString) budget=\(Int(budget))s") + + // Playlist page scrape (metadata only) — the first stage of a YouTube playlist import. + do { + let resolved = try await YouTubePlaylistResolver().resolvePlaylist(playlistID: "PLFgquLnL59alCl_2TQvOiD5Vgm1hCaGSI") + print("PROBE playlist resolve OK: \(resolved.items.count) items, title=\(resolved.title ?? "-")") + } catch { + print("PROBE playlist resolve FAILED: \(error)") + } + + let schema = Schema([Playlist.self, Track.self]) + let container = try ModelContainer(for: schema, configurations: [ModelConfiguration(isStoredInMemoryOnly: true)]) + let context = container.mainContext + let queue = PreparationQueue() + + let playlist = Playlist(title: "Probe", subtitle: "probe", gradientSeed: 1) + context.insert(playlist) + var tracks: [Track] = [] + for (i, id) in Self.videoIDs.enumerated() { + // Force a real download: evict any cached copy from an earlier run. + for container in ["m4a", "webm", "mp4"] { + try? FileManager.default.removeItem(at: AudioCache.fileURL(videoID: id, container: container)) + } + let track = Track(title: "vid \(id)", artist: "probe", durationSeconds: 0, gradientSeed: i, sortIndex: i, + prepState: .pending, youtubeVideoID: id, + sourceURLString: "https://www.youtube.com/watch?v=\(id)") + playlist.tracks.append(track); context.insert(track); tracks.append(track) + } + for (i, query) in Self.queries.enumerated() { + let track = Track(title: query, artist: "probe", durationSeconds: 0, gradientSeed: 100 + i, sortIndex: 100 + i, + prepState: .pending, searchQuery: query) + playlist.tracks.append(track); context.insert(track); tracks.append(track) + } + try context.save() + for track in tracks { queue.enqueue(track, in: context) } + + var lastStates = "" + while Date().timeIntervalSince(start) < budget { + try await Task.sleep(nanoseconds: 3_000_000_000) + let states = tracks.map { "\($0.youtubeVideoID ?? $0.searchQuery ?? "?")=\($0.prepState.rawValue)" }.joined(separator: " ") + if states != lastStates { + print("PROBE t+\(Int(Date().timeIntervalSince(start)))s \(states)") + lastStates = states + } + if tracks.allSatisfy({ $0.prepState == .ready || $0.prepState == .failed }) { break } + } + + print("PROBE ===== LOG DUMP (com.continuity.app) =====") + do { + let store = try OSLogStore(scope: .currentProcessIdentifier) + let entries = try store.getEntries(at: store.position(date: start)) + var printed = 0 + for case let entry as OSLogEntryLog in entries where entry.subsystem == "com.continuity.app" { + printed += 1 + if printed > 400 { print("PROBE ... (truncated)"); break } + let ts = String(format: "%.1f", entry.date.timeIntervalSince(start)) + print("PROBE LOG t+\(ts)s [\(entry.category)] \(entry.level.rawValue): \(entry.composedMessage)") + } + print("PROBE log entries printed: \(printed)") + } catch { + print("PROBE OSLogStore unavailable: \(error)") + } + + print("PROBE ===== FINAL =====") + for track in tracks { + let path = track.localRelativePath.map { AudioCache.url(forRelativePath: $0).path } ?? "-" + let bytes = (try? FileManager.default.attributesOfItem(atPath: path)[.size] as? Int) ?? 0 + print("PROBE FINAL \(track.youtubeVideoID ?? track.searchQuery ?? "?") state=\(track.prepState.rawValue) dur=\(Int(track.durationSeconds))s bytes=\(bytes) bpm=\(track.bpm ?? 0)") + } + let ready = tracks.filter { $0.prepState == .ready }.count + print("PROBE RESULT ready=\(ready)/\(tracks.count) elapsed=\(Int(Date().timeIntervalSince(start)))s") + XCTAssertEqual(ready, tracks.count, "not every probe track became ready — see PROBE LOG lines above") + } +}