From eb9df5d86af78a348d98782ba309129b3476da7b Mon Sep 17 00:00:00 2001 From: swairshah Date: Wed, 12 Aug 2026 18:31:08 -0700 Subject: [PATCH 1/9] Fix title overlap with sidebar toggle --- Sources/Replay/ContentView.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Sources/Replay/ContentView.swift b/Sources/Replay/ContentView.swift index 5953fb3..7bdd41c 100644 --- a/Sources/Replay/ContentView.swift +++ b/Sources/Replay/ContentView.swift @@ -841,7 +841,10 @@ private struct VideoDetail: View { .fixedSize() } } - .padding(.leading, sidebarCollapsed ? 154 : 14) + // NavigationSplitView keeps its native sidebar toggle in the detail + // title bar even while the sidebar is visible. Reserve the toggle's + // footprint here so long titles cannot render underneath it. + .padding(.leading, sidebarCollapsed ? 154 : 50) .padding(.trailing, 14) .frame(height: 56) } From 52c7513c5e72d26d209cbc22bc89217bee84dcc8 Mon Sep 17 00:00:00 2001 From: swairshah Date: Thu, 13 Aug 2026 02:36:06 -0700 Subject: [PATCH 2/9] feat(chapters): add AI-generated study guides Generate and persist chapter summaries, key points, and exercises from offline subtitles using pi. Add progress, retry, cancellation, regeneration guidance, and chapter-guide controls to the sidebar. Keep timestamp seeking separate from title-based note toggling and cover the enrichment logic with a standalone check suite. --- Sources/Replay/ChapterEnrichment.swift | 556 +++++++++++++++++++++++++ Sources/Replay/ContentView.swift | 346 +++++++++++++-- Sources/Replay/EnrichmentEngine.swift | 386 +++++++++++++++++ Sources/Replay/QueueStore.swift | 124 ++++++ docs/chapter-enrichment-plan.md | 161 +++++++ scripts/test.sh | 6 + tools/chapter_enrichment_check.swift | 264 ++++++++++++ 7 files changed, 1813 insertions(+), 30 deletions(-) create mode 100644 Sources/Replay/ChapterEnrichment.swift create mode 100644 Sources/Replay/EnrichmentEngine.swift create mode 100644 docs/chapter-enrichment-plan.md create mode 100644 tools/chapter_enrichment_check.swift diff --git a/Sources/Replay/ChapterEnrichment.swift b/Sources/Replay/ChapterEnrichment.swift new file mode 100644 index 0000000..42c8a8a --- /dev/null +++ b/Sources/Replay/ChapterEnrichment.swift @@ -0,0 +1,556 @@ +import Foundation + +struct ChapterExercise: Codable, Hashable { + var question: String + var solution: String +} + +struct ChapterEnrichment: Codable, Hashable, Identifiable { + var chapterID: String + var chapterTitle: String + var summary: String + var keyPoints: [String] + var exercises: [ChapterExercise] + var generatedAt: Date + + var id: String { chapterID } +} + +struct VideoEnrichment: Codable, Hashable { + static let currentVersion = 1 + + var version: Int + var itemID: UUID + var generatedAt: Date + var chapters: [ChapterEnrichment] + + func enrichment(forChapterID chapterID: String) -> ChapterEnrichment? { + chapters.first { $0.chapterID == chapterID } + } +} + +/// Pure logic for chapter enrichment: transcript slicing, prompt building, +/// and model-output parsing. Kept free of process/UI concerns so it can be +/// exercised by `tools/chapter_enrichment_check.swift`. +enum ChapterEnrichmentLogic { + /// Maximum transcript characters sent to the model per chapter. Longer + /// transcripts keep the head and tail, which usually carry the chapter's + /// setup and conclusion. + static let transcriptCharacterBudget = 14_000 + + // MARK: Transcript slicing + + /// Returns the effective end time of a chapter, falling back to the next + /// chapter's start or the video duration. + static func endTime( + for chapter: VideoChapter, + in chapters: [VideoChapter], + videoDuration: Double? + ) -> Double { + if let end = chapter.endTime, end > chapter.startTime { return end } + if let index = chapters.firstIndex(of: chapter), index + 1 < chapters.count { + return chapters[index + 1].startTime + } + if let videoDuration, videoDuration > chapter.startTime { return videoDuration } + return .greatestFiniteMagnitude + } + + /// Joins the cue text overlapping a chapter's time range into one + /// transcript string. Consecutive duplicate cue lines (an artifact of + /// YouTube auto-captions) are collapsed. + static func transcript( + for chapter: VideoChapter, + in chapters: [VideoChapter], + cues: [VideoSubtitleCue], + videoDuration: Double? + ) -> String { + let end = endTime(for: chapter, in: chapters, videoDuration: videoDuration) + var lines: [String] = [] + for cue in cues where cue.endTime > chapter.startTime && cue.startTime < end { + for rawLine in cue.text.components(separatedBy: "\n") { + let line = rawLine.trimmingCharacters(in: .whitespacesAndNewlines) + guard !line.isEmpty, line != lines.last else { continue } + lines.append(line) + } + } + return truncateMiddle(lines.joined(separator: " "), limit: transcriptCharacterBudget) + } + + /// Keeps the head and tail of overly long text, marking the elision. + static func truncateMiddle(_ text: String, limit: Int) -> String { + guard text.count > limit, limit > 64 else { return text } + let headCount = (limit * 2) / 3 + let tailCount = limit - headCount + let head = text.prefix(headCount) + let tail = text.suffix(tailCount) + return "\(head)\n[... transcript truncated ...]\n\(tail)" + } + + // MARK: Prompt + + static func prompt( + videoTitle: String, + videoAuthor: String, + chapter: VideoChapter, + chapterIndex: Int, + chapterCount: Int, + allChapterTitles: [String], + transcript: String, + previous: ChapterEnrichment? = nil, + guidance: String? = nil + ) -> String { + let outline = allChapterTitles.enumerated() + .map { "\($0.offset + 1). \($0.element)" } + .joined(separator: "\n") + let author = videoAuthor.isEmpty ? "unknown" : videoAuthor + var revisionBlock = "" + if let guidance, !guidance.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + revisionBlock += """ + + --- USER GUIDANCE (HIGH PRIORITY) --- + The person studying this video asked for the following change: + + \(guidance.trimmingCharacters(in: .whitespacesAndNewlines)) + + Decide for yourself whether this needs a full rewrite or just a \ + targeted modification of the previous version below. Keep whatever \ + already serves the reader well. Either way, respond with the \ + complete JSON object in the required shape. + --- END USER GUIDANCE --- + + """ + } + if let previous { + revisionBlock += """ + + --- PREVIOUS VERSION OF THIS CHAPTER'S GUIDE --- + \(previousVersionJSON(previous)) + --- END PREVIOUS VERSION --- + + """ + } + return """ + You are generating study material for ONE chapter of a video, based on \ + its transcript. Work only from the transcript below. Do not use tools, \ + do not browse, and do not invent facts the transcript does not support. + + Video: \(videoTitle) + Author: \(author) + Chapter \(chapterIndex + 1) of \(chapterCount): \(chapter.title) + + Full chapter outline of the video (for context only — summarize ONLY \ + the current chapter): + \(outline) + + Respond with a single JSON object and nothing else. No prose before or \ + after, no Markdown fences. Exact shape: + + { + "summary": "3-5 sentence plain-language summary of THIS chapter", + "keyPoints": ["2-5 short bullet strings with the concrete takeaways"], + "exercises": [ + { + "question": "a specific exercise or comprehension question", + "solution": "a concise but complete solution or model answer" + } + ] + } + + Rules: + - 2 to 4 exercises, ordered easiest to hardest. + - Exercises must be answerable from this chapter's content alone. + - Prefer concrete recall/application questions over vague reflection. + - Mathematical notation is welcome where it helps: use simple inline \ + LaTeX delimited by $...$ (for example $2^{10}$ or $\\frac{a}{b}$). \ + Keep it simple; prefer plain text when math adds nothing. + - If the transcript contains numbers, formulas, code, or named \ + techniques, use them in the exercises. + - Keep the summary specific: name the actual ideas, not just topics. + - Valid JSON only: escape newlines inside strings as \\n. + + \(revisionBlock) + --- CHAPTER TRANSCRIPT --- + \(transcript) + """ + } + + /// Serializes an existing chapter guide into the same JSON shape the + /// model is asked to produce, for revision prompts. + static func previousVersionJSON(_ enrichment: ChapterEnrichment) -> String { + let payload: [String: Any] = [ + "summary": enrichment.summary, + "keyPoints": enrichment.keyPoints, + "exercises": enrichment.exercises.map { ["question": $0.question, "solution": $0.solution] } + ] + guard let data = try? JSONSerialization.data(withJSONObject: payload, options: [.prettyPrinted, .sortedKeys]), + let json = String(data: data, encoding: .utf8) else { return "{}" } + return json + } + + // MARK: Output parsing + + private struct ModelOutput: Decodable { + struct ModelExercise: Decodable { + let question: String + let solution: String? + } + + let summary: String + let keyPoints: [String]? + let exercises: [ModelExercise]? + } + + /// Parses a pi sub-agent's stdout into a `ChapterEnrichment`. Tolerates + /// Markdown fences and stray prose around the JSON object. + static func parse( + output: String, + chapterID: String, + chapterTitle: String, + generatedAt: Date = Date() + ) -> ChapterEnrichment? { + guard let json = extractJSONObject(from: output), + let data = json.data(using: .utf8), + let decoded = try? JSONDecoder().decode(ModelOutput.self, from: data) else { + return nil + } + let summary = decoded.summary.trimmingCharacters(in: .whitespacesAndNewlines) + guard !summary.isEmpty else { return nil } + let keyPoints = (decoded.keyPoints ?? []) + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + let exercises = (decoded.exercises ?? []).compactMap { exercise -> ChapterExercise? in + let question = exercise.question.trimmingCharacters(in: .whitespacesAndNewlines) + guard !question.isEmpty else { return nil } + let solution = (exercise.solution ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + return ChapterExercise(question: question, solution: solution) + } + return ChapterEnrichment( + chapterID: chapterID, + chapterTitle: chapterTitle, + summary: summary, + keyPoints: keyPoints, + exercises: exercises, + generatedAt: generatedAt + ) + } + + /// Extracts the first balanced top-level JSON object from mixed output, + /// respecting string literals and escapes. + static func extractJSONObject(from text: String) -> String? { + guard let start = text.firstIndex(of: "{") else { return nil } + var depth = 0 + var inString = false + var escaped = false + var index = start + while index < text.endIndex { + let character = text[index] + if escaped { + escaped = false + } else if inString { + if character == "\\" { escaped = true } + else if character == "\"" { inString = false } + } else { + switch character { + case "\"": inString = true + case "{": depth += 1 + case "}": + depth -= 1 + if depth == 0 { return String(text[start...index]) } + default: break + } + } + index = text.index(after: index) + } + return nil + } + + // MARK: Display text (lightweight math rendering) + + /// Converts inline LaTeX math (`$...$`, `$$...$$`, `\(...\)`) into plain + /// Unicode so guide text renders nicely in SwiftUI without a web view. + /// Handles the notation small models actually emit: Greek letters, common + /// operators, fractions, roots, and super/subscripts. Dollar amounts like + /// "$2,000 and $5" are left untouched (a span only counts as math when it + /// contains math-ish structure). + static func displayText(_ text: String) -> String { + var result = replaceDelimitedMath(in: text, open: "\\(", close: "\\)") + result = replaceDelimitedMath(in: result, open: "$$", close: "$$") + result = replaceDollarMath(in: result) + return result + } + + private static func replaceDelimitedMath(in text: String, open: String, close: String) -> String { + var result = "" + var remainder = Substring(text) + while let openRange = remainder.range(of: open) { + let afterOpen = remainder[openRange.upperBound...] + guard let closeRange = afterOpen.range(of: close) else { break } + result += remainder[.. String { + var result = "" + var remainder = Substring(text) + while let openIndex = remainder.firstIndex(of: "$") { + let afterOpen = remainder[remainder.index(after: openIndex)...] + guard let closeIndex = afterOpen.firstIndex(of: "$") else { break } + let body = String(afterOpen[.. Bool { + guard !body.isEmpty, !body.contains("\n") else { return false } + if body.contains("\\") || body.contains("^") || body.contains("_") { return true } + if body.contains("=") || body.contains("{") { return true } + // Single-token algebra such as $x$, $n$, $O(n)$. + let compact = body.replacingOccurrences(of: " ", with: "") + if compact.count <= 8, compact.range(of: #"^[A-Za-z][A-Za-z0-9()+\-*/]*$"#, options: .regularExpression) != nil { + return true + } + return false + } + + private static let superscriptMap: [Character: Character] = [ + "0": "\u{2070}", "1": "\u{00B9}", "2": "\u{00B2}", "3": "\u{00B3}", "4": "\u{2074}", + "5": "\u{2075}", "6": "\u{2076}", "7": "\u{2077}", "8": "\u{2078}", "9": "\u{2079}", + "+": "\u{207A}", "-": "\u{207B}", "=": "\u{207C}", "(": "\u{207D}", ")": "\u{207E}", + "a": "\u{1D43}", "b": "\u{1D47}", "c": "\u{1D9C}", "d": "\u{1D48}", "e": "\u{1D49}", + "f": "\u{1DA0}", "g": "\u{1D4D}", "h": "\u{02B0}", "i": "\u{2071}", "j": "\u{02B2}", + "k": "\u{1D4F}", "l": "\u{02E1}", "m": "\u{1D50}", "n": "\u{207F}", "o": "\u{1D52}", + "p": "\u{1D56}", "r": "\u{02B3}", "s": "\u{02E2}", "t": "\u{1D57}", "u": "\u{1D58}", + "v": "\u{1D5B}", "w": "\u{02B7}", "x": "\u{02E3}", "y": "\u{02B8}", "z": "\u{1DBB}", + "T": "\u{1D40}", "N": "\u{1D3A}", "L": "\u{1D38}", "K": "\u{1D37}", "D": "\u{1D30}" + ] + + private static let subscriptMap: [Character: Character] = [ + "0": "\u{2080}", "1": "\u{2081}", "2": "\u{2082}", "3": "\u{2083}", "4": "\u{2084}", + "5": "\u{2085}", "6": "\u{2086}", "7": "\u{2087}", "8": "\u{2088}", "9": "\u{2089}", + "+": "\u{208A}", "-": "\u{208B}", "=": "\u{208C}", "(": "\u{208D}", ")": "\u{208E}", + "a": "\u{2090}", "e": "\u{2091}", "h": "\u{2095}", "i": "\u{1D62}", "j": "\u{2C7C}", + "k": "\u{2096}", "l": "\u{2097}", "m": "\u{2098}", "n": "\u{2099}", "o": "\u{2092}", + "p": "\u{209A}", "r": "\u{1D63}", "s": "\u{209B}", "t": "\u{209C}", "u": "\u{1D64}", + "v": "\u{1D65}", "x": "\u{2093}" + ] + + private static let mathSymbols: [(String, String)] = [ + ("varepsilon", "\u{03B5}"), ("varphi", "\u{03C6}"), ("epsilon", "\u{03B5}"), + ("upsilon", "\u{03C5}"), ("Upsilon", "\u{03A5}"), ("lambda", "\u{03BB}"), + ("Lambda", "\u{039B}"), ("alpha", "\u{03B1}"), ("gamma", "\u{03B3}"), + ("Gamma", "\u{0393}"), ("delta", "\u{03B4}"), ("Delta", "\u{0394}"), + ("theta", "\u{03B8}"), ("Theta", "\u{0398}"), ("kappa", "\u{03BA}"), + ("sigma", "\u{03C3}"), ("Sigma", "\u{03A3}"), ("omega", "\u{03C9}"), + ("Omega", "\u{03A9}"), ("beta", "\u{03B2}"), ("zeta", "\u{03B6}"), + ("eta", "\u{03B7}"), ("iota", "\u{03B9}"), ("mu", "\u{03BC}"), ("nu", "\u{03BD}"), + ("xi", "\u{03BE}"), ("Xi", "\u{039E}"), ("rho", "\u{03C1}"), ("tau", "\u{03C4}"), + ("phi", "\u{03C6}"), ("Phi", "\u{03A6}"), ("chi", "\u{03C7}"), ("psi", "\u{03C8}"), + ("Psi", "\u{03A8}"), ("pi", "\u{03C0}"), ("Pi", "\u{03A0}"), + ("rightarrow", "\u{2192}"), ("leftarrow", "\u{2190}"), ("Rightarrow", "\u{21D2}"), + ("Leftarrow", "\u{21D0}"), ("mapsto", "\u{21A6}"), ("to", "\u{2192}"), + ("cdots", "\u{22EF}"), ("ldots", "\u{2026}"), ("dots", "\u{2026}"), + ("approx", "\u{2248}"), ("propto", "\u{221D}"), ("simeq", "\u{2243}"), + ("sim", "\u{223C}"), ("times", "\u{00D7}"), ("cdot", "\u{00B7}"), + ("div", "\u{00F7}"), ("pm", "\u{00B1}"), ("mp", "\u{2213}"), + ("leq", "\u{2264}"), ("geq", "\u{2265}"), ("le", "\u{2264}"), ("ge", "\u{2265}"), + ("neq", "\u{2260}"), ("ne", "\u{2260}"), ("infty", "\u{221E}"), + ("partial", "\u{2202}"), ("nabla", "\u{2207}"), ("sum", "\u{2211}"), + ("prod", "\u{220F}"), ("int", "\u{222B}"), ("in", "\u{2208}"), + ("notin", "\u{2209}"), ("subset", "\u{2282}"), ("supset", "\u{2283}"), + ("cup", "\u{222A}"), ("cap", "\u{2229}"), ("forall", "\u{2200}"), + ("exists", "\u{2203}"), ("ell", "\u{2113}"), ("hbar", "\u{210F}"), + ("circ", "\u{2218}"), ("bullet", "\u{2022}"), ("star", "\u{22C6}"), + ("degree", "\u{00B0}") + ] + + /// Converts one math span's body to plain Unicode. + static func unicodeMath(_ body: String) -> String { + var math = body + + // \frac{a}{b} → a/b, parenthesizing multi-character operands. + while let converted = convertFirst( + pattern: #"\\[dt]?frac\{([^{}]*)\}\{([^{}]*)\}"#, + in: math, + transform: { groups in + let numerator = groups[0].count > 1 ? "(\(groups[0]))" : groups[0] + let denominator = groups[1].count > 1 ? "(\(groups[1]))" : groups[1] + return "\(numerator)/\(denominator)" + } + ) { math = converted } + + // \sqrt{x} → √(x) + while let converted = convertFirst(pattern: #"\\sqrt\{([^{}]*)\}"#, in: math, transform: { groups in + groups[0].count > 1 ? "\u{221A}(\(groups[0]))" : "\u{221A}\(groups[0])" + }) { math = converted } + + // Unwrap text-style commands: \text{x}, \mathrm{x}, \operatorname{x}, ... + while let converted = convertFirst( + pattern: #"\\(?:text|textrm|textit|textbf|mathrm|mathit|mathbf|mathsf|mathcal|operatorname)\{([^{}]*)\}"#, + in: math, + transform: { $0[0] } + ) { math = converted } + + // Named symbols (word-boundary aware). + for (name, symbol) in mathSymbols { + math = math.replacingOccurrences( + of: "\\\\\(name)(?![A-Za-z])", + with: symbol, + options: .regularExpression + ) + } + + // Spacing commands and grouping fences. + for token in ["\\,", "\\;", "\\:", "\\!", "\\left", "\\right"] { + math = math.replacingOccurrences(of: token, with: token == "\\!" ? "" : " ") + } + + // Super/subscripts: ^{...} / ^x and _{...} / _x. Single pass, left + // to right (the fallback can leave `^x` unchanged, so re-scanning + // would never terminate). + math = convertAll(pattern: #"\^\{([^{}]+)\}|\^([A-Za-z0-9+\-=()])"#, in: math) { groups in + scriptText(groups.first(where: { !$0.isEmpty }) ?? "", using: superscriptMap, prefix: "^") + } + math = convertAll(pattern: #"_\{([^{}]+)\}|_([A-Za-z0-9+\-=()])"#, in: math) { groups in + scriptText(groups.first(where: { !$0.isEmpty }) ?? "", using: subscriptMap, prefix: "_") + } + + // Remaining commands become bare words (\log → log); drop stray braces. + math = math.replacingOccurrences(of: #"\\([A-Za-z]+)"#, with: "$1", options: .regularExpression) + math = math.replacingOccurrences(of: "{", with: "").replacingOccurrences(of: "}", with: "") + math = math.replacingOccurrences(of: #"\s{2,}"#, with: " ", options: .regularExpression) + return math.trimmingCharacters(in: .whitespaces) + } + + /// Maps script text to Unicode super/subscript characters, falling back + /// to `^body` / `_body` when any character has no Unicode form. + private static func scriptText(_ body: String, using map: [Character: Character], prefix: String) -> String { + var mapped = "" + for character in body { + if character == " " { continue } + guard let replacement = map[character] else { return prefix + body } + mapped.append(replacement) + } + return mapped + } + + /// Replaces every match of `pattern` in one left-to-right pass, handing + /// capture groups to `transform`. + private static func convertAll( + pattern: String, + in text: String, + transform: ([String]) -> String + ) -> String { + guard let regex = try? NSRegularExpression(pattern: pattern) else { return text } + let matches = regex.matches(in: text, range: NSRange(text.startIndex..., in: text)) + guard !matches.isEmpty else { return text } + var result = "" + var cursor = text.startIndex + for match in matches { + guard let matchRange = Range(match.range, in: text) else { continue } + var groups: [String] = [] + for index in 1.. String + ) -> String? { + guard let regex = try? NSRegularExpression(pattern: pattern), + let match = regex.firstMatch(in: text, range: NSRange(text.startIndex..., in: text)), + let matchRange = Range(match.range, in: text) else { return nil } + var groups: [String] = [] + for index in 1.. String? { + // Prefer an embedded provider error message, e.g. + // {"type":"error","error":{"message":"You're out of extra usage..."}} + if let range = stderr.range(of: #""message"\s*:\s*"((?:[^"\\]|\\.)*)""#, options: .regularExpression) { + let match = String(stderr[range]) + if let valueStart = match.range(of: #":\s*"#, options: .regularExpression) { + var value = String(match[valueStart.upperBound...]) + value = value.trimmingCharacters(in: CharacterSet(charactersIn: "\"")) + value = value + .replacingOccurrences(of: "\\\"", with: "\"") + .replacingOccurrences(of: "\\n", with: " ") + if !value.isEmpty { return String(value.prefix(220)) } + } + } + let lines = stderr + .components(separatedBy: .newlines) + .map { $0.trimmingCharacters(in: .whitespaces) } + .filter { !$0.isEmpty && !$0.hasPrefix("Download ") && !$0.hasPrefix("Warning:") && !$0.hasPrefix("at ") } + return lines.first.map { String($0.prefix(220)) } + } + + // MARK: Persistence + + static func load(from url: URL) -> VideoEnrichment? { + guard let data = try? Data(contentsOf: url) else { return nil } + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = .iso8601 + guard let enrichment = try? decoder.decode(VideoEnrichment.self, from: data), + enrichment.version == VideoEnrichment.currentVersion else { return nil } + return enrichment + } + + static func save(_ enrichment: VideoEnrichment, to url: URL) throws { + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + encoder.dateEncodingStrategy = .iso8601 + let data = try encoder.encode(enrichment) + try FileManager.default.createDirectory( + at: url.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + try data.write(to: url, options: .atomic) + } +} diff --git a/Sources/Replay/ContentView.swift b/Sources/Replay/ContentView.swift index 7bdd41c..4ff2044 100644 --- a/Sources/Replay/ContentView.swift +++ b/Sources/Replay/ContentView.swift @@ -782,7 +782,7 @@ private struct VideoDetail: View { centerPane .inspector(isPresented: $chaptersPresented) { chapterSidebar - .inspectorColumnWidth(min: 232, ideal: 300, max: 400) + .inspectorColumnWidth(min: 232, ideal: 330, max: 500) } } else { HSplitView { @@ -791,7 +791,7 @@ private struct VideoDetail: View { if chaptersPresented { chapterSidebar - .frame(minWidth: 232, idealWidth: 300, maxWidth: 400) + .frame(minWidth: 232, idealWidth: 330, maxWidth: 500) } } } @@ -922,8 +922,14 @@ private struct VideoDetail: View { chapters: item.availableChapters, currentTime: playback.currentTime, isPresented: chaptersPresented, + enrichment: store.enrichments[item.id], + activity: store.enrichmentActivity[item.id], + canEnrich: item.state == .ready && item.subtitleFileURL != nil, toggle: toggleChapters, - select: seekToChapter + select: seekToChapter, + enrich: { force, guidance in store.enrichChapters(for: item.id, force: force, guidance: guidance) }, + cancelEnrichment: { store.cancelEnrichment(for: item.id) }, + dismissFailure: { store.dismissEnrichmentFailure(for: item.id) } ) .ignoresSafeArea(.container, edges: .top) } @@ -1747,8 +1753,19 @@ private struct ChapterSidebar: View { let chapters: [VideoChapter] let currentTime: Double let isPresented: Bool + let enrichment: VideoEnrichment? + let activity: QueueStore.EnrichmentActivity? + let canEnrich: Bool let toggle: () -> Void let select: (VideoChapter) -> Void + let enrich: (_ force: Bool, _ guidance: String?) -> Void + let cancelEnrichment: () -> Void + let dismissFailure: () -> Void + + @State private var expandedChapters: Set = [] + @State private var revealedSolutions: Set = [] + @State private var regeneratePopoverShown = false + @State private var regenerateGuidance = "" var body: some View { VStack(alignment: .leading, spacing: 0) { @@ -1781,43 +1798,312 @@ private struct ChapterSidebar: View { ScrollView { LazyVStack(alignment: .leading, spacing: 3) { ForEach(chapters) { chapter in - Button { - select(chapter) - } label: { - HStack(alignment: .firstTextBaseline, spacing: 10) { - Text(formatTime(chapter.startTime)) - .font(.caption.monospacedDigit()) - .foregroundStyle(isCurrent(chapter) ? Color.accentColor : Color.secondary) - .frame(width: 48, alignment: .trailing) - Text(chapter.title) - .font(.callout.weight(isCurrent(chapter) ? .semibold : .regular)) - .foregroundStyle(.primary) - .multilineTextAlignment(.leading) - .lineLimit(2) - .fixedSize(horizontal: false, vertical: true) - Spacer(minLength: 0) - } - .padding(.horizontal, 10) - .padding(.vertical, 10) - .background { - if isCurrent(chapter) { - RoundedRectangle(cornerRadius: 10, style: .continuous) - .fill(Color.accentColor.opacity(0.13)) - } - } - .contentShape(Rectangle()) - } - .buttonStyle(.plain) + chapterRow(chapter) } } .padding(.horizontal, 8) .padding(.vertical, 8) } .scrollIndicators(.hidden) + + if canEnrich || activity != nil { + Divider() + enrichmentFooter + } } .background(.ultraThinMaterial) } + @ViewBuilder + private func chapterRow(_ chapter: VideoChapter) -> some View { + let chapterEnrichment = enrichment?.enrichment(forChapterID: chapter.id) + VStack(alignment: .leading, spacing: 0) { + HStack(alignment: .firstTextBaseline, spacing: 10) { + Button { + select(chapter) + } label: { + Text(formatTime(chapter.startTime)) + .font(.caption.monospacedDigit()) + .foregroundStyle(isCurrent(chapter) ? Color.accentColor : Color.secondary) + .frame(width: 48, alignment: .trailing) + .contentShape(Rectangle()) + } + .buttonStyle(.plain) + .help("Go to \(formatTime(chapter.startTime))") + + if chapterEnrichment != nil { + Button { + toggleExpanded(chapter.id) + } label: { + HStack(alignment: .firstTextBaseline, spacing: 6) { + chapterTitle(chapter) + Spacer(minLength: 0) + Image(systemName: expandedChapters.contains(chapter.id) ? "chevron.down" : "chevron.right") + .font(.caption.weight(.semibold)) + .foregroundStyle(.secondary) + .frame(width: 22, height: 22) + } + .contentShape(Rectangle()) + } + .buttonStyle(.plain) + .help(expandedChapters.contains(chapter.id) ? "Hide summary and exercises" : "Show summary and exercises") + } else { + chapterTitle(chapter) + Spacer(minLength: 0) + } + } + .padding(.leading, 10) + .padding(.trailing, 6) + .padding(.vertical, 10) + + if let chapterEnrichment, expandedChapters.contains(chapter.id) { + enrichmentDetail(chapterEnrichment) + .padding(.leading, 16) + .padding(.trailing, 14) + .padding(.top, 2) + .padding(.bottom, 16) + } + } + .background { + if isCurrent(chapter) { + RoundedRectangle(cornerRadius: 10, style: .continuous) + .fill(Color.accentColor.opacity(0.13)) + } + } + } + + private func chapterTitle(_ chapter: VideoChapter) -> some View { + Text(chapter.title) + .font(.callout.weight(isCurrent(chapter) ? .semibold : .regular)) + .foregroundStyle(.primary) + .multilineTextAlignment(.leading) + .lineLimit(2) + .fixedSize(horizontal: false, vertical: true) + } + + /// Reading typography for the chapter guide, modeled on Clearly's + /// preview: a serif reading face (New York on macOS), comfortable size, + /// and ~1.7 line height. Line spacing below is (lineHeight - 1) × size. + private enum GuideType { + static let body = Font.system(size: 14.5, weight: .regular, design: .serif) + static let bodyLineSpacing: CGFloat = 14.5 * 0.7 + static let question = Font.system(size: 14.5, weight: .medium, design: .serif) + static let sectionLabel = Font.system(size: 10.5, weight: .semibold) + static let ordinal = Font.system(size: 12, weight: .semibold, design: .rounded) + } + + private func enrichmentDetail(_ chapterEnrichment: ChapterEnrichment) -> some View { + VStack(alignment: .leading, spacing: 14) { + Text(ChapterEnrichmentLogic.displayText(chapterEnrichment.summary)) + .font(GuideType.body) + .foregroundStyle(.primary) + .lineSpacing(GuideType.bodyLineSpacing) + .fixedSize(horizontal: false, vertical: true) + .textSelection(.enabled) + + if !chapterEnrichment.keyPoints.isEmpty { + VStack(alignment: .leading, spacing: 7) { + ForEach(Array(chapterEnrichment.keyPoints.enumerated()), id: \.offset) { _, point in + HStack(alignment: .firstTextBaseline, spacing: 9) { + Circle() + .fill(.tertiary) + .frame(width: 4, height: 4) + .alignmentGuide(.firstTextBaseline) { $0[VerticalAlignment.center] + 5 } + Text(ChapterEnrichmentLogic.displayText(point)) + .font(GuideType.body) + .foregroundStyle(.primary.opacity(0.85)) + .lineSpacing(GuideType.bodyLineSpacing * 0.75) + .fixedSize(horizontal: false, vertical: true) + .textSelection(.enabled) + } + } + } + } + + if !chapterEnrichment.exercises.isEmpty { + VStack(alignment: .leading, spacing: 12) { + Text("EXERCISES") + .font(GuideType.sectionLabel) + .kerning(1.1) + .foregroundStyle(.secondary) + .padding(.top, 4) + ForEach(Array(chapterEnrichment.exercises.enumerated()), id: \.offset) { index, exercise in + exerciseView(exercise, index: index, chapterID: chapterEnrichment.chapterID) + } + } + } + } + } + + private func exerciseView(_ exercise: ChapterExercise, index: Int, chapterID: String) -> some View { + let solutionKey = "\(chapterID)#\(index)" + return VStack(alignment: .leading, spacing: 8) { + HStack(alignment: .firstTextBaseline, spacing: 9) { + Text("\(index + 1)") + .font(GuideType.ordinal) + .monospacedDigit() + .foregroundStyle(.secondary) + .frame(width: 18, height: 18) + .background(Color.secondary.opacity(0.12), in: Circle()) + .alignmentGuide(.firstTextBaseline) { $0[VerticalAlignment.center] + 5 } + Text(ChapterEnrichmentLogic.displayText(exercise.question)) + .font(GuideType.question) + .foregroundStyle(.primary) + .lineSpacing(GuideType.bodyLineSpacing * 0.75) + .fixedSize(horizontal: false, vertical: true) + .textSelection(.enabled) + } + if !exercise.solution.isEmpty { + let isRevealed = revealedSolutions.contains(solutionKey) + VStack(alignment: .leading, spacing: 6) { + Button(isRevealed ? "Hide solution" : "Show solution") { + toggleSolution(solutionKey) + } + .buttonStyle(.plain) + .font(.callout.weight(.medium)) + .foregroundStyle(Color.accentColor) + if isRevealed { + Text(ChapterEnrichmentLogic.displayText(exercise.solution)) + .font(GuideType.body) + .foregroundStyle(.primary.opacity(0.75)) + .lineSpacing(GuideType.bodyLineSpacing * 0.75) + .fixedSize(horizontal: false, vertical: true) + .textSelection(.enabled) + .padding(.leading, 12) + .overlay(alignment: .leading) { + RoundedRectangle(cornerRadius: 1) + .fill(Color.accentColor.opacity(0.45)) + .frame(width: 2) + } + } + } + .padding(.leading, 27) + } + } + } + + @ViewBuilder + private var enrichmentFooter: some View { + Group { + switch activity { + case .running(let completed, let total): + HStack(spacing: 8) { + ProgressView(value: Double(completed), total: Double(max(total, 1))) + .progressViewStyle(.linear) + Text("\(completed)/\(total)") + .font(.caption.monospacedDigit()) + .foregroundStyle(.secondary) + Button { + cancelEnrichment() + } label: { + Image(systemName: "xmark.circle.fill") + .foregroundStyle(.secondary) + } + .buttonStyle(.plain) + .help("Cancel") + } + case .failed(let message): + HStack(alignment: .firstTextBaseline, spacing: 6) { + Image(systemName: "exclamationmark.triangle") + .font(.caption) + .foregroundStyle(.orange) + Text(message) + .font(.caption) + .foregroundStyle(.secondary) + .lineLimit(3) + Spacer(minLength: 4) + Button("Retry") { + dismissFailure() + enrich(false, nil) + } + .buttonStyle(.plain) + .font(.caption.weight(.medium)) + .foregroundStyle(Color.accentColor) + } + case nil: + if enrichment == nil { + Button { + enrich(false, nil) + } label: { + Label("Summaries & exercises", systemImage: "note.text") + .font(.caption.weight(.medium)) + .frame(maxWidth: .infinity) + } + .watchGlassButton() + .help("Generate a summary and exercises for each chapter") + } else { + HStack(spacing: 6) { + Label("Chapter guide ready", systemImage: "note.text") + .font(.caption) + .foregroundStyle(.secondary) + Spacer(minLength: 4) + Button("Regenerate") { + regeneratePopoverShown = true + } + .buttonStyle(.plain) + .font(.caption.weight(.medium)) + .foregroundStyle(Color.accentColor) + .popover(isPresented: $regeneratePopoverShown, arrowEdge: .bottom) { + regeneratePopover + } + } + } + } + } + .padding(.horizontal, 12) + .padding(.vertical, 9) + } + + private var regeneratePopover: some View { + VStack(alignment: .leading, spacing: 10) { + Text("Regenerate chapter guide") + .font(.headline) + Text("Optionally tell the model what to change — it will revise the existing guide, or rewrite it when that serves better.") + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + TextField("e.g. add more exercises, make summaries shorter…", text: $regenerateGuidance, axis: .vertical) + .textFieldStyle(.roundedBorder) + .lineLimit(2...5) + .frame(width: 280) + .onSubmit(submitRegenerate) + HStack { + Spacer() + Button("Cancel") { + regeneratePopoverShown = false + } + Button(regenerateGuidance.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ? "Regenerate from scratch" : "Regenerate") { + submitRegenerate() + } + .keyboardShortcut(.defaultAction) + } + } + .padding(14) + } + + private func submitRegenerate() { + let guidance = regenerateGuidance.trimmingCharacters(in: .whitespacesAndNewlines) + regeneratePopoverShown = false + regenerateGuidance = "" + enrich(true, guidance.isEmpty ? nil : guidance) + } + + private func toggleSolution(_ solutionKey: String) { + if revealedSolutions.contains(solutionKey) { + revealedSolutions.remove(solutionKey) + } else { + revealedSolutions.insert(solutionKey) + } + } + + private func toggleExpanded(_ chapterID: String) { + if expandedChapters.contains(chapterID) { + expandedChapters.remove(chapterID) + } else { + expandedChapters.insert(chapterID) + } + } + private func isCurrent(_ chapter: VideoChapter) -> Bool { guard currentTime >= chapter.startTime else { return false } if let endTime = chapter.endTime { return currentTime < endTime } diff --git a/Sources/Replay/EnrichmentEngine.swift b/Sources/Replay/EnrichmentEngine.swift new file mode 100644 index 0000000..09a3fd3 --- /dev/null +++ b/Sources/Replay/EnrichmentEngine.swift @@ -0,0 +1,386 @@ +import Foundation + +/// Generates per-chapter summaries and exercises by running the pi coding +/// agent through the bundled Deno runtime (`deno run npm:...`), one +/// subprocess per chapter with bounded concurrency. Results are persisted by +/// the caller; this engine only orchestrates subprocesses and parsing. +final class EnrichmentEngine { + struct Progress { + let completedChapters: Int + let totalChapters: Int + } + + struct Outcome { + let enrichment: VideoEnrichment + let failedChapterTitles: [String] + } + + enum EngineError: LocalizedError { + case missingTool(String) + case missingSubtitles + case cancelled + case failed(String) + + var errorDescription: String? { + switch self { + case .missingTool(let tool): + return "\(tool) is not available. Reinstall Replay or install it with Homebrew." + case .missingSubtitles: + return "This video has no offline subtitles to summarize." + case .cancelled: + return "Enrichment cancelled." + case .failed(let message): + return message + } + } + } + + + + /// Runtime knobs, overridable per machine without a rebuild: + /// `defaults write com.mg.replay ` or environment variables. + enum Configuration { + /// Pinned pi package for reproducible runs. + static let defaultPiPackage = "npm:@earendil-works/pi-coding-agent@0.84.1" + /// Verified working under the bundled Deno (v2.9.5+). Note: the + /// Codex provider opens a WebSocket, which crashes on Deno < 2.9 + /// (undici MessageEvent incompatibility) — keep the bundled Deno + /// current. + static let defaultProvider = "openai-codex" + /// Without an explicit model, pi keeps its configured default model, + /// which may belong to a different provider. Always pin a model when + /// using the default provider. + static let defaultModel = "gpt-5.6-sol" + + static func value(key: String, environment: String) -> String? { + if let fromDefaults = UserDefaults.standard.string(forKey: key), !fromDefaults.isEmpty { + return fromDefaults + } + if let fromEnvironment = ProcessInfo.processInfo.environment[environment], !fromEnvironment.isEmpty { + return fromEnvironment + } + return nil + } + + static var piPackage: String { + value(key: "enrichmentPiPackage", environment: "REPLAY_PI_PACKAGE") ?? defaultPiPackage + } + + static var provider: String? { + value(key: "enrichmentPiProvider", environment: "REPLAY_PI_PROVIDER") ?? defaultProvider + } + + static var model: String? { + if let override = value(key: "enrichmentPiModel", environment: "REPLAY_PI_MODEL") { + return override + } + // Only apply the pinned default model when the provider is also + // the default; a custom provider needs a matching custom model. + return provider == defaultProvider ? defaultModel : nil + } + } + + private static let maximumConcurrentChapters = 2 + private static let chapterTimeout: TimeInterval = 420 + + private let processLock = NSLock() + private var processes: [UUID: [Process]] = [:] + private var cancelledItems: Set = [] + + func enrich( + item: WatchItem, + existing: VideoEnrichment?, + guidance: String? = nil, + workDirectory: URL, + denoCacheDirectory: URL, + onProgress: @escaping (Progress) -> Void, + completion: @escaping (Result) -> Void + ) { + DispatchQueue.global(qos: .utility).async { [weak self] in + guard let self else { return } + do { + let outcome = try self.performEnrichment( + item: item, + existing: existing, + guidance: guidance, + workDirectory: workDirectory, + denoCacheDirectory: denoCacheDirectory, + onProgress: onProgress + ) + completion(.success(outcome)) + } catch { + completion(.failure(error)) + } + self.clear(itemID: item.id) + } + } + + func cancel(itemID: UUID) { + processLock.lock() + cancelledItems.insert(itemID) + let running = processes[itemID] ?? [] + processLock.unlock() + running.forEach { $0.terminate() } + } + + // MARK: - Orchestration + + private func performEnrichment( + item: WatchItem, + existing: VideoEnrichment?, + guidance: String?, + workDirectory: URL, + denoCacheDirectory: URL, + onProgress: @escaping (Progress) -> Void + ) throws -> Outcome { + let trimmedGuidance = guidance?.trimmingCharacters(in: .whitespacesAndNewlines) + let revisionGuidance = (trimmedGuidance?.isEmpty ?? true) ? nil : trimmedGuidance + processLock.lock() + cancelledItems.remove(item.id) + processLock.unlock() + + guard let subtitleURL = item.subtitleFileURL, + let track = VideoSubtitleTrack(contentsOf: subtitleURL) else { + throw EngineError.missingSubtitles + } + let deno = try requiredTool(named: "deno") + + // Videos without creator chapters are treated as one whole-video chapter. + let chapters = item.availableChapters.isEmpty + ? [VideoChapter(title: item.title, startTime: 0, endTime: item.duration)] + : item.availableChapters + + try FileManager.default.createDirectory(at: workDirectory, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: denoCacheDirectory, withIntermediateDirectories: true) + + let totalChapters = chapters.count + let stateLock = NSLock() + var results: [Int: ChapterEnrichment] = [:] + var failures: [Int: String] = [:] + var completedCount = 0 + + let queue = OperationQueue() + queue.maxConcurrentOperationCount = Self.maximumConcurrentChapters + queue.qualityOfService = .utility + + for (index, chapter) in chapters.enumerated() { + // Reuse chapters that were already generated in a previous run — + // unless the user gave revision guidance, in which case every + // chapter is re-run with its previous version as context. + if revisionGuidance == nil, let reused = existing?.enrichment(forChapterID: chapter.id) { + stateLock.lock() + results[index] = reused + completedCount += 1 + let progress = Progress(completedChapters: completedCount, totalChapters: totalChapters) + stateLock.unlock() + onProgress(progress) + continue + } + + queue.addOperation { [weak self] in + guard let self, !self.isCancelled(itemID: item.id) else { return } + let result = self.enrichChapter( + itemID: item.id, + item: item, + chapter: chapter, + chapterIndex: index, + chapters: chapters, + cues: track.cues, + previous: existing?.enrichment(forChapterID: chapter.id), + guidance: revisionGuidance, + deno: deno, + workDirectory: workDirectory, + denoCacheDirectory: denoCacheDirectory + ) + stateLock.lock() + switch result { + case .success(let enrichment): results[index] = enrichment + case .failure(let error): failures[index] = error.localizedDescription + } + completedCount += 1 + let progress = Progress(completedChapters: completedCount, totalChapters: totalChapters) + stateLock.unlock() + onProgress(progress) + } + } + + queue.waitUntilAllOperationsAreFinished() + + if isCancelled(itemID: item.id) { throw EngineError.cancelled } + + let ordered = results.keys.sorted().compactMap { results[$0] } + guard !ordered.isEmpty else { + let detail = failures.values.first ?? "The pi sub-agents produced no usable output." + throw EngineError.failed(detail) + } + let failedTitles = failures.keys.sorted().map { chapters[$0].title } + let enrichment = VideoEnrichment( + version: VideoEnrichment.currentVersion, + itemID: item.id, + generatedAt: Date(), + chapters: ordered + ) + return Outcome(enrichment: enrichment, failedChapterTitles: failedTitles) + } + + private func enrichChapter( + itemID: UUID, + item: WatchItem, + chapter: VideoChapter, + chapterIndex: Int, + chapters: [VideoChapter], + cues: [VideoSubtitleCue], + previous: ChapterEnrichment?, + guidance: String?, + deno: URL, + workDirectory: URL, + denoCacheDirectory: URL + ) -> Result { + let transcript = ChapterEnrichmentLogic.transcript( + for: chapter, + in: chapters, + cues: cues, + videoDuration: item.duration + ) + guard transcript.count >= 40 else { + return .failure(EngineError.failed("Chapter “\(chapter.title)” has too little transcript text.")) + } + + let prompt = ChapterEnrichmentLogic.prompt( + videoTitle: item.title, + videoAuthor: item.author, + chapter: chapter, + chapterIndex: chapterIndex, + chapterCount: chapters.count, + allChapterTitles: chapters.map(\.title), + transcript: transcript, + previous: guidance != nil ? previous : nil, + guidance: guidance + ) + let slug = String(format: "chapter-%02d", chapterIndex + 1) + let promptFile = workDirectory.appendingPathComponent("\(slug).prompt.md") + let stderrFile = workDirectory.appendingPathComponent("\(slug).stderr.txt") + let stdoutFile = workDirectory.appendingPathComponent("\(slug).stdout.txt") + do { + try prompt.write(to: promptFile, atomically: true, encoding: .utf8) + } catch { + return .failure(error) + } + + let process = Process() + process.executableURL = deno + // Headless, hermetic pi run: no session, no extensions, no skills, no + // tools. Extensions in particular can open WebSockets, which crash + // under Deno's npm compatibility layer. + var arguments = [ + "run", "-A", "--quiet", Configuration.piPackage, + "-p", "--no-session", "-ne", "--no-skills", + "--no-prompt-templates", "--no-context-files", "--no-tools" + ] + if let provider = Configuration.provider { + arguments += ["--provider", provider] + } + if let model = Configuration.model { + arguments += ["--model", model] + } + arguments.append("@\(promptFile.path)") + process.arguments = arguments + process.currentDirectoryURL = workDirectory + var environment = ProcessInfo.processInfo.environment + environment["DENO_DIR"] = denoCacheDirectory.path + environment["NO_COLOR"] = "1" + process.environment = environment + + let stdout = Pipe() + let stderr = Pipe() + process.standardOutput = stdout + process.standardError = stderr + + do { + try process.run() + } catch { + return .failure(error) + } + register(process, for: itemID) + + // Watchdog: kill hung sub-agents so the queue always finishes. + let watchdog = DispatchWorkItem { [weak process] in process?.terminate() } + DispatchQueue.global(qos: .utility).asyncAfter( + deadline: .now() + Self.chapterTimeout, + execute: watchdog + ) + + let stdoutData = stdout.fileHandleForReading.readDataToEndOfFile() + let stderrData = stderr.fileHandleForReading.readDataToEndOfFile() + process.waitUntilExit() + watchdog.cancel() + unregister(process, for: itemID) + + let output = String(decoding: stdoutData, as: UTF8.self) + try? output.write(to: stdoutFile, atomically: true, encoding: .utf8) + try? String(decoding: stderrData, as: UTF8.self).write(to: stderrFile, atomically: true, encoding: .utf8) + + if isCancelled(itemID: itemID) { + return .failure(EngineError.cancelled) + } + guard process.terminationStatus == 0 else { + let stderrText = String(decoding: stderrData, as: UTF8.self) + let detail = ChapterEnrichmentLogic.errorSummary(fromStderr: stderrText) + ?? "pi exited with status \(process.terminationStatus)" + return .failure(EngineError.failed(detail)) + } + guard let enrichment = ChapterEnrichmentLogic.parse( + output: output, + chapterID: chapter.id, + chapterTitle: chapter.title + ) else { + return .failure(EngineError.failed("Could not parse the model output for “\(chapter.title)”. See \(stdoutFile.lastPathComponent).")) + } + return .success(enrichment) + } + + // MARK: - Helpers + + private func isCancelled(itemID: UUID) -> Bool { + processLock.lock() + defer { processLock.unlock() } + return cancelledItems.contains(itemID) + } + + private func register(_ process: Process, for itemID: UUID) { + processLock.lock() + processes[itemID, default: []].append(process) + processLock.unlock() + } + + private func unregister(_ process: Process, for itemID: UUID) { + processLock.lock() + processes[itemID]?.removeAll { $0 === process } + processLock.unlock() + } + + private func clear(itemID: UUID) { + processLock.lock() + processes[itemID] = nil + cancelledItems.remove(itemID) + processLock.unlock() + } + + private func requiredTool(named name: String) throws -> URL { + guard let tool = findTool(named: name) else { throw EngineError.missingTool(name) } + return tool + } + + private func findTool(named name: String) -> URL? { + var candidates: [String] = [] + if let resources = Bundle.main.resourceURL { + candidates.append(resources.appendingPathComponent("Tools/\(name)").path) + } + candidates += [ + "/opt/homebrew/bin/\(name)", + "/usr/local/bin/\(name)", + "/usr/bin/\(name)" + ] + return candidates.map(URL.init(fileURLWithPath:)).first { FileManager.default.isExecutableFile(atPath: $0.path) } + } +} diff --git a/Sources/Replay/QueueStore.swift b/Sources/Replay/QueueStore.swift index 2c14ce4..400141f 100644 --- a/Sources/Replay/QueueStore.swift +++ b/Sources/Replay/QueueStore.swift @@ -52,6 +52,11 @@ final class QueueStore: ObservableObject { case existing } + enum EnrichmentActivity: Equatable { + case running(completed: Int, total: Int) + case failed(String) + } + struct IntakeNotice: Identifiable, Equatable { let id = UUID() let title: String @@ -63,8 +68,13 @@ final class QueueStore: ObservableObject { @Published var selection: UUID? @Published var lastIntakeError: String? @Published private(set) var intakeNotice: IntakeNotice? + @Published private(set) var enrichments: [UUID: VideoEnrichment] = [:] + @Published private(set) var enrichmentActivity: [UUID: EnrichmentActivity] = [:] private let downloader = DownloadEngine() + private let enricher = EnrichmentEngine() + private let enrichmentRoot: URL + private let denoCacheFolder: URL private let networkMonitor = NetworkMonitor() private let powerMonitor = PowerModeMonitor() private let maximumConcurrentDownloads = 3 @@ -95,6 +105,8 @@ final class QueueStore: ObservableObject { let applicationSupport = migration.applicationSupport dataFile = applicationSupport.appendingPathComponent("queue.json") + enrichmentRoot = applicationSupport.appendingPathComponent("Enrichment", isDirectory: true) + denoCacheFolder = applicationSupport.appendingPathComponent("DenoCache", isDirectory: true) persistenceWriter = QueuePersistenceWriter(dataFile: dataFile) mediaFolder = migration.mediaFolder try? fileManager.createDirectory(at: applicationSupport, withIntermediateDirectories: true) @@ -116,6 +128,7 @@ final class QueueStore: ObservableObject { } save() selection = queueItems.first?.id ?? archivedItems.first?.id + loadPersistedEnrichments() networkMonitor.onBecameOnline = { [weak self] in self?.resumeWaitingDownloads() @@ -458,9 +471,120 @@ final class QueueStore: ObservableObject { } } + // MARK: - Chapter enrichment + + func enrichmentFileURL(for id: UUID) -> URL { + enrichmentRoot + .appendingPathComponent(id.uuidString, isDirectory: true) + .appendingPathComponent("enrichment.json") + } + + func isEnriching(_ id: UUID) -> Bool { + if case .running = enrichmentActivity[id] { return true } + return false + } + + func enrichChapters(for id: UUID, force: Bool = false, guidance: String? = nil) { + guard let item = item(with: id), item.state == .ready, !isEnriching(id) else { return } + guard item.subtitleFileURL != nil else { + enrichmentActivity[id] = .failed("This video has no offline subtitles to summarize.") + return + } + let trimmedGuidance = guidance?.trimmingCharacters(in: .whitespacesAndNewlines) + let revisionGuidance = (trimmedGuidance?.isEmpty ?? true) ? nil : trimmedGuidance + // With guidance, the previous guide is context for revision; a plain + // force regenerates from scratch. + let existing = (force && revisionGuidance == nil) ? nil : enrichments[id] + let chapterCount = max(1, item.availableChapters.count) + enrichmentActivity[id] = .running(completed: 0, total: chapterCount) + + let workDirectory = enrichmentFileURL(for: id).deletingLastPathComponent() + enricher.enrich( + item: item, + existing: existing, + guidance: revisionGuidance, + workDirectory: workDirectory, + denoCacheDirectory: denoCacheFolder, + onProgress: { [weak self] progress in + DispatchQueue.main.async { + guard let self, self.isEnriching(id) else { return } + self.enrichmentActivity[id] = .running( + completed: progress.completedChapters, + total: progress.totalChapters + ) + } + }, + completion: { [weak self] result in + DispatchQueue.main.async { + self?.finishEnrichment(result, for: id) + } + } + ) + } + + func cancelEnrichment(for id: UUID) { + enricher.cancel(itemID: id) + enrichmentActivity[id] = nil + } + + func dismissEnrichmentFailure(for id: UUID) { + if case .failed = enrichmentActivity[id] { enrichmentActivity[id] = nil } + } + + private func finishEnrichment(_ result: Result, for id: UUID) { + switch result { + case .success(let outcome): + enrichments[id] = outcome.enrichment + enrichmentActivity[id] = nil + try? ChapterEnrichmentLogic.save(outcome.enrichment, to: enrichmentFileURL(for: id)) + if outcome.failedChapterTitles.isEmpty { + showIntakeNotice( + title: "Chapter guide ready", + detail: "Summaries and exercises were generated", + systemImage: "note.text" + ) + } else { + let count = outcome.failedChapterTitles.count + showIntakeNotice( + title: "Chapter guide partly ready", + detail: "\(count) chapter\(count == 1 ? "" : "s") failed — run again to retry", + systemImage: "exclamationmark.triangle" + ) + } + case .failure(let error): + if case EnrichmentEngine.EngineError.cancelled = error { + enrichmentActivity[id] = nil + } else { + enrichmentActivity[id] = .failed(error.localizedDescription) + } + } + } + + private func loadPersistedEnrichments() { + let candidates = items.map { ($0.id, enrichmentFileURL(for: $0.id)) } + Task.detached(priority: .utility) { [weak self] in + var loaded: [UUID: VideoEnrichment] = [:] + for (id, url) in candidates { + if let enrichment = ChapterEnrichmentLogic.load(from: url) { + loaded[id] = enrichment + } + } + guard !loaded.isEmpty else { return } + let snapshot = loaded + await MainActor.run { [weak self] in + guard let self else { return } + self.enrichments.merge(snapshot) { current, _ in current } + } + } + } + func remove(_ id: UUID, deleteMedia: Bool = true) { cancelRecovery(for: id) downloader.cancel(itemID: id) + enricher.cancel(itemID: id) + enrichments[id] = nil + enrichmentActivity[id] = nil + try? FileManager.default.removeItem(at: enrichmentFileURL(for: id).deletingLastPathComponent()) if deleteMedia { let prefix = id.uuidString + "." let files = (try? FileManager.default.contentsOfDirectory( diff --git a/docs/chapter-enrichment-plan.md b/docs/chapter-enrichment-plan.md new file mode 100644 index 0000000..c4843b2 --- /dev/null +++ b/docs/chapter-enrichment-plan.md @@ -0,0 +1,161 @@ +# Chapter Enrichment: per-chapter summaries + exercises via pi-on-Deno + +> **Status: implemented.** See `Sources/Replay/ChapterEnrichment.swift` +> (models, transcript slicing, prompt, parsing), `Sources/Replay/EnrichmentEngine.swift` +> (deno+pi subprocess orchestration), `QueueStore` (triggering, persistence, +> activity state), the `ChapterSidebar` in `ContentView.swift` (expandable +> rows + footer controls), and `tools/chapter_enrichment_check.swift` (tests, +> wired into `scripts/test.sh`). +> +> **Field notes from implementation:** +> - pi runs fine under the bundled Deno via `deno run -A npm:@earendil-works/pi-coding-agent@0.84.1`. +> - Default provider/model: **openai-codex / gpt-5.6-sol** (verified end-to-end: +> a real chapter enriched in ~18s). Override with `defaults write com.mg.replay +> enrichmentPiProvider/enrichmentPiModel/enrichmentPiPackage ...` or the +> `REPLAY_PI_PROVIDER` / `REPLAY_PI_MODEL` / `REPLAY_PI_PACKAGE` env vars. +> - **Deno version matters:** the Codex provider opens a WebSocket, which +> crashes on Deno < 2.9 (`Cannot assign to read only property +> 'Symbol(Symbol.toStringTag)' of MessageEvent` in undici). The app bundles +> Deno v2.9.5, which works. Dev builds falling back to an older Homebrew +> deno will hit this; `--provider anthropic --model claude-sonnet-4-5` +> (SSE-based) works even on old Deno. +> - pi is run fully hermetic: `-p --no-session -ne --no-skills +> --no-prompt-templates --no-context-files --no-tools` (extensions can open +> WebSockets, which also crash under Deno). +> - Enrichment JSON lives at `Application Support/Replay/Enrichment//enrichment.json` +> with per-chapter prompt/stdout/stderr files beside it for debugging. +> The Deno npm cache is pinned to `Application Support/Replay/DenoCache`. + +## Goal + +For any downloaded video with chapters, generate a **small summary and a few +exercises per chapter**, viewable inline in the chapter sidebar, fully offline +after generation. Modeled on `~/Documents/Notes/Clippings/enrich_clippings_agentic.py` +(pi sub-agent per unit of work, strict output contract), but running pi through +the **Deno binary already bundled in Replay** — no Node install required. + +## What we already have + +| Piece | Where | Notes | +|---|---|---| +| Chapters (title, start, end) | `WatchItem.chapters` via `ChapterMetadata.swift` | populated from yt-dlp info JSON | +| Offline transcript | `WatchItem.subtitleFilePath` + `VideoSubtitles.swift` | VTT cues with start/end times — sliceable per chapter | +| Deno binary | `Resources/Tools/deno` via `DownloadEngine.findTool` | already used for yt-dlp `--js-runtimes` | +| pi under Deno | verified: `deno run -A npm:@earendil-works/pi-coding-agent -p ...` | reads `~/.pi` auth/config; reached the model API in a live test | +| Persistence pattern | `QueueStore` → `queue.json` in Application Support | same dir can hold enrichment JSON per item | + +## Architecture + +``` +User clicks "Enrich chapters" (or auto after download, opt-in) + │ + ▼ +EnrichmentEngine (new, Swift actor — sibling of DownloadEngine) + 1. Load VideoSubtitleTrack, slice cues into per-chapter transcript text + (cue.startTime within [chapter.start, chapter.end)). + 2. For each chapter: build prompt file in a work dir + (Application Support/Enrichment//chapter-03.prompt.md). + 3. Spawn pi subprocess per chapter, bounded concurrency (2–3): + deno run -A npm:@earendil-works/pi-coding-agent@ \ + -p --no-session @chapter-03.prompt.md + with DENO_DIR pinned to Application Support/DenoCache. + 4. Parse strict-format output → ChapterEnrichment. + 5. Persist enrichment.json next to prompts; update WatchItem. + │ + ▼ +ChapterSidebar rows become expandable → summary + exercises (+ solutions +behind a disclosure), with per-chapter status (pending/running/done/failed). +``` + +### New model types + +```swift +struct ChapterEnrichment: Codable, Hashable { + var chapterID: String // VideoChapter.id + var summary: String // 3–5 sentences + var keyPoints: [String] // 2–5 bullets + var exercises: [Exercise] // 2–4 items + var generatedAt: Date + var model: String? +} + +struct Exercise: Codable, Hashable { + var question: String + var solution: String +} + +enum EnrichmentState: Codable { case none, running(progress: Double), ready, failed(String) } +``` + +`WatchItem` gains `enrichmentFilePath: String?` (mirrors `subtitleFilePath`); +the enrichment JSON itself lives in Application Support so `queue.json` stays +small. Cached results are reused; a "Regenerate" action forces re-run +(the `--force` idea from the clippings script). + +### Prompt contract (per chapter) + +Inputs: video title/author, chapter title + index + duration, the chapter's +transcript slice (truncated to a token budget, e.g. ~12k chars, head+tail), +and the list of all chapter titles for context. + +Output: **JSON only**, fenced, matching `ChapterEnrichment` fields — same +"hard rules / required shape" style as `markdown_prompt_for_link` in the +clippings script, but JSON is easier to parse in Swift than delimited +Markdown. Anti-hallucination rule: only claim what the transcript supports. + +Run pi with tools disabled (`-ne` equivalent) since no browsing is needed — +faster, cheaper, deterministic. Note from the clippings script: `-ne` also +disables auth extensions; if the user's provider needs one (e.g. +`pi-anthropic-auth`), re-enable it explicitly via `-e`, or skip `-ne` and rely +on the prompt to forbid tool use. + +### Videos without chapters / without subtitles + +- **No chapters:** offer a single whole-video summary + exercises (treat the + video as one chapter). Optionally later: ask pi to *propose* chapter splits + from the transcript. +- **No subtitles:** disable the feature for that item with an explanatory + tooltip (v1). Later option: whisper/ffmpeg transcription. + +## Key decisions & risks + +1. **Pinned package version.** `npm:@earendil-works/pi-coding-agent@X.Y.Z` + pinned in code; first run downloads to `DENO_DIR` (needs network once). + Optionally pre-warm the cache right after a download completes. +2. **Auth.** pi reads `~/.pi`. If no provider is configured, fail fast with a + friendly "Set up pi first" message (check by running `pi --version`-style + probe or catching the auth error). Replay itself stores no keys in v1. +3. **Cost/latency.** One model call per chapter; a 20-chapter video = 20 calls. + Mitigations: bounded concurrency, per-chapter caching (re-runs only + failures), and a batched mode (all chapters in one call) for videos under + ~15 min of transcript. +4. **Sandboxing.** Replay already shells out to yt-dlp/deno with network, so + no new entitlement work expected. +5. **Output parsing.** Reuse the clippings script's lesson: strip fenced + wrapper, tolerate minor deviations, write `.stderr.txt` beside outputs for + debugging, never crash the queue on a bad chapter — mark it `failed`. + +## Milestones + +1. **Spike (throwaway script)** — `scripts/enrich_chapters_spike.sh`: take an + existing downloaded item's VTT + chapters JSON, produce per-chapter + enrichment via bundled deno + pi. Validates prompt, output shape, timing, + cost. *(~half day)* +2. **EnrichmentEngine + models** — Swift actor, subtitle slicing, subprocess + management, JSON persistence, caching. Unit-check harness in `tools/` + (`chapter_enrichment_check.swift`, matching the existing check style) for + slicing + parsing logic. *(~1 day)* +3. **UI** — expandable chapter rows in `ChapterSidebar`, status indicators, + "Enrich chapters" / "Regenerate" actions, error surfacing. *(~1 day)* +4. **Polish** — settings toggle (auto-enrich after download), concurrency + limits, cancellation when item is deleted, Low Power Mode pause parity + with downloads. *(~half day)* + +## Open questions + +- Auto-enrich after every download, or manual-only in v1? (Suggest manual.) +- Show exercises inline in the sidebar, or a dedicated "Study" pane per video? +- Should solutions be generated up front (offline-first) or on demand? + (Suggest up front — one call anyway.) +- Model choice: pi's default, or a cheap/fast override baked into the prompt + command (`--model`)? diff --git a/scripts/test.sh b/scripts/test.sh index 9a19403..121a406 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -29,6 +29,12 @@ compile_and_run subtitle_parser \ "$project_dir/Sources/Replay/VideoSubtitles.swift" \ "$project_dir/tools/subtitle_parser_check.swift" +compile_and_run chapter_enrichment \ + "$project_dir/Sources/Replay/WatchItem.swift" \ + "$project_dir/Sources/Replay/VideoSubtitles.swift" \ + "$project_dir/Sources/Replay/ChapterEnrichment.swift" \ + "$project_dir/tools/chapter_enrichment_check.swift" + compile_and_run power_mode \ "$project_dir/Sources/Replay/PowerModeMonitor.swift" \ "$project_dir/tools/power_mode_check.swift" diff --git a/tools/chapter_enrichment_check.swift b/tools/chapter_enrichment_check.swift new file mode 100644 index 0000000..6789fe7 --- /dev/null +++ b/tools/chapter_enrichment_check.swift @@ -0,0 +1,264 @@ +import Foundation + +@main +struct ChapterEnrichmentCheck { + static func main() { + checkEndTimeResolution() + checkTranscriptSlicing() + checkTruncation() + checkJSONExtraction() + checkOutputParsing() + checkErrorSummary() + checkMathRendering() + checkRevisionPrompt() + checkPersistenceRoundTrip() + print("chapter_enrichment_check=passed") + } + + static let chapters = [ + VideoChapter(title: "Intro", startTime: 0, endTime: nil), + VideoChapter(title: "Main", startTime: 10, endTime: nil), + VideoChapter(title: "Outro", startTime: 20, endTime: nil) + ] + + static let cues = [ + VideoSubtitleCue(startTime: 0, endTime: 4, text: "welcome to the video"), + VideoSubtitleCue(startTime: 4, endTime: 9, text: "today we cover queues"), + VideoSubtitleCue(startTime: 9, endTime: 12, text: "first a definition"), + VideoSubtitleCue(startTime: 12, endTime: 15, text: "first a definition"), + VideoSubtitleCue(startTime: 15, endTime: 19, text: "a queue is FIFO"), + VideoSubtitleCue(startTime: 21, endTime: 25, text: "thanks for watching") + ] + + static func checkEndTimeResolution() { + precondition(ChapterEnrichmentLogic.endTime(for: chapters[0], in: chapters, videoDuration: 30) == 10) + precondition(ChapterEnrichmentLogic.endTime(for: chapters[2], in: chapters, videoDuration: 30) == 30) + precondition(ChapterEnrichmentLogic.endTime(for: chapters[2], in: chapters, videoDuration: nil) == .greatestFiniteMagnitude) + let explicit = VideoChapter(title: "X", startTime: 5, endTime: 8) + precondition(ChapterEnrichmentLogic.endTime(for: explicit, in: [explicit], videoDuration: 30) == 8) + } + + static func checkTranscriptSlicing() { + let intro = ChapterEnrichmentLogic.transcript(for: chapters[0], in: chapters, cues: cues, videoDuration: 30) + // The 9-12s cue overlaps the 0-10s chapter boundary and is included. + precondition(intro == "welcome to the video today we cover queues first a definition") + + let main = ChapterEnrichmentLogic.transcript(for: chapters[1], in: chapters, cues: cues, videoDuration: 30) + // Consecutive duplicate cue text is collapsed once. + precondition(main == "first a definition a queue is FIFO") + + let outro = ChapterEnrichmentLogic.transcript(for: chapters[2], in: chapters, cues: cues, videoDuration: 30) + precondition(outro == "thanks for watching") + } + + static func checkTruncation() { + let short = ChapterEnrichmentLogic.truncateMiddle("hello", limit: 100) + precondition(short == "hello") + let long = String(repeating: "a", count: 500) + String(repeating: "z", count: 500) + let truncated = ChapterEnrichmentLogic.truncateMiddle(long, limit: 300) + precondition(truncated.contains("[... transcript truncated ...]")) + precondition(truncated.hasPrefix("aaa")) + precondition(truncated.hasSuffix("zzz")) + precondition(truncated.count < long.count) + } + + static func checkJSONExtraction() { + let fenced = """ + Here is the result: + ```json + {"summary": "S", "keyPoints": ["a {brace} inside"], "exercises": []} + ``` + Done. + """ + let extracted = ChapterEnrichmentLogic.extractJSONObject(from: fenced) + precondition(extracted == "{\"summary\": \"S\", \"keyPoints\": [\"a {brace} inside\"], \"exercises\": []}") + + let withEscapes = "{\"summary\": \"quote \\\" and brace } in string\"}" + precondition(ChapterEnrichmentLogic.extractJSONObject(from: withEscapes) == withEscapes) + + precondition(ChapterEnrichmentLogic.extractJSONObject(from: "no json here") == nil) + precondition(ChapterEnrichmentLogic.extractJSONObject(from: "{ unbalanced") == nil) + } + + static func checkOutputParsing() { + let output = """ + { + "summary": " The chapter defines FIFO queues. ", + "keyPoints": ["Queues are FIFO", " ", "Stacks are LIFO"], + "exercises": [ + {"question": "What does FIFO mean?", "solution": "First in, first out."}, + {"question": " ", "solution": "ignored"}, + {"question": "Name the opposite discipline.", "solution": null} + ] + } + """ + guard let parsed = ChapterEnrichmentLogic.parse(output: output, chapterID: "10.0-Main", chapterTitle: "Main") else { + preconditionFailure("Expected parseable output") + } + precondition(parsed.chapterID == "10.0-Main") + precondition(parsed.summary == "The chapter defines FIFO queues.") + precondition(parsed.keyPoints == ["Queues are FIFO", "Stacks are LIFO"]) + precondition(parsed.exercises.count == 2) + precondition(parsed.exercises[0].solution == "First in, first out.") + precondition(parsed.exercises[1].solution.isEmpty) + + precondition(ChapterEnrichmentLogic.parse(output: "not json", chapterID: "x", chapterTitle: "X") == nil) + precondition(ChapterEnrichmentLogic.parse(output: "{\"summary\": \" \"}", chapterID: "x", chapterTitle: "X") == nil) + + // Prompt sanity: contains the transcript and demands JSON. + let prompt = ChapterEnrichmentLogic.prompt( + videoTitle: "T", + videoAuthor: "", + chapter: chapters[1], + chapterIndex: 1, + chapterCount: 3, + allChapterTitles: chapters.map(\.title), + transcript: "TRANSCRIPT-SENTINEL" + ) + precondition(prompt.contains("TRANSCRIPT-SENTINEL")) + precondition(prompt.contains("Chapter 2 of 3: Main")) + precondition(prompt.contains("single JSON object")) + precondition(prompt.contains("Author: unknown")) + } + + static func checkErrorSummary() { + let providerError = """ + Warning: client version mismatch + 400 {"type":"error","error":{"type":"invalid_request_error","message":"You're out of extra usage. Add more at claude.ai/settings/usage and keep going."},"request_id":"req_x"} + """ + let summary = ChapterEnrichmentLogic.errorSummary(fromStderr: providerError) + precondition(summary == "You're out of extra usage. Add more at claude.ai/settings/usage and keep going.") + + let crash = """ + Warning: something + error: Uncaught TypeError: Cannot assign to read only property + at new Event (ext:deno_web/02_event.js:138:29) + """ + precondition(ChapterEnrichmentLogic.errorSummary(fromStderr: crash) == "error: Uncaught TypeError: Cannot assign to read only property") + + precondition(ChapterEnrichmentLogic.errorSummary(fromStderr: "") == nil) + precondition(ChapterEnrichmentLogic.errorSummary(fromStderr: "Warning: only warnings\n") == nil) + } + + static func checkMathRendering() { + let render = ChapterEnrichmentLogic.displayText + + // Plain text and dollar amounts are untouched. + precondition(render("no math here") == "no math here") + precondition(render("costs $2,000 and $5 per run") == "costs $2,000 and $5 per run") + precondition(render("a batch of $2,000 tokens") == "a batch of $2,000 tokens") + + // Superscripts, subscripts, Greek, operators. + precondition(render("needs $2^{10}$ tokens") == "needs 2\u{00B9}\u{2070} tokens") + precondition(render("$x_i$ and $x_{max}$") == "x\u{1D62} and x\u{2098}\u{2090}\u{2093}") + precondition(render("$x_{qd}$") == "x_qd") + precondition(render("$\\alpha \\cdot \\beta$") == "\u{03B1} \u{00B7} \u{03B2}") + precondition(render("$a \\times b \\leq c$") == "a \u{00D7} b \u{2264} c") + precondition(render("$O(n \\log n)$") == "O(n log n)") + + // Fractions, roots, text unwrapping. + precondition(render("$\\frac{a}{b}$") == "a/b") + precondition(render("$\\frac{a+b}{2}$") == "(a+b)/2") + precondition(render("$\\sqrt{2}$") == "\u{221A}2") + precondition(render("$\\text{cost} = 4$") == "cost = 4") + + // \( \) and $$ $$ delimiters. + precondition(render("then \\(k^2\\) grows") == "then k\u{00B2} grows") + precondition(render("$$E = mc^2$$") == "E = mc\u{00B2}") + + // Unmappable scripts degrade gracefully and never loop. + precondition(render("$W^{Q}$") == "W^Q") + precondition(render("$x^T W_q$") == "x\u{1D40} W_q") + + // Single-token algebra like $n$ is treated as math (delimiters removed). + precondition(render("pick $n$ samples") == "pick n samples") + } + + static func checkRevisionPrompt() { + let previous = ChapterEnrichment( + chapterID: "10.0-Main", + chapterTitle: "Main", + summary: "Old summary", + keyPoints: ["old point"], + exercises: [ChapterExercise(question: "old q", solution: "old a")], + generatedAt: Date() + ) + let prompt = ChapterEnrichmentLogic.prompt( + videoTitle: "T", + videoAuthor: "A", + chapter: chapters[1], + chapterIndex: 1, + chapterCount: 3, + allChapterTitles: chapters.map(\.title), + transcript: "TRANSCRIPT", + previous: previous, + guidance: "add more exercises" + ) + precondition(prompt.contains("USER GUIDANCE (HIGH PRIORITY)")) + precondition(prompt.contains("add more exercises")) + precondition(prompt.contains("PREVIOUS VERSION OF THIS CHAPTER'S GUIDE")) + precondition(prompt.contains("Old summary")) + precondition(prompt.contains("old q")) + + // Without guidance/previous, no revision blocks appear. + let plain = ChapterEnrichmentLogic.prompt( + videoTitle: "T", + videoAuthor: "A", + chapter: chapters[1], + chapterIndex: 1, + chapterCount: 3, + allChapterTitles: chapters.map(\.title), + transcript: "TRANSCRIPT" + ) + precondition(!plain.contains("USER GUIDANCE")) + precondition(!plain.contains("PREVIOUS VERSION")) + + // The previous-version JSON matches the model output schema. + let json = ChapterEnrichmentLogic.previousVersionJSON(previous) + let reparsed = ChapterEnrichmentLogic.parse(output: json, chapterID: "x", chapterTitle: "X") + precondition(reparsed?.summary == "Old summary") + precondition(reparsed?.exercises.first?.question == "old q") + } + + static func checkPersistenceRoundTrip() { + let enrichment = VideoEnrichment( + version: VideoEnrichment.currentVersion, + itemID: UUID(), + generatedAt: Date(), + chapters: [ + ChapterEnrichment( + chapterID: "0.0-Intro", + chapterTitle: "Intro", + summary: "S", + keyPoints: ["k"], + exercises: [ChapterExercise(question: "q", solution: "a")], + generatedAt: Date() + ) + ] + ) + let dir = FileManager.default.temporaryDirectory + .appendingPathComponent("enrichment-check-\(UUID().uuidString)") + let file = dir.appendingPathComponent("enrichment.json") + defer { try? FileManager.default.removeItem(at: dir) } + + try! ChapterEnrichmentLogic.save(enrichment, to: file) + guard let loaded = ChapterEnrichmentLogic.load(from: file) else { + preconditionFailure("Expected round-trip load") + } + precondition(loaded.itemID == enrichment.itemID) + // ISO8601 encoding drops sub-second precision, so compare content fields. + precondition(loaded.chapters.map(\.chapterID) == enrichment.chapters.map(\.chapterID)) + precondition(loaded.chapters.map(\.exercises) == enrichment.chapters.map(\.exercises)) + precondition(loaded.chapters.map(\.keyPoints) == enrichment.chapters.map(\.keyPoints)) + precondition(loaded.enrichment(forChapterID: "0.0-Intro")?.summary == "S") + precondition(loaded.enrichment(forChapterID: "missing") == nil) + + // Unknown versions are rejected rather than misread. + var future = enrichment + future.version = 99 + try! ChapterEnrichmentLogic.save(future, to: file) + precondition(ChapterEnrichmentLogic.load(from: file) == nil) + + precondition(ChapterEnrichmentLogic.load(from: dir.appendingPathComponent("absent.json")) == nil) + } +} From a9a0bfb5a7c6b32aa070f5e4c7bd850b950406d8 Mon Sep 17 00:00:00 2001 From: swairshah Date: Thu, 13 Aug 2026 19:39:10 -0700 Subject: [PATCH 3/9] feat(chapters): add adjustable guide text size Add persistent minus and plus controls for chapter titles, timestamps, summaries, exercises, and solutions. --- Sources/Replay/ContentView.swift | 118 +++++++++++++++++++++++++------ 1 file changed, 96 insertions(+), 22 deletions(-) diff --git a/Sources/Replay/ContentView.swift b/Sources/Replay/ContentView.swift index 4ff2044..f2a1db1 100644 --- a/Sources/Replay/ContentView.swift +++ b/Sources/Replay/ContentView.swift @@ -1766,6 +1766,14 @@ private struct ChapterSidebar: View { @State private var revealedSolutions: Set = [] @State private var regeneratePopoverShown = false @State private var regenerateGuidance = "" + @AppStorage("chapterSidebarFontSize") private var sidebarFontSize = 14.5 + + private enum SidebarType { + static let defaultSize = 14.5 + static let minimumSize = 11.5 + static let maximumSize = 22.5 + static let step = 1.0 + } var body: some View { VStack(alignment: .leading, spacing: 0) { @@ -1795,6 +1803,10 @@ private struct ChapterSidebar: View { Divider() + fontSizeControls + + Divider() + ScrollView { LazyVStack(alignment: .leading, spacing: 3) { ForEach(chapters) { chapter in @@ -1814,6 +1826,54 @@ private struct ChapterSidebar: View { .background(.ultraThinMaterial) } + private var fontSizeControls: some View { + HStack(spacing: 8) { + Text("Text size") + .font(.caption) + .foregroundStyle(.secondary) + + Spacer(minLength: 8) + + fontSizeButton("−", help: "Decrease sidebar text size", disabled: sidebarFontSize <= SidebarType.minimumSize) { + sidebarFontSize = max(SidebarType.minimumSize, sidebarFontSize - SidebarType.step) + } + + Text("\(Int(sidebarFontSize.rounded()))") + .font(.caption.monospacedDigit()) + .foregroundStyle(.secondary) + .frame(width: 24) + + fontSizeButton("+", help: "Increase sidebar text size", disabled: sidebarFontSize >= SidebarType.maximumSize) { + sidebarFontSize = min(SidebarType.maximumSize, sidebarFontSize + SidebarType.step) + } + } + .padding(.horizontal, 12) + .frame(height: 46) + } + + private func fontSizeButton( + _ label: String, + help: String, + disabled: Bool, + action: @escaping () -> Void + ) -> some View { + Button { + withAnimation(.easeOut(duration: 0.15)) { action() } + } label: { + Text(label) + .font(.system(size: 18, weight: .medium, design: .rounded)) + .frame(width: 40, height: 40) + .contentShape(Rectangle()) + .background( + Color.primary.opacity(disabled ? 0.035 : 0.075), + in: RoundedRectangle(cornerRadius: 9, style: .continuous) + ) + } + .buttonStyle(.plain) + .disabled(disabled) + .help(help) + } + @ViewBuilder private func chapterRow(_ chapter: VideoChapter) -> some View { let chapterEnrichment = enrichment?.enrichment(forChapterID: chapter.id) @@ -1823,9 +1883,9 @@ private struct ChapterSidebar: View { select(chapter) } label: { Text(formatTime(chapter.startTime)) - .font(.caption.monospacedDigit()) + .font(.system(size: max(10, sidebarFontSize - 2), design: .monospaced)) .foregroundStyle(isCurrent(chapter) ? Color.accentColor : Color.secondary) - .frame(width: 48, alignment: .trailing) + .frame(width: timestampColumnWidth, alignment: .trailing) .contentShape(Rectangle()) } .buttonStyle(.plain) @@ -1872,32 +1932,46 @@ private struct ChapterSidebar: View { } } + private var timestampColumnWidth: CGFloat { + 48 + max(0, sidebarFontSize - SidebarType.defaultSize) * 1.5 + } + private func chapterTitle(_ chapter: VideoChapter) -> some View { Text(chapter.title) - .font(.callout.weight(isCurrent(chapter) ? .semibold : .regular)) + .font(.system(size: sidebarFontSize, weight: isCurrent(chapter) ? .semibold : .regular)) .foregroundStyle(.primary) .multilineTextAlignment(.leading) - .lineLimit(2) + .lineLimit(3) .fixedSize(horizontal: false, vertical: true) } /// Reading typography for the chapter guide, modeled on Clearly's /// preview: a serif reading face (New York on macOS), comfortable size, - /// and ~1.7 line height. Line spacing below is (lineHeight - 1) × size. - private enum GuideType { - static let body = Font.system(size: 14.5, weight: .regular, design: .serif) - static let bodyLineSpacing: CGFloat = 14.5 * 0.7 - static let question = Font.system(size: 14.5, weight: .medium, design: .serif) - static let sectionLabel = Font.system(size: 10.5, weight: .semibold) - static let ordinal = Font.system(size: 12, weight: .semibold, design: .rounded) + /// and ~1.7 line height. Every value follows the persisted sidebar size. + private var guideBodyFont: Font { + .system(size: sidebarFontSize, weight: .regular, design: .serif) + } + + private var guideQuestionFont: Font { + .system(size: sidebarFontSize, weight: .medium, design: .serif) + } + + private var guideLineSpacing: CGFloat { sidebarFontSize * 0.7 } + + private var guideSectionLabelFont: Font { + .system(size: max(9.5, sidebarFontSize - 4), weight: .semibold) + } + + private var guideOrdinalFont: Font { + .system(size: max(10, sidebarFontSize - 2.5), weight: .semibold, design: .rounded) } private func enrichmentDetail(_ chapterEnrichment: ChapterEnrichment) -> some View { VStack(alignment: .leading, spacing: 14) { Text(ChapterEnrichmentLogic.displayText(chapterEnrichment.summary)) - .font(GuideType.body) + .font(guideBodyFont) .foregroundStyle(.primary) - .lineSpacing(GuideType.bodyLineSpacing) + .lineSpacing(guideLineSpacing) .fixedSize(horizontal: false, vertical: true) .textSelection(.enabled) @@ -1910,9 +1984,9 @@ private struct ChapterSidebar: View { .frame(width: 4, height: 4) .alignmentGuide(.firstTextBaseline) { $0[VerticalAlignment.center] + 5 } Text(ChapterEnrichmentLogic.displayText(point)) - .font(GuideType.body) + .font(guideBodyFont) .foregroundStyle(.primary.opacity(0.85)) - .lineSpacing(GuideType.bodyLineSpacing * 0.75) + .lineSpacing(guideLineSpacing * 0.75) .fixedSize(horizontal: false, vertical: true) .textSelection(.enabled) } @@ -1923,7 +1997,7 @@ private struct ChapterSidebar: View { if !chapterEnrichment.exercises.isEmpty { VStack(alignment: .leading, spacing: 12) { Text("EXERCISES") - .font(GuideType.sectionLabel) + .font(guideSectionLabelFont) .kerning(1.1) .foregroundStyle(.secondary) .padding(.top, 4) @@ -1940,16 +2014,16 @@ private struct ChapterSidebar: View { return VStack(alignment: .leading, spacing: 8) { HStack(alignment: .firstTextBaseline, spacing: 9) { Text("\(index + 1)") - .font(GuideType.ordinal) + .font(guideOrdinalFont) .monospacedDigit() .foregroundStyle(.secondary) .frame(width: 18, height: 18) .background(Color.secondary.opacity(0.12), in: Circle()) .alignmentGuide(.firstTextBaseline) { $0[VerticalAlignment.center] + 5 } Text(ChapterEnrichmentLogic.displayText(exercise.question)) - .font(GuideType.question) + .font(guideQuestionFont) .foregroundStyle(.primary) - .lineSpacing(GuideType.bodyLineSpacing * 0.75) + .lineSpacing(guideLineSpacing * 0.75) .fixedSize(horizontal: false, vertical: true) .textSelection(.enabled) } @@ -1960,13 +2034,13 @@ private struct ChapterSidebar: View { toggleSolution(solutionKey) } .buttonStyle(.plain) - .font(.callout.weight(.medium)) + .font(.system(size: sidebarFontSize, weight: .medium)) .foregroundStyle(Color.accentColor) if isRevealed { Text(ChapterEnrichmentLogic.displayText(exercise.solution)) - .font(GuideType.body) + .font(guideBodyFont) .foregroundStyle(.primary.opacity(0.75)) - .lineSpacing(GuideType.bodyLineSpacing * 0.75) + .lineSpacing(guideLineSpacing * 0.75) .fixedSize(horizontal: false, vertical: true) .textSelection(.enabled) .padding(.leading, 12) From 5fd0a2717f6743a3a42fdd65d4b134ce9e538ef6 Mon Sep 17 00:00:00 2001 From: swairshah Date: Thu, 13 Aug 2026 19:39:27 -0700 Subject: [PATCH 4/9] fix(chapters): require applied technical exercises Prompt technical talks with implementation, derivation, debugging, experiment, and design tasks. Reject recall-only questions and incomplete exercise sets during parsing. --- Sources/Replay/ChapterEnrichment.swift | 56 +++++++++++++++++++++----- docs/chapter-enrichment-plan.md | 6 ++- tools/chapter_enrichment_check.swift | 29 +++++++++---- 3 files changed, 74 insertions(+), 17 deletions(-) diff --git a/Sources/Replay/ChapterEnrichment.swift b/Sources/Replay/ChapterEnrichment.swift index 42c8a8a..4e371b3 100644 --- a/Sources/Replay/ChapterEnrichment.swift +++ b/Sources/Replay/ChapterEnrichment.swift @@ -150,21 +150,26 @@ enum ChapterEnrichmentLogic { "keyPoints": ["2-5 short bullet strings with the concrete takeaways"], "exercises": [ { - "question": "a specific exercise or comprehension question", - "solution": "a concise but complete solution or model answer" + "question": "a concrete application or problem-solving task", + "solution": "a worked, technically precise solution or model answer" } ] } - Rules: - - 2 to 4 exercises, ordered easiest to hardest. - - Exercises must be answerable from this chapter's content alone. - - Prefer concrete recall/application questions over vague reflection. + Exercise quality rules (HIGH PRIORITY): + - 2 to 4 exercises, ordered from a focused application to a harder synthesis or design task. + - NEVER ask the learner to recall the talk: no "what did the speaker say", "list/name/identify", "according to the speaker", or "describe the procedure" questions. + - Every exercise must require the learner to DO something with the chapter's ideas, not merely restate them. + - Silently classify the chapter before writing exercises. If it contains algorithms, code, mathematics, model architectures, training objectives, benchmarks, experimental methodology, systems, scientific mechanisms, or engineering trade-offs, treat it as TECHNICAL. + - For a TECHNICAL chapter, every exercise must be technical. Use tasks such as: trace a concrete example; derive or compute a result; write pseudocode; design an experiment or ablation; diagnose a failure; predict behavior after changing a variable; compare methods under a stated constraint; critique a benchmark; or design a system using the mechanism taught. + - Ground technical exercises in the actual named methods, quantities, equations, objectives, data flows, or failure modes in this transcript. Include enough scenario detail that the task can be solved without rewatching the talk. + - When the transcript lacks an explicit numeric example, invent only harmless exercise inputs or hypothetical scenarios; do not invent factual claims about the talk. + - Solutions should show the reasoning or intermediate steps, not just state the answer. + - For nontechnical material, still require transfer, analysis, or decision-making—never speaker recall. + - Exercises must be answerable using this chapter's content alone. - Mathematical notation is welcome where it helps: use simple inline \ LaTeX delimited by $...$ (for example $2^{10}$ or $\\frac{a}{b}$). \ Keep it simple; prefer plain text when math adds nothing. - - If the transcript contains numbers, formulas, code, or named \ - techniques, use them in the exercises. - Keep the summary specific: name the actual ideas, not just topics. - Valid JSON only: escape newlines inside strings as \\n. @@ -220,10 +225,15 @@ enum ChapterEnrichmentLogic { .filter { !$0.isEmpty } let exercises = (decoded.exercises ?? []).compactMap { exercise -> ChapterExercise? in let question = exercise.question.trimmingCharacters(in: .whitespacesAndNewlines) - guard !question.isEmpty else { return nil } + guard !question.isEmpty, !isRecallOnlyExercise(question) else { return nil } let solution = (exercise.solution ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + guard !solution.isEmpty else { return nil } return ChapterExercise(question: question, solution: solution) } + // It is better to retry a chapter than persist a quiz made entirely of + // recall prompts. The model contract requires at least two substantive + // exercises, and this also catches truncated output. + guard exercises.count >= 2 else { return nil } return ChapterEnrichment( chapterID: chapterID, chapterTitle: chapterTitle, @@ -234,6 +244,34 @@ enum ChapterEnrichmentLogic { ) } + /// Rejects obvious lecture-recall prompts. This is deliberately narrow: + /// causal "why" questions can still demand real understanding, while + /// speaker-reporting, enumeration, and bare fact questions cannot. + static func isRecallOnlyExercise(_ question: String) -> Bool { + let normalized = question + .lowercased() + .trimmingCharacters(in: .whitespacesAndNewlines) + let appliedSignals = [ + "given ", "suppose ", "consider ", "scenario", "would ", + "calculate", "compute", "derive", "design", "predict", "diagnose", + "debug", "trace", "construct", "implement", "pseudocode", "ablation", + "experiment", "compare", "evaluate", "critique", "modify", "trade-off" + ] + if appliedSignals.contains(where: normalized.contains) { return false } + + let speakerRecall = [ + "did the speaker", "does the speaker", "speaker's view", "speaker’s view", + "according to the speaker", "the speaker recommend", "the speaker mean" + ] + if speakerRecall.contains(where: normalized.contains) { return true } + + let recallPrefixes = [ + "what ", "who ", "when ", "where ", "which ", "list ", "name ", + "identify ", "state ", "recall ", "describe the ", "summarize the " + ] + return recallPrefixes.contains(where: normalized.hasPrefix) + } + /// Extracts the first balanced top-level JSON object from mixed output, /// respecting string literals and escapes. static func extractJSONObject(from text: String) -> String? { diff --git a/docs/chapter-enrichment-plan.md b/docs/chapter-enrichment-plan.md index c4843b2..da3f6c5 100644 --- a/docs/chapter-enrichment-plan.md +++ b/docs/chapter-enrichment-plan.md @@ -96,7 +96,11 @@ small. Cached results are reused; a "Regenerate" action forces re-run Inputs: video title/author, chapter title + index + duration, the chapter's transcript slice (truncated to a token budget, e.g. ~12k chars, head+tail), -and the list of all chapter titles for context. +and the list of all chapter titles for context. The guide prompt classifies +technical chapters and requires every exercise to apply the material through +tracing, derivation, implementation, debugging, experiment design, benchmark +critique, prediction, or engineering trade-offs. Speaker-recall and bare +listing questions are forbidden and filtered during parsing. Output: **JSON only**, fenced, matching `ChapterEnrichment` fields — same "hard rules / required shape" style as `markdown_prompt_for_link` in the diff --git a/tools/chapter_enrichment_check.swift b/tools/chapter_enrichment_check.swift index 6789fe7..b4be8af 100644 --- a/tools/chapter_enrichment_check.swift +++ b/tools/chapter_enrichment_check.swift @@ -86,9 +86,9 @@ struct ChapterEnrichmentCheck { "summary": " The chapter defines FIFO queues. ", "keyPoints": ["Queues are FIFO", " ", "Stacks are LIFO"], "exercises": [ - {"question": "What does FIFO mean?", "solution": "First in, first out."}, + {"question": "Trace a FIFO queue through enqueue(A), enqueue(B), and dequeue().", "solution": "The queue becomes [A], then [A, B]; dequeue returns A and leaves [B]."}, {"question": " ", "solution": "ignored"}, - {"question": "Name the opposite discipline.", "solution": null} + {"question": "Given jobs J1, J2, and J3 arriving in that order, compare FIFO execution with a LIFO stack.", "solution": "FIFO runs J1, J2, J3; LIFO runs J3, J2, J1, so the disciplines reverse the service order."} ] } """ @@ -99,8 +99,20 @@ struct ChapterEnrichmentCheck { precondition(parsed.summary == "The chapter defines FIFO queues.") precondition(parsed.keyPoints == ["Queues are FIFO", "Stacks are LIFO"]) precondition(parsed.exercises.count == 2) - precondition(parsed.exercises[0].solution == "First in, first out.") - precondition(parsed.exercises[1].solution.isEmpty) + precondition(parsed.exercises[0].solution.contains("dequeue returns A")) + precondition(parsed.exercises[1].solution.contains("reverse the service order")) + + let recallOnly = """ + {"summary":"S","exercises":[ + {"question":"What two capabilities did the speaker identify?","solution":"A and B."}, + {"question":"Describe the benchmark procedure.","solution":"Adapt and average."} + ]} + """ + precondition(ChapterEnrichmentLogic.parse(output: recallOnly, chapterID: "x", chapterTitle: "X") == nil) + precondition(ChapterEnrichmentLogic.isRecallOnlyExercise("According to the speaker, what is VTAB?")) + precondition(ChapterEnrichmentLogic.isRecallOnlyExercise("List the stages of CLIP training.")) + precondition(!ChapterEnrichmentLogic.isRecallOnlyExercise("Given a 3×3 similarity matrix, compute the CLIP loss.")) + precondition(!ChapterEnrichmentLogic.isRecallOnlyExercise("Design an ablation that isolates the effect of captioning loss.")) precondition(ChapterEnrichmentLogic.parse(output: "not json", chapterID: "x", chapterTitle: "X") == nil) precondition(ChapterEnrichmentLogic.parse(output: "{\"summary\": \" \"}", chapterID: "x", chapterTitle: "X") == nil) @@ -180,7 +192,10 @@ struct ChapterEnrichmentCheck { chapterTitle: "Main", summary: "Old summary", keyPoints: ["old point"], - exercises: [ChapterExercise(question: "old q", solution: "old a")], + exercises: [ + ChapterExercise(question: "Design an experiment using the old idea.", solution: "Run the experiment and compare outcomes."), + ChapterExercise(question: "Given an edge case, diagnose the old method.", solution: "Trace the edge case and identify the failure.") + ], generatedAt: Date() ) let prompt = ChapterEnrichmentLogic.prompt( @@ -198,7 +213,7 @@ struct ChapterEnrichmentCheck { precondition(prompt.contains("add more exercises")) precondition(prompt.contains("PREVIOUS VERSION OF THIS CHAPTER'S GUIDE")) precondition(prompt.contains("Old summary")) - precondition(prompt.contains("old q")) + precondition(prompt.contains("Design an experiment using the old idea.")) // Without guidance/previous, no revision blocks appear. let plain = ChapterEnrichmentLogic.prompt( @@ -217,7 +232,7 @@ struct ChapterEnrichmentCheck { let json = ChapterEnrichmentLogic.previousVersionJSON(previous) let reparsed = ChapterEnrichmentLogic.parse(output: json, chapterID: "x", chapterTitle: "X") precondition(reparsed?.summary == "Old summary") - precondition(reparsed?.exercises.first?.question == "old q") + precondition(reparsed?.exercises.first?.question == "Design an experiment using the old idea.") } static func checkPersistenceRoundTrip() { From 7aa4f7f8b61fd9d7a7967969969276cddd0a3943 Mon Sep 17 00:00:00 2001 From: swairshah Date: Thu, 13 Aug 2026 19:39:53 -0700 Subject: [PATCH 5/9] feat(chapters): generate guides without source chapters Use a dedicated pi subprocess to derive semantic, cue-aligned chapter outlines from subtitles. Persist generated chapters, expose them throughout playback and the sidebar, and fall back to deterministic sections when planning fails. --- Sources/Replay/ChapterEnrichment.swift | 211 +++++++++++++++++++++++++ Sources/Replay/ContentView.swift | 94 ++++++++--- Sources/Replay/EnrichmentEngine.swift | 135 ++++++++++++---- Sources/Replay/QueueStore.swift | 13 +- docs/chapter-enrichment-plan.md | 28 ++-- tools/chapter_enrichment_check.swift | 81 ++++++++++ 6 files changed, 497 insertions(+), 65 deletions(-) diff --git a/Sources/Replay/ChapterEnrichment.swift b/Sources/Replay/ChapterEnrichment.swift index 4e371b3..8f39b93 100644 --- a/Sources/Replay/ChapterEnrichment.swift +++ b/Sources/Replay/ChapterEnrichment.swift @@ -23,6 +23,24 @@ struct VideoEnrichment: Codable, Hashable { var itemID: UUID var generatedAt: Date var chapters: [ChapterEnrichment] + /// Present only when the source video had no creator-authored chapters. + /// Optional so guides written before generated outlines were introduced + /// continue to decode. + var generatedChapters: [VideoChapter]? + + init( + version: Int, + itemID: UUID, + generatedAt: Date, + chapters: [ChapterEnrichment], + generatedChapters: [VideoChapter]? = nil + ) { + self.version = version + self.itemID = itemID + self.generatedAt = generatedAt + self.chapters = chapters + self.generatedChapters = generatedChapters + } func enrichment(forChapterID chapterID: String) -> ChapterEnrichment? { chapters.first { $0.chapterID == chapterID } @@ -37,6 +55,10 @@ enum ChapterEnrichmentLogic { /// transcripts keep the head and tail, which usually carry the chapter's /// setup and conclusion. static let transcriptCharacterBudget = 14_000 + static let chapterPlanningCharacterBudget = 80_000 + static let generatedChapterTargetDuration: Double = 6 * 60 + static let generatedChapterMinimumDuration: Double = 75 + static let generatedChapterMaximumDuration: Double = 12 * 60 // MARK: Transcript slicing @@ -86,6 +108,195 @@ enum ChapterEnrichmentLogic { return "\(head)\n[... transcript truncated ...]\n\(tail)" } + // MARK: Generated chapter planning + + /// Produces a timestamped, evenly sampled transcript for the chapter + /// planner. Sampling across the entire video is preferable to truncating + /// the middle because topic boundaries can occur anywhere. + static func timestampedTranscript( + cues: [VideoSubtitleCue], + limit: Int = chapterPlanningCharacterBudget + ) -> String { + var lines: [String] = [] + var previousText: String? + for cue in cues { + let text = cue.text + .replacingOccurrences(of: "\n", with: " ") + .trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty, text != previousText else { continue } + lines.append("[\(timestamp(cue.startTime))] \(text)") + previousText = text + } + guard !lines.isEmpty else { return "" } + let fullLength = lines.reduce(0) { $0 + $1.count + 1 } + guard fullLength > limit, limit > 0 else { return lines.joined(separator: "\n") } + + let stride = max(2, Int(ceil(Double(fullLength) / Double(limit)))) + var sampled: [String] = [] + for (index, line) in lines.enumerated() where index == 0 || index == lines.count - 1 || index % stride == 0 { + sampled.append(line) + } + return sampled.joined(separator: "\n") + } + + static func chapterPlanningPrompt( + videoTitle: String, + videoAuthor: String, + cues: [VideoSubtitleCue], + videoDuration: Double? + ) -> String { + let duration = effectiveDuration(videoDuration, cues: cues) + let targetCount = max(1, min(16, Int(ceil(duration / generatedChapterTargetDuration)))) + let author = videoAuthor.isEmpty ? "unknown" : videoAuthor + return """ + You are dividing a video transcript into semantic chapters. Work only \ + from the timestamped transcript below. Do not browse or use tools. + + Video: \(videoTitle) + Author: \(author) + Duration: \(timestamp(duration)) + + Return a single JSON object and nothing else, with this exact shape: + { + "chapters": [ + {"title": "concise topic-specific title", "startTime": 0} + ] + } + + Rules: + - Aim for about \(targetCount) chapters, but follow genuine conceptual boundaries rather than equal time slices. + - Each chapter must be one coherent teachable unit: a mechanism, method, experiment, result, argument, or worked example. + - Split when the talk moves from motivation to a mechanism, from one method to another, or between unrelated Q&A topics. + - For technical material, preserve the concrete technical subject in the title: name the algorithm, objective, architecture, benchmark, failure mode, or trade-off being taught. + - Titles must be concise and specific. Never use generic titles such as "Introduction", "Background", "Discussion", "Key Takeaways", "Part 1", or bare "Q&A". + - Do not title a chapter after presentation structure ("The speaker explains…"); title it after the knowledge itself. + - startTime is seconds from the start as a JSON number. + - Every startTime after 0 MUST exactly match a timestamp shown below. + - The first chapter MUST start at 0. + - Keep chapters in chronological order with no duplicate starts. + - Prefer chapters around 4-8 minutes; none may exceed 12 minutes. + - Avoid chapters shorter than 75 seconds unless the whole video is short. + - Cover the complete video. + + --- TIMESTAMPED TRANSCRIPT --- + \(timestampedTranscript(cues: cues)) + """ + } + + private struct ChapterPlanOutput: Decodable { + struct PlannedChapter: Decodable { + let title: String + let startTime: Double + } + let chapters: [PlannedChapter] + } + + /// Parses and validates a model-created outline. Boundaries are snapped to + /// actual subtitle cue starts; malformed, tiny, or overly sparse outlines + /// are rejected so callers can use the deterministic fallback. + static func parseGeneratedChapters( + output: String, + cues: [VideoSubtitleCue], + videoDuration: Double? + ) -> [VideoChapter]? { + guard !cues.isEmpty, + let json = extractJSONObject(from: output), + let data = json.data(using: .utf8), + let decoded = try? JSONDecoder().decode(ChapterPlanOutput.self, from: data) else { + return nil + } + let duration = effectiveDuration(videoDuration, cues: cues) + guard duration > 0 else { return nil } + + let cueStarts = cues.map(\.startTime) + var candidates = decoded.chapters.compactMap { planned -> (String, Double)? in + let title = planned.title.trimmingCharacters(in: .whitespacesAndNewlines) + guard !title.isEmpty, planned.startTime.isFinite else { return nil } + let clipped = min(max(0, planned.startTime), duration) + return (String(title.prefix(80)), nearestCueStart(to: clipped, cueStarts: cueStarts)) + } + .sorted { $0.1 < $1.1 } + guard !candidates.isEmpty else { return nil } + + // The model's first topical title applies from video start even when + // the first spoken subtitle begins a few seconds in. + candidates[0].1 = 0 + var accepted: [(String, Double)] = [] + for candidate in candidates { + guard accepted.last.map({ candidate.1 - $0.1 >= generatedChapterMinimumDuration }) ?? true else { + continue + } + accepted.append(candidate) + } + if accepted.count > 1, + let last = accepted.last, + duration - last.1 < generatedChapterMinimumDuration { + accepted.removeLast() + } + guard !accepted.isEmpty else { return nil } + if duration > generatedChapterMaximumDuration, accepted.count < 2 { return nil } + + let starts = accepted.map(\.1) + [duration] + for index in 0.. generatedChapterMaximumDuration { + return nil + } + return accepted.enumerated().map { index, value in + VideoChapter( + title: value.0, + startTime: value.1, + endTime: starts[index + 1] + ) + } + } + + /// Cue-aligned fixed windows used when the planner subprocess fails or + /// returns an invalid outline. This keeps the guide useful and guarantees + /// complete coverage without pretending the generic labels are semantic. + static func fallbackGeneratedChapters( + cues: [VideoSubtitleCue], + videoDuration: Double? + ) -> [VideoChapter] { + let duration = effectiveDuration(videoDuration, cues: cues) + guard duration > 0 else { return [] } + let count = max(1, min(16, Int(ceil(duration / generatedChapterTargetDuration)))) + let cueStarts = cues.map(\.startTime) + var starts = [0.0] + if count > 1 { + for index in 1..= generatedChapterMinimumDuration, + duration - snapped >= generatedChapterMinimumDuration { + starts.append(snapped) + } + } + } + return starts.enumerated().map { index, start in + let end = index + 1 < starts.count ? starts[index + 1] : duration + return VideoChapter(title: "Part \(index + 1)", startTime: start, endTime: end) + } + } + + private static func effectiveDuration(_ duration: Double?, cues: [VideoSubtitleCue]) -> Double { + let cueEnd = cues.last?.endTime ?? 0 + guard let duration, duration.isFinite, duration > 0 else { return cueEnd } + return max(duration, cueEnd) + } + + private static func nearestCueStart(to time: Double, cueStarts: [Double]) -> Double { + cueStarts.min(by: { abs($0 - time) < abs($1 - time) }) ?? time + } + + private static func timestamp(_ time: Double) -> String { + let seconds = max(0, Int(time.rounded())) + let hours = seconds / 3_600 + let minutes = (seconds % 3_600) / 60 + let remainder = seconds % 60 + return hours > 0 + ? String(format: "%d:%02d:%02d", hours, minutes, remainder) + : String(format: "%02d:%02d", minutes, remainder) + } + // MARK: Prompt static func prompt( diff --git a/Sources/Replay/ContentView.swift b/Sources/Replay/ContentView.swift index f2a1db1..83fbbfa 100644 --- a/Sources/Replay/ContentView.swift +++ b/Sources/Replay/ContentView.swift @@ -46,7 +46,7 @@ struct ContentView: View { if width < 1160, columnVisibility != .detailOnly, let selectedItem = store.selectedItem, - !selectedItem.availableChapters.isEmpty { + store.canShowChapterSidebar(for: selectedItem) { withAnimation(.easeInOut(duration: 0.22)) { columnVisibility = .detailOnly } @@ -770,13 +770,25 @@ private struct VideoDetail: View { } } + private var displayedChapters: [VideoChapter] { + store.displayChapters(for: item) + } + + private var hasChapterSidebar: Bool { + store.canShowChapterSidebar(for: item) + } + + private var usesGeneratedChapters: Bool { + item.availableChapters.isEmpty && !displayedChapters.isEmpty + } + private var usesCompactToolbarActions: Bool { - !item.availableChapters.isEmpty + hasChapterSidebar } @ViewBuilder private var chapterLayout: some View { - if item.availableChapters.isEmpty { + if !hasChapterSidebar { centerPane } else if #available(macOS 14.0, *) { centerPane @@ -830,7 +842,7 @@ private struct VideoDetail: View { } .fixedSize() - if !item.availableChapters.isEmpty, !chaptersPresented { + if hasChapterSidebar, !chaptersPresented { TitlebarInteractiveHost { Button(action: toggleChapters) { Image(systemName: "sidebar.trailing") @@ -897,7 +909,7 @@ private struct VideoDetail: View { PlaybackControls( snapshot: playback, knownDuration: item.duration, - chapters: item.availableChapters, + chapters: displayedChapters, togglePlayback: { PlaybackCommandCenter.shared.togglePlayback() }, skip: { PlaybackCommandCenter.shared.skip(by: $0) }, seek: seekToTime, @@ -919,9 +931,11 @@ private struct VideoDetail: View { private var chapterSidebar: some View { ChapterSidebar( - chapters: item.availableChapters, + chapters: displayedChapters, currentTime: playback.currentTime, isPresented: chaptersPresented, + usesGeneratedChapters: usesGeneratedChapters, + hasSubtitles: item.subtitleFileURL != nil, enrichment: store.enrichments[item.id], activity: store.enrichmentActivity[item.id], canEnrich: item.state == .ready && item.subtitleFileURL != nil, @@ -1133,7 +1147,7 @@ private struct VideoDetail: View { } private var prefersOneSidePane: Bool { - windowWidth < 1160 && !item.availableChapters.isEmpty + windowWidth < 1160 && hasChapterSidebar } private func collapseSidebarForNarrowChapterLayoutIfNeeded() { @@ -1753,6 +1767,8 @@ private struct ChapterSidebar: View { let chapters: [VideoChapter] let currentTime: Double let isPresented: Bool + let usesGeneratedChapters: Bool + let hasSubtitles: Bool let enrichment: VideoEnrichment? let activity: QueueStore.EnrichmentActivity? let canEnrich: Bool @@ -1778,14 +1794,19 @@ private struct ChapterSidebar: View { var body: some View { VStack(alignment: .leading, spacing: 0) { HStack(spacing: 10) { - Label("Chapters", systemImage: "list.bullet.rectangle") - .font(.headline.weight(.semibold)) - Text("\(chapters.count)") - .font(.caption.monospacedDigit()) - .foregroundStyle(.secondary) - .padding(.horizontal, 7) - .padding(.vertical, 3) - .background(Color.secondary.opacity(0.1), in: Capsule()) + Label( + chapters.isEmpty ? "Chapter guide" : (usesGeneratedChapters ? "Generated chapters" : "Chapters"), + systemImage: "list.bullet.rectangle" + ) + .font(.headline.weight(.semibold)) + if !chapters.isEmpty { + Text("\(chapters.count)") + .font(.caption.monospacedDigit()) + .foregroundStyle(.secondary) + .padding(.horizontal, 7) + .padding(.vertical, 3) + .background(Color.secondary.opacity(0.1), in: Capsule()) + } Spacer(minLength: 8) if isPresented { TitlebarInteractiveHost { @@ -1808,13 +1829,33 @@ private struct ChapterSidebar: View { Divider() ScrollView { - LazyVStack(alignment: .leading, spacing: 3) { - ForEach(chapters) { chapter in - chapterRow(chapter) + if chapters.isEmpty { + VStack(spacing: 10) { + Image(systemName: "list.bullet.rectangle") + .font(.system(size: 28, weight: .light)) + .foregroundStyle(Color.accentColor) + Text(hasSubtitles ? "No creator chapters" : "No chapters or subtitles") + .font(.system(size: sidebarFontSize + 2, weight: .semibold)) + Text(hasSubtitles + ? "Replay can find topic boundaries in the offline subtitles, then create notes and exercises for each section." + : "Replay needs offline subtitles before it can generate chapters and a study guide.") + .font(.system(size: sidebarFontSize)) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + .fixedSize(horizontal: false, vertical: true) } + .frame(maxWidth: .infinity) + .padding(.horizontal, 24) + .padding(.vertical, 36) + } else { + LazyVStack(alignment: .leading, spacing: 3) { + ForEach(chapters) { chapter in + chapterRow(chapter) + } + } + .padding(.horizontal, 8) + .padding(.vertical, 8) } - .padding(.horizontal, 8) - .padding(.vertical, 8) } .scrollIndicators(.hidden) @@ -2095,16 +2136,19 @@ private struct ChapterSidebar: View { .foregroundStyle(Color.accentColor) } case nil: - if enrichment == nil { + if enrichment == nil || chapters.isEmpty { Button { enrich(false, nil) } label: { - Label("Summaries & exercises", systemImage: "note.text") - .font(.caption.weight(.medium)) - .frame(maxWidth: .infinity) + Label( + chapters.isEmpty ? "Generate chapters & guide" : "Summaries & exercises", + systemImage: chapters.isEmpty ? "list.bullet.rectangle" : "note.text" + ) + .font(.caption.weight(.medium)) + .frame(maxWidth: .infinity) } .watchGlassButton() - .help("Generate a summary and exercises for each chapter") + .help(chapters.isEmpty ? "Find chapters and generate a study guide from subtitles" : "Generate a summary and exercises for each chapter") } else { HStack(spacing: 6) { Label("Chapter guide ready", systemImage: "note.text") diff --git a/Sources/Replay/EnrichmentEngine.swift b/Sources/Replay/EnrichmentEngine.swift index 09a3fd3..05b81b6 100644 --- a/Sources/Replay/EnrichmentEngine.swift +++ b/Sources/Replay/EnrichmentEngine.swift @@ -1,9 +1,10 @@ import Foundation -/// Generates per-chapter summaries and exercises by running the pi coding -/// agent through the bundled Deno runtime (`deno run npm:...`), one -/// subprocess per chapter with bounded concurrency. Results are persisted by -/// the caller; this engine only orchestrates subprocesses and parsing. +/// Generates chapter outlines, summaries, and exercises by running the pi +/// coding agent through the bundled Deno runtime (`deno run npm:...`). Videos +/// without source chapters get one planning subprocess first; guide generation +/// then uses one subprocess per chapter with bounded concurrency. Results are +/// persisted by the caller; this engine only orchestrates and parses. final class EnrichmentEngine { struct Progress { let completedChapters: Int @@ -145,15 +146,36 @@ final class EnrichmentEngine { } let deno = try requiredTool(named: "deno") - // Videos without creator chapters are treated as one whole-video chapter. - let chapters = item.availableChapters.isEmpty - ? [VideoChapter(title: item.title, startTime: 0, endTime: item.duration)] - : item.availableChapters - try FileManager.default.createDirectory(at: workDirectory, withIntermediateDirectories: true) try FileManager.default.createDirectory(at: denoCacheDirectory, withIntermediateDirectories: true) + let generatedChapters: [VideoChapter]? + let chapters: [VideoChapter] + if item.availableChapters.isEmpty { + if let persisted = existing?.generatedChapters, !persisted.isEmpty { + generatedChapters = persisted + chapters = persisted + } else { + let planned = generateChapterOutline( + item: item, + cues: track.cues, + deno: deno, + workDirectory: workDirectory, + denoCacheDirectory: denoCacheDirectory + ) + generatedChapters = planned + chapters = planned + } + } else { + generatedChapters = nil + chapters = item.availableChapters + } + guard !chapters.isEmpty else { + throw EngineError.failed("The subtitle track was too short to create chapters.") + } + let totalChapters = chapters.count + onProgress(Progress(completedChapters: 0, totalChapters: totalChapters)) let stateLock = NSLock() var results: [Int: ChapterEnrichment] = [:] var failures: [Int: String] = [:] @@ -218,11 +240,47 @@ final class EnrichmentEngine { version: VideoEnrichment.currentVersion, itemID: item.id, generatedAt: Date(), - chapters: ordered + chapters: ordered, + generatedChapters: generatedChapters ) return Outcome(enrichment: enrichment, failedChapterTitles: failedTitles) } + private func generateChapterOutline( + item: WatchItem, + cues: [VideoSubtitleCue], + deno: URL, + workDirectory: URL, + denoCacheDirectory: URL + ) -> [VideoChapter] { + let prompt = ChapterEnrichmentLogic.chapterPlanningPrompt( + videoTitle: item.title, + videoAuthor: item.author, + cues: cues, + videoDuration: item.duration + ) + let result = runPi( + prompt: prompt, + itemID: item.id, + slug: "chapter-plan", + deno: deno, + workDirectory: workDirectory, + denoCacheDirectory: denoCacheDirectory + ) + if case .success(let output) = result, + let chapters = ChapterEnrichmentLogic.parseGeneratedChapters( + output: output, + cues: cues, + videoDuration: item.duration + ) { + return chapters + } + return ChapterEnrichmentLogic.fallbackGeneratedChapters( + cues: cues, + videoDuration: item.duration + ) + } + private func enrichChapter( itemID: UUID, item: WatchItem, @@ -258,6 +316,38 @@ final class EnrichmentEngine { guidance: guidance ) let slug = String(format: "chapter-%02d", chapterIndex + 1) + switch runPi( + prompt: prompt, + itemID: itemID, + slug: slug, + deno: deno, + workDirectory: workDirectory, + denoCacheDirectory: denoCacheDirectory + ) { + case .success(let output): + guard let enrichment = ChapterEnrichmentLogic.parse( + output: output, + chapterID: chapter.id, + chapterTitle: chapter.title + ) else { + return .failure(EngineError.failed("Could not parse the model output for “\(chapter.title)”. See \(slug).stdout.txt.")) + } + return .success(enrichment) + case .failure(let error): + return .failure(error) + } + } + + /// Runs one headless, hermetic pi subprocess and records its prompt and + /// streams beside the enrichment output for diagnosis. + private func runPi( + prompt: String, + itemID: UUID, + slug: String, + deno: URL, + workDirectory: URL, + denoCacheDirectory: URL + ) -> Result { let promptFile = workDirectory.appendingPathComponent("\(slug).prompt.md") let stderrFile = workDirectory.appendingPathComponent("\(slug).stderr.txt") let stdoutFile = workDirectory.appendingPathComponent("\(slug).stdout.txt") @@ -269,9 +359,8 @@ final class EnrichmentEngine { let process = Process() process.executableURL = deno - // Headless, hermetic pi run: no session, no extensions, no skills, no - // tools. Extensions in particular can open WebSockets, which crash - // under Deno's npm compatibility layer. + // No session, extensions, skills, context, or tools. The subprocess + // only transforms the supplied transcript into structured JSON. var arguments = [ "run", "-A", "--quiet", Configuration.piPackage, "-p", "--no-session", "-ne", "--no-skills", @@ -295,7 +384,6 @@ final class EnrichmentEngine { let stderr = Pipe() process.standardOutput = stdout process.standardError = stderr - do { try process.run() } catch { @@ -303,13 +391,11 @@ final class EnrichmentEngine { } register(process, for: itemID) - // Watchdog: kill hung sub-agents so the queue always finishes. let watchdog = DispatchWorkItem { [weak process] in process?.terminate() } DispatchQueue.global(qos: .utility).asyncAfter( deadline: .now() + Self.chapterTimeout, execute: watchdog ) - let stdoutData = stdout.fileHandleForReading.readDataToEndOfFile() let stderrData = stderr.fileHandleForReading.readDataToEndOfFile() process.waitUntilExit() @@ -317,26 +403,17 @@ final class EnrichmentEngine { unregister(process, for: itemID) let output = String(decoding: stdoutData, as: UTF8.self) + let stderrText = String(decoding: stderrData, as: UTF8.self) try? output.write(to: stdoutFile, atomically: true, encoding: .utf8) - try? String(decoding: stderrData, as: UTF8.self).write(to: stderrFile, atomically: true, encoding: .utf8) + try? stderrText.write(to: stderrFile, atomically: true, encoding: .utf8) - if isCancelled(itemID: itemID) { - return .failure(EngineError.cancelled) - } + if isCancelled(itemID: itemID) { return .failure(EngineError.cancelled) } guard process.terminationStatus == 0 else { - let stderrText = String(decoding: stderrData, as: UTF8.self) let detail = ChapterEnrichmentLogic.errorSummary(fromStderr: stderrText) ?? "pi exited with status \(process.terminationStatus)" return .failure(EngineError.failed(detail)) } - guard let enrichment = ChapterEnrichmentLogic.parse( - output: output, - chapterID: chapter.id, - chapterTitle: chapter.title - ) else { - return .failure(EngineError.failed("Could not parse the model output for “\(chapter.title)”. See \(stdoutFile.lastPathComponent).")) - } - return .success(enrichment) + return .success(output) } // MARK: - Helpers diff --git a/Sources/Replay/QueueStore.swift b/Sources/Replay/QueueStore.swift index 400141f..a3e9fa3 100644 --- a/Sources/Replay/QueueStore.swift +++ b/Sources/Replay/QueueStore.swift @@ -484,6 +484,17 @@ final class QueueStore: ObservableObject { return false } + /// Creator chapters always win. A generated outline is only displayed for + /// videos whose source metadata did not contain chapters. + func displayChapters(for item: WatchItem) -> [VideoChapter] { + if !item.availableChapters.isEmpty { return item.availableChapters } + return enrichments[item.id]?.generatedChapters ?? [] + } + + func canShowChapterSidebar(for item: WatchItem) -> Bool { + !displayChapters(for: item).isEmpty || item.state == .ready + } + func enrichChapters(for id: UUID, force: Bool = false, guidance: String? = nil) { guard let item = item(with: id), item.state == .ready, !isEnriching(id) else { return } guard item.subtitleFileURL != nil else { @@ -495,7 +506,7 @@ final class QueueStore: ObservableObject { // With guidance, the previous guide is context for revision; a plain // force regenerates from scratch. let existing = (force && revisionGuidance == nil) ? nil : enrichments[id] - let chapterCount = max(1, item.availableChapters.count) + let chapterCount = max(1, displayChapters(for: item).count) enrichmentActivity[id] = .running(completed: 0, total: chapterCount) let workDirectory = enrichmentFileURL(for: id).deletingLastPathComponent() diff --git a/docs/chapter-enrichment-plan.md b/docs/chapter-enrichment-plan.md index da3f6c5..b498008 100644 --- a/docs/chapter-enrichment-plan.md +++ b/docs/chapter-enrichment-plan.md @@ -24,13 +24,17 @@ > WebSockets, which also crash under Deno). > - Enrichment JSON lives at `Application Support/Replay/Enrichment//enrichment.json` > with per-chapter prompt/stdout/stderr files beside it for debugging. +> Videos without creator chapters first run `chapter-plan.prompt.md` through +> pi; the validated generated outline is persisted in the same JSON. > The Deno npm cache is pinned to `Application Support/Replay/DenoCache`. ## Goal -For any downloaded video with chapters, generate a **small summary and a few +For any downloaded video with subtitles, generate a **small summary and a few exercises per chapter**, viewable inline in the chapter sidebar, fully offline -after generation. Modeled on `~/Documents/Notes/Clippings/enrich_clippings_agentic.py` +after generation. When creator chapters are absent, first ask pi to derive a +semantic, timestamped chapter outline from the transcript. Modeled on +`~/Documents/Notes/Clippings/enrich_clippings_agentic.py` (pi sub-agent per unit of work, strict output contract), but running pi through the **Deno binary already bundled in Replay** — no Node install required. @@ -51,9 +55,11 @@ User clicks "Enrich chapters" (or auto after download, opt-in) │ ▼ EnrichmentEngine (new, Swift actor — sibling of DownloadEngine) - 1. Load VideoSubtitleTrack, slice cues into per-chapter transcript text - (cue.startTime within [chapter.start, chapter.end)). - 2. For each chapter: build prompt file in a work dir + 1. Load VideoSubtitleTrack. If creator chapters are absent, run a dedicated + pi subprocess to propose semantic boundaries and titles, then snap and + validate its timestamps against subtitle cues (fixed windows are the + deterministic fallback). + 2. Slice cues into per-chapter transcript text and build a prompt file (Application Support/Enrichment//chapter-03.prompt.md). 3. Spawn pi subprocess per chapter, bounded concurrency (2–3): deno run -A npm:@earendil-works/pi-coding-agent@ \ @@ -115,11 +121,13 @@ on the prompt to forbid tool use. ### Videos without chapters / without subtitles -- **No chapters:** offer a single whole-video summary + exercises (treat the - video as one chapter). Optionally later: ask pi to *propose* chapter splits - from the transcript. -- **No subtitles:** disable the feature for that item with an explanatory - tooltip (v1). Later option: whisper/ffmpeg transcription. +- **No chapters:** show the chapter-guide sidebar and offer **Generate chapters + & guide**. A pi subprocess proposes semantic titles and cue-aligned starts; + invalid output falls back to roughly six-minute cue-aligned sections. + Generated chapters are visibly labeled and persisted separately from source + metadata. +- **No subtitles:** keep the sidebar visible with an explanatory empty state; + generation remains disabled. Later option: whisper/ffmpeg transcription. ## Key decisions & risks diff --git a/tools/chapter_enrichment_check.swift b/tools/chapter_enrichment_check.swift index b4be8af..7e8ceda 100644 --- a/tools/chapter_enrichment_check.swift +++ b/tools/chapter_enrichment_check.swift @@ -8,6 +8,8 @@ struct ChapterEnrichmentCheck { checkTruncation() checkJSONExtraction() checkOutputParsing() + checkGeneratedChapterPlanning() + checkGeneratedChapterFallback() checkErrorSummary() checkMathRendering() checkRevisionPrompt() @@ -131,6 +133,81 @@ struct ChapterEnrichmentCheck { precondition(prompt.contains("Chapter 2 of 3: Main")) precondition(prompt.contains("single JSON object")) precondition(prompt.contains("Author: unknown")) + precondition(prompt.contains("every exercise must be technical")) + precondition(prompt.contains("NEVER ask the learner to recall the talk")) + precondition(prompt.contains("design an experiment or ablation")) + } + + static func checkGeneratedChapterPlanning() { + let planningCues = stride(from: 0.0, through: 1_800.0, by: 60.0).map { + VideoSubtitleCue(startTime: $0, endTime: $0 + 20, text: "topic at \(Int($0))") + } + let transcript = ChapterEnrichmentLogic.timestampedTranscript(cues: planningCues) + precondition(transcript.contains("[00:00] topic at 0")) + precondition(transcript.contains("[30:00] topic at 1800")) + + let prompt = ChapterEnrichmentLogic.chapterPlanningPrompt( + videoTitle: "A long lesson", + videoAuthor: "Teacher", + cues: planningCues, + videoDuration: 1_820 + ) + precondition(prompt.contains("semantic chapters")) + precondition(prompt.contains("startTime")) + precondition(prompt.contains("A long lesson")) + precondition(prompt.contains("one coherent teachable unit")) + precondition(prompt.contains("algorithm, objective, architecture, benchmark")) + precondition(prompt.contains("between unrelated Q&A topics")) + + let output = """ + Here is the outline: + {"chapters":[ + {"title":"Opening idea","startTime":12}, + {"title":"Core technique","startTime":603}, + {"title":"Worked examples","startTime":1192} + ]} + """ + guard let generated = ChapterEnrichmentLogic.parseGeneratedChapters( + output: output, + cues: planningCues, + videoDuration: 1_820 + ) else { + preconditionFailure("Expected a valid generated outline") + } + precondition(generated.map(\.title) == ["Opening idea", "Core technique", "Worked examples"]) + precondition(generated.map(\.startTime) == [0, 600, 1_200]) + precondition(generated[0].endTime == 600) + precondition(generated.last?.endTime == 1_820) + + // A single chapter cannot cover a long video because it would exceed + // the planner's maximum chapter duration. + let sparse = "{\"chapters\":[{\"title\":\"Everything\",\"startTime\":0}]}" + precondition(ChapterEnrichmentLogic.parseGeneratedChapters( + output: sparse, + cues: planningCues, + videoDuration: 1_820 + ) == nil) + precondition(ChapterEnrichmentLogic.parseGeneratedChapters( + output: "not json", + cues: planningCues, + videoDuration: 1_820 + ) == nil) + } + + static func checkGeneratedChapterFallback() { + let planningCues = stride(from: 0.0, through: 1_400.0, by: 20.0).map { + VideoSubtitleCue(startTime: $0, endTime: $0 + 10, text: "cue \(Int($0))") + } + let fallback = ChapterEnrichmentLogic.fallbackGeneratedChapters( + cues: planningCues, + videoDuration: 1_440 + ) + precondition(fallback.count == 4) + precondition(fallback.map(\.title) == ["Part 1", "Part 2", "Part 3", "Part 4"]) + precondition(fallback.first?.startTime == 0) + precondition(fallback.last?.endTime == 1_440) + precondition(Set(planningCues.map(\.startTime)).contains(fallback[1].startTime)) + precondition(ChapterEnrichmentLogic.fallbackGeneratedChapters(cues: [], videoDuration: nil).isEmpty) } static func checkErrorSummary() { @@ -249,6 +326,9 @@ struct ChapterEnrichmentCheck { exercises: [ChapterExercise(question: "q", solution: "a")], generatedAt: Date() ) + ], + generatedChapters: [ + VideoChapter(title: "Generated intro", startTime: 0, endTime: 30) ] ) let dir = FileManager.default.temporaryDirectory @@ -267,6 +347,7 @@ struct ChapterEnrichmentCheck { precondition(loaded.chapters.map(\.keyPoints) == enrichment.chapters.map(\.keyPoints)) precondition(loaded.enrichment(forChapterID: "0.0-Intro")?.summary == "S") precondition(loaded.enrichment(forChapterID: "missing") == nil) + precondition(loaded.generatedChapters?.first?.title == "Generated intro") // Unknown versions are rejected rather than misread. var future = enrichment From 2c2a5a525ab8e542c7320c86cf40a4893b275582 Mon Sep 17 00:00:00 2001 From: swairshah Date: Thu, 13 Aug 2026 19:40:15 -0700 Subject: [PATCH 6/9] fix(sidebar): make queue interactions deterministic Align rendered rows with their hit regions, move reordering to a dedicated handle, and replace unreliable double-click renaming with a context-menu action. --- Sources/Replay/ContentView.swift | 61 ++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/Sources/Replay/ContentView.swift b/Sources/Replay/ContentView.swift index 83fbbfa..fcbf0c0 100644 --- a/Sources/Replay/ContentView.swift +++ b/Sources/Replay/ContentView.swift @@ -8,6 +8,7 @@ struct ContentView: View { @State private var urlText = "" @State private var isDropTarget = false @State private var itemToDelete: WatchItem? + @State private var renameRequestID: UUID? @State private var detailSelection: UUID? @State private var detailSelectionTask: Task? @State private var knownItemIDs: Set = [] @@ -141,7 +142,6 @@ struct ContentView: View { queueList } .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topLeading) - .ignoresSafeArea(.container, edges: .top) } } @@ -157,10 +157,6 @@ struct ContentView: View { ScrollViewReader { proxy in ScrollView(.vertical) { LazyVStack(alignment: .leading, spacing: 4, pinnedViews: [.sectionHeaders]) { - Color.clear - .frame(height: 0) - .id("queue-top") - if !queueItems.isEmpty { ForEach(queueItems) { item in sidebarRow(item) @@ -172,10 +168,6 @@ struct ContentView: View { ) } } - .simultaneousGesture( - DragGesture(minimumDistance: 4, coordinateSpace: .named("queue-list")) - .onChanged { updateQueueDrag(item.id, at: $0.location) } - ) } } @@ -212,15 +204,6 @@ struct ContentView: View { proxy.scrollTo(addedID, anchor: .top) } } - .onAppear { - // A persisted selection can be far down the queue. The - // list itself should nevertheless start at its real top - // after every launch, with the normal eight-point inset - // above the newest item. - DispatchQueue.main.async { - proxy.scrollTo("queue-top", anchor: .top) - } - } } } } @@ -230,7 +213,12 @@ struct ContentView: View { item: item, isSelected: store.selection == item.id, select: { store.selection = item.id }, - rename: { store.rename(item.id, to: $0) } + renameRequested: renameRequestID == item.id, + rename: { store.rename(item.id, to: $0) }, + finishRenameRequest: { + if renameRequestID == item.id { renameRequestID = nil } + }, + reorder: { updateQueueDrag(item.id, at: $0) } ) .equatable() .id(item.id) @@ -243,6 +231,7 @@ struct ContentView: View { Button("Retry Download") { store.startDownload(for: item.id) } } Button("Open Original") { store.openOriginal(item.id) } + Button("Rename") { renameRequestID = item.id } Divider() Button("Remove", role: .destructive) { itemToDelete = item } } @@ -398,13 +387,18 @@ private struct QueueRow: View, Equatable { let item: WatchItem let isSelected: Bool let select: () -> Void + let renameRequested: Bool let rename: (String) -> Void + let finishRenameRequest: () -> Void + let reorder: (CGPoint) -> Void @State private var isEditingTitle = false @State private var draftTitle = "" @FocusState private var isTitleFocused: Bool static func == (lhs: QueueRow, rhs: QueueRow) -> Bool { - lhs.item == rhs.item && lhs.isSelected == rhs.isSelected + lhs.item == rhs.item && + lhs.isSelected == rhs.isSelected && + lhs.renameRequested == rhs.renameRequested } var body: some View { @@ -434,6 +428,10 @@ private struct QueueRow: View, Equatable { } Spacer(minLength: 0) + + if !item.isWatched { + reorderHandle + } } .frame(minWidth: 0, maxWidth: .infinity, alignment: .leading) .padding(.horizontal, 8) @@ -450,6 +448,9 @@ private struct QueueRow: View, Equatable { } .contentShape(RoundedRectangle(cornerRadius: 13, style: .continuous)) .onTapGesture(perform: select) + .onChange(of: renameRequested) { requested in + if requested { beginEditing() } + } .accessibilityElement(children: .combine) .accessibilityAddTraits(isSelected ? [.isButton, .isSelected] : .isButton) .accessibilityAction(.default, select) @@ -483,15 +484,28 @@ private struct QueueRow: View, Equatable { .lineLimit(2) .multilineTextAlignment(.leading) .contentShape(Rectangle()) - .onTapGesture(count: 2, perform: beginEditing) - .help("Double-click to rename") + .help("Right-click to rename") } } + private var reorderHandle: some View { + Image(systemName: "line.3.horizontal") + .font(.caption.weight(.semibold)) + .foregroundStyle(.tertiary) + .frame(width: 28, height: 40) + .contentShape(Rectangle()) + .gesture( + DragGesture(minimumDistance: 6, coordinateSpace: .named("queue-list")) + .onChanged { reorder($0.location) } + ) + .help("Drag to reorder") + } + private func beginEditing() { - select() + guard !isEditingTitle else { return } draftTitle = item.title isEditingTitle = true + select() DispatchQueue.main.async { isTitleFocused = true } @@ -507,6 +521,7 @@ private struct QueueRow: View, Equatable { } isEditingTitle = false isTitleFocused = false + finishRenameRequest() } private var icon: String { From e5beafcffe8f516b01378c7df0198f9b32d9a493 Mon Sep 17 00:00:00 2001 From: swairshah Date: Thu, 13 Aug 2026 19:40:27 -0700 Subject: [PATCH 7/9] feat(sidebar): add animated title search Filter queue and archived videos by title with an expanding search control. Shrink the URL input alongside it, restore the full list on Escape, and provide a no-results state. --- Sources/Replay/ContentView.swift | 195 ++++++++++++++++++++++++++++--- 1 file changed, 179 insertions(+), 16 deletions(-) diff --git a/Sources/Replay/ContentView.swift b/Sources/Replay/ContentView.swift index fcbf0c0..21b0c56 100644 --- a/Sources/Replay/ContentView.swift +++ b/Sources/Replay/ContentView.swift @@ -6,6 +6,8 @@ struct ContentView: View { @EnvironmentObject private var store: QueueStore @EnvironmentObject private var inbox: URLInbox @State private var urlText = "" + @State private var searchText = "" + @State private var isSearchPresented = false @State private var isDropTarget = false @State private var itemToDelete: WatchItem? @State private var renameRequestID: UUID? @@ -17,6 +19,7 @@ struct ContentView: View { @State private var windowWidth: CGFloat = 1320 @State private var urlBarFrame: CGRect = .zero @FocusState private var isURLFieldFocused: Bool + @FocusState private var isSearchFieldFocused: Bool var body: some View { NavigationSplitView(columnVisibility: $columnVisibility) { @@ -123,19 +126,7 @@ struct ContentView: View { .allowsHitTesting(false) VStack(spacing: 0) { - DropAndAddBar( - urlText: $urlText, - isDropTarget: $isDropTarget, - isURLFieldFocused: $isURLFieldFocused, - submit: submitURL, - receiveProviders: receiveProviders - ) - .padding(.leading, 82) - .padding(.trailing, 54) - // NavigationSplitView supplies the native title-bar inset. Keep - // the add field in the unshifted center of that header so it - // shares a baseline with the traffic-light cluster. - .frame(height: 46) + sidebarHeader Divider() @@ -145,14 +136,100 @@ struct ContentView: View { } } + private var sidebarHeader: some View { + GeometryReader { geometry in + let expandedWidth = max(120, geometry.size.width - 136) + HStack(spacing: 28) { + Group { + if isSearchPresented { + SidebarSearchBar( + text: $searchText, + isFocused: $isSearchFieldFocused, + close: closeSearch + ) + } else { + sidebarHeaderIconButton( + systemImage: "magnifyingglass", + help: "Search video titles (⌘F)", + action: openSearch + ) + .keyboardShortcut("f", modifiers: .command) + } + } + .frame(width: isSearchPresented ? expandedWidth : 40) + + Group { + if isSearchPresented { + sidebarHeaderIconButton( + systemImage: "link", + help: "Add a video", + action: activateURLInput + ) + } else { + DropAndAddBar( + urlText: $urlText, + isDropTarget: $isDropTarget, + isURLFieldFocused: $isURLFieldFocused, + submit: submitURL, + receiveProviders: receiveProviders + ) + } + } + .frame(width: isSearchPresented ? 40 : expandedWidth) + } + .padding(.leading, 14) + .frame(maxHeight: .infinity) + .animation(.easeInOut(duration: 0.22), value: isSearchPresented) + } + .frame(height: 46) + } + + private func sidebarHeaderIconButton( + systemImage: String, + help: String, + action: @escaping () -> Void + ) -> some View { + Button(action: action) { + Image(systemName: systemImage) + .font(.system(size: 13, weight: .medium)) + .foregroundStyle(.secondary) + .frame(width: 40, height: 32) + .contentShape(Rectangle()) + .watchGlass( + .clear, + interactive: true, + in: RoundedRectangle(cornerRadius: 15, style: .continuous) + ) + .overlay { + RoundedRectangle(cornerRadius: 15, style: .continuous) + .strokeBorder(Color.primary.opacity(0.07)) + } + } + .buttonStyle(.plain) + .help(help) + } + + private var normalizedSearchQuery: String { + searchText.trimmingCharacters(in: .whitespacesAndNewlines) + } + + private func matchesSearch(_ item: WatchItem) -> Bool { + normalizedSearchQuery.isEmpty || item.title.localizedCaseInsensitiveContains(normalizedSearchQuery) + } + @ViewBuilder private var queueList: some View { - let queueItems = store.queueItems - let archivedItems = store.archivedItems + let allQueueItems = store.queueItems + let allArchivedItems = store.archivedItems + let queueItems = allQueueItems.filter(matchesSearch) + let archivedItems = allArchivedItems.filter(matchesSearch) - if queueItems.isEmpty && archivedItems.isEmpty { + if allQueueItems.isEmpty && allArchivedItems.isEmpty { SidebarEmptyState() .frame(maxWidth: .infinity, maxHeight: .infinity) + } else if queueItems.isEmpty && archivedItems.isEmpty && !normalizedSearchQuery.isEmpty { + SidebarSearchEmptyState(query: normalizedSearchQuery) + .frame(maxWidth: .infinity, maxHeight: .infinity) } else { ScrollViewReader { proxy in ScrollView(.vertical) { @@ -276,6 +353,32 @@ struct ContentView: View { } } + private func openSearch() { + isURLFieldFocused = false + withAnimation(.easeInOut(duration: 0.22)) { + isSearchPresented = true + } + DispatchQueue.main.async { + isSearchFieldFocused = true + } + } + + private func closeSearch() { + isSearchFieldFocused = false + searchText = "" + withAnimation(.easeInOut(duration: 0.22)) { + isSearchPresented = false + } + NSApp.keyWindow?.makeFirstResponder(nil) + } + + private func activateURLInput() { + closeSearch() + DispatchQueue.main.async { + isURLFieldFocused = true + } + } + private func submitURL() { let value = urlText urlText = "" @@ -637,6 +740,26 @@ private struct QueueThumbnail: View { } } +private struct SidebarSearchEmptyState: View { + let query: String + + var body: some View { + VStack(spacing: 10) { + Image(systemName: "magnifyingglass") + .font(.system(size: 25, weight: .light)) + .foregroundStyle(.secondary) + Text("No matching videos") + .font(.headline) + Text("No titles contain “\(query)”. Press Escape to show the full list.") + .font(.caption) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + .frame(maxWidth: 210) + } + .padding() + } +} + private struct SidebarEmptyState: View { var body: some View { VStack(spacing: 10) { @@ -2255,6 +2378,46 @@ private struct ChapterSidebar: View { } } +private struct SidebarSearchBar: View { + @Binding var text: String + var isFocused: FocusState.Binding + let close: () -> Void + + var body: some View { + HStack(spacing: 8) { + Image(systemName: "magnifyingglass") + .font(.system(size: 13, weight: .medium)) + .foregroundStyle(.secondary) + + TextField("Search video titles", text: $text) + .textFieldStyle(.plain) + .focused(isFocused) + .onExitCommand(perform: close) + + Button(action: close) { + Image(systemName: "xmark") + .font(.system(size: 12, weight: .bold)) + } + .watchGlassButton(prominent: true) + .controlSize(.small) + .keyboardShortcut(.cancelAction) + .help("Close search") + } + .padding(.leading, 12) + .padding(.trailing, 7) + .frame(maxWidth: .infinity, minHeight: 32, maxHeight: 32) + .watchGlass( + .clear, + interactive: true, + in: RoundedRectangle(cornerRadius: 15, style: .continuous) + ) + .overlay { + RoundedRectangle(cornerRadius: 15, style: .continuous) + .strokeBorder(Color.primary.opacity(0.07)) + } + } +} + private struct DropAndAddBar: View { @Binding var urlText: String @Binding var isDropTarget: Bool From ba604f43e2784d650b850042f1c57e6a38965143 Mon Sep 17 00:00:00 2001 From: swairshah Date: Thu, 13 Aug 2026 19:40:33 -0700 Subject: [PATCH 8/9] fix(player): prevent mini-player activation crash Order the floating panel with a plain host before attaching AVPlayerView so ViewBridge never processes a remote video view during window activation. --- Sources/Replay/LocalVideoPlayer.swift | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/Sources/Replay/LocalVideoPlayer.swift b/Sources/Replay/LocalVideoPlayer.swift index 808cc2d..0ed25b3 100644 --- a/Sources/Replay/LocalVideoPlayer.swift +++ b/Sources/Replay/LocalVideoPlayer.swift @@ -1232,7 +1232,15 @@ struct LocalVideoPlayer: NSViewRepresentable { panel.hasShadow = true panel.animationBehavior = .none panel.isReleasedWhenClosed = false - panel.contentView = floatingView + + // Order a plain host onscreen before inserting AVPlayerView. On + // macOS 27, ordering a window that already contains AVPlayerView's + // NSRemoteView can raise an Objective-C ViewBridge exception and + // abort the process during app deactivation. + let panelHost = NSView(frame: NSRect(origin: .zero, size: FloatingPlayerLayout.size)) + panelHost.wantsLayer = true + panelHost.layer?.backgroundColor = NSColor.black.cgColor + panel.contentView = panelHost floatingView.onHoverChanged = { [weak panel] isHovering in panel?.standardWindowButton(.closeButton)?.isHidden = !isHovering } @@ -1246,17 +1254,22 @@ struct LocalVideoPlayer: NSViewRepresentable { ?? NSRect(origin: .zero, size: FloatingPlayerLayout.size) panel.setFrame(FloatingPlayerLayout.frame(in: visibleFrame), display: false) - // Move the existing player output only after the panel is fully - // configured. The panel is shown at its final location, so the sole - // transition is opacity—there is never a position animation. - playerView?.playerLayer.player = nil - fullscreenPlayerView?.playerLayer.player = nil - floatingView.player = player + // Show the empty host at its final location first, then move the + // existing player output into it. The sole transition is opacity; + // there is never a position animation or a remote view present + // while AppKit orders the panel onscreen. backgroundPanel = panel backgroundPlayerView = floatingView panel.alphaValue = 0 panel.orderFrontRegardless() + floatingView.frame = panelHost.bounds + floatingView.autoresizingMask = [.width, .height] + panelHost.addSubview(floatingView) + playerView?.playerLayer.player = nil + fullscreenPlayerView?.playerLayer.player = nil + floatingView.player = player + let duration = NSWorkspace.shared.accessibilityDisplayShouldReduceMotion ? 0 : 0.15 guard duration > 0 else { panel.alphaValue = 1 From 00cd3208887df91769319a5d5cf64e731b883e76 Mon Sep 17 00:00:00 2001 From: swairshah Date: Fri, 11 Sep 2026 00:56:06 -0700 Subject: [PATCH 9/9] Fix YouTube downloads after client changes --- Sources/Replay/DownloadEngine.swift | 2 +- scripts/prepare_runtime_tools.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/Replay/DownloadEngine.swift b/Sources/Replay/DownloadEngine.swift index 21baee8..aa09f8a 100644 --- a/Sources/Replay/DownloadEngine.swift +++ b/Sources/Replay/DownloadEngine.swift @@ -69,7 +69,7 @@ final class DownloadEngine { "--no-color", "--paths", destination.path, "--output", "\(itemID.uuidString).%(ext)s", - "--format", "bv*[height<=1080][ext=mp4]+ba[ext=m4a]/b[height<=1080][ext=mp4]/b[height<=1080]/best", + "--format", "bv*[height<=1080][ext=mp4][protocol^=http]+ba[ext=m4a][protocol^=http]/b[height<=1080][ext=mp4][protocol^=http]/b[height<=1080]/best", "--merge-output-format", "mp4", "--write-thumbnail", "--convert-thumbnails", "jpg", diff --git a/scripts/prepare_runtime_tools.sh b/scripts/prepare_runtime_tools.sh index e2eedfd..ae986c7 100755 --- a/scripts/prepare_runtime_tools.sh +++ b/scripts/prepare_runtime_tools.sh @@ -7,7 +7,7 @@ downloads_dir="$runtime_root/downloads" source_dir="$runtime_root/source" output_dir="$runtime_root/universal" -yt_dlp_version="2026.07.04" +yt_dlp_version="2026.08.19" deno_version="v2.9.5" ffmpeg_version="8.1.1"