From 7efd57aefc4ad80e6cf44791878a62b8914006a7 Mon Sep 17 00:00:00 2001 From: Quang <20378quang@gmail.com> Date: Mon, 21 Sep 2026 12:34:00 -0400 Subject: [PATCH 1/4] fix: report a run of dropped audio as one transcription-gap incident MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recognition falling behind capture does not lose audio once. It keeps losing it for as long as it stays behind, and the pipeline reported each accumulated half second as its own gap, so a meeting that spent minutes behind buried its transcript under hundreds of near-identical blockquotes. Observations are now accumulated into a TranscriptGapIncident and written as one marker when the incident ends. The marker states two quantities and keeps them apart: the range the incident spanned, and the audio it actually cost. The total is a sum over distinct evicted buffers — the bounded backlog hands each one back exactly once — and is never derived from the range, because recognition goes on transcribing between the losses. What separates one incident from the next is evidence rather than a delay. TranscriptionAudioInput reports closesIncident once the backlog has accepted a full capacity of audio without evicting any of it, which it could not do while still full; MeetingRuntime closes an incident at every boundary the pipeline cannot see — pause, recogniser restart, capture ending, stop, persistence failure — so no failure is hidden inside another. --- ScribeKit/Diagnostics/DiagnosticReport.swift | 6 +- ScribeKit/Meeting/LiveTranscriptModel.swift | 10 +- ScribeKit/Meeting/MeetingRuntime.swift | 120 +++++++++++++++- ScribeKit/Models/TranscriptGap.swift | 55 +++++++- ScribeKit/Models/TranscriptGapIncident.swift | 124 +++++++++++++++++ ScribeKit/Models/TranscriptionEvent.swift | 128 +++++++++++++++--- .../TranscriptMarkdownFormatter.swift | 52 ++++++- .../AppleSpeechTranscriber.swift | 8 +- .../TranscriptionAudioInput.swift | 109 +++++++++++---- 9 files changed, 537 insertions(+), 75 deletions(-) create mode 100644 ScribeKit/Models/TranscriptGapIncident.swift diff --git a/ScribeKit/Diagnostics/DiagnosticReport.swift b/ScribeKit/Diagnostics/DiagnosticReport.swift index a79f72e..7383f9a 100644 --- a/ScribeKit/Diagnostics/DiagnosticReport.swift +++ b/ScribeKit/Diagnostics/DiagnosticReport.swift @@ -225,10 +225,12 @@ nonisolated struct DiagnosticReport: Codable, Equatable, Sendable { /// How many finalised spans reached the transcript. let transcriptSpanCount: Int - /// How many gaps were written into it. + /// How many gap markers were written into it, which is one per + /// incident rather than one per reported loss. let gapCount: Int - /// How much time those gaps account for. + /// How much audio the meeting could not transcribe, summed over every + /// loss the pipeline reported rather than over the markers. let untranscribedSeconds: Double /// Whether a recording file was opened. diff --git a/ScribeKit/Meeting/LiveTranscriptModel.swift b/ScribeKit/Meeting/LiveTranscriptModel.swift index 97cd59e..0b3d2e0 100644 --- a/ScribeKit/Meeting/LiveTranscriptModel.swift +++ b/ScribeKit/Meeting/LiveTranscriptModel.swift @@ -70,15 +70,19 @@ final class LiveTranscriptModel { guard !segment.displayText.isEmpty else { return } finalizedSegments.append(segment) case let .interrupted(interruption): - lastInterruption = interruption switch interruption { - case let .audioDropped(seconds, _): - untranscribedSeconds += seconds + case let .audioDropped(drop): + untranscribedSeconds += drop.seconds + // A report that only carries the news that recognition caught + // up closes a gap incident and has no loss of its own; showing + // it would replace a real interruption on screen with nothing. + guard drop.seconds > 0 else { return } case .recognitionFailed: // The hypothesis was never finalised and the recogniser that // produced it has stopped, so it is not transcript material. partialSegment = nil } + lastInterruption = interruption } } diff --git a/ScribeKit/Meeting/MeetingRuntime.swift b/ScribeKit/Meeting/MeetingRuntime.swift index 8a30f5f..a0d3c28 100644 --- a/ScribeKit/Meeting/MeetingRuntime.swift +++ b/ScribeKit/Meeting/MeetingRuntime.swift @@ -141,8 +141,25 @@ final class MeetingRuntime { private(set) var transcriptSpanCount = 0 /// How many gap markers were written into it. + /// + /// One per incident, not one per observation: a recogniser that spends + /// four minutes behind capture produces one marker and this counts one. private(set) var gapCount = 0 + /// The transcription-gap incident that is still going on, if one is. + /// + /// Observations of dropped audio arrive several times a second for as + /// long as recognition is behind, and every one of them describes the same + /// condition. They are accumulated here and written to the transcript once, + /// when the pipeline says the condition is over — so a sustained backlog + /// costs the document one blockquote rather than one per half second. + /// + /// Holding one in memory is only honest because the session record knows + /// about it: ``persistence`` is told the moment an incident opens, so a + /// ScribeKit that is killed while one is going on leaves a record that says + /// so rather than taking the incident with it. + private var openGapIncident: TranscriptGapIncident? + /// How long the current meeting has been running. let elapsed: MeetingElapsedClock @@ -511,6 +528,7 @@ final class MeetingRuntime { recognitionRestartCount = 0 transcriptSpanCount = 0 gapCount = 0 + openGapIncident = nil ScribeKitLog.lifecycle.info( """ Meeting start requested: \(request.sources.count, privacy: .public) source(s), \ @@ -626,6 +644,10 @@ final class MeetingRuntime { await transcriber.stop() transcriptionState = .idle await drainPendingEvents() + // A pause ends the condition by ending the capture that fed it, so the + // incident is closed here rather than left to reach across the pause + // and be reported as one stretch of trouble that it was not. + await closeGapIncident() // Taken after the drain, so pre-pause spans keep the base they were // recognised under, and read from the media clock rather than the wall @@ -786,6 +808,11 @@ final class MeetingRuntime { /// ``SessionCompletionOutcome/failed``: an artifact that did not close is /// the more serious fact about the meeting. private func closeSession(outcome: SessionCompletionOutcome) async { + // Whatever ended the meeting also ended any gap incident that was + // going on, and the marker for it belongs in the document before the + // closing block rather than nowhere. It is written while the session is + // still open, which is the only time it can be. + await closeGapIncident() let audioFailure = finishRetainedAudio() guard persistenceState.isActive else { return } let layout = persistenceState.layout @@ -904,8 +931,17 @@ final class MeetingRuntime { case let .final(segment): await persist(segment) case let .interrupted(interruption): - if let gap = interruption.gap { await persist(gap) } - if case .recognitionFailed = interruption { await recoverRecognition() } + switch interruption { + case let .audioDropped(drop): + await accumulate(drop) + case .recognitionFailed: + // A restart is a boundary of its own: the run that lost the + // audio is being torn down, the next one counts from a new + // origin, and folding what follows into what came before would + // hide one failure inside another. + await closeGapIncident() + await recoverRecognition() + } } } @@ -928,11 +964,13 @@ final class MeetingRuntime { return .partial(rebased(segment)) case let .final(segment): return .final(rebased(segment)) - case let .interrupted(.audioDropped(seconds, startTime)): - return .interrupted(.audioDropped( - seconds: seconds, - startTime: startTime.map { $0 + mediaOffsetBase } - )) + case let .interrupted(.audioDropped(drop)): + return .interrupted(.audioDropped(DroppedAudio( + seconds: drop.seconds, + startTime: drop.startTime.map { $0 + mediaOffsetBase }, + endTime: drop.endTime.map { $0 + mediaOffsetBase }, + closesIncident: drop.closesIncident + ))) case .interrupted: return event } @@ -986,6 +1024,70 @@ final class MeetingRuntime { } } + /// Folds one observation of dropped audio into the incident it belongs to. + /// + /// Whether it belongs to the open one is the pipeline's judgement and not + /// this type's: an observation carries the evidence about whether the + /// backlog had caught up before it, so there is no delay to tune here and + /// no interface timer deciding what counts as the same trouble. What this + /// adds is the boundaries the pipeline cannot see — a pause, a recogniser + /// restart, the end of a meeting — which close an incident explicitly. + /// + /// The session record is told as soon as an incident opens, before any of + /// it is written to the transcript, so the window in which a crash could + /// lose the whole incident is the width of one metadata write rather than + /// the length of the incident. + /// + /// - Parameter drop: What the pipeline reported, already on the meeting's + /// own media timeline. + private func accumulate(_ drop: DroppedAudio) async { + if openGapIncident?.accepts(drop) == true { + openGapIncident?.extend(with: drop) + } else { + await closeGapIncident() + guard drop.seconds > 0 else { return } + openGapIncident = TranscriptGapIncident(drop) + } + guard let incident = openGapIncident else { return } + guard !incident.isClosed else { + // The observation that opened this one also ended it, so it goes + // straight into the document and the record is never told about an + // incident that was outstanding for no time at all. + await closeGapIncident() + return + } + // Repeated for every observation and written once: the store compares + // the moment against the one already recorded, so a condition lasting + // minutes costs the record a single write. + await persistence.noteOpenGapIncident(startingAt: incident.startTime) + } + + /// Writes the open gap incident to the transcript as one marker and + /// forgets it. + /// + /// Called both when the pipeline says recognition caught up and at every + /// boundary that ends the condition by ending what produced it: a pause, a + /// recogniser restart, a stop, a capture stream that died. An incident that + /// never cost any audio is dropped rather than written, because a + /// blockquote stating that nothing was lost is not transcript material. + private func closeGapIncident() async { + guard let incident = openGapIncident else { return } + openGapIncident = nil + guard !incident.isEmpty else { + await persistence.noteOpenGapIncident(startingAt: nil) + return + } + ScribeKitLog.recognition.notice( + "Transcription gap incident closed after \(incident.observationCount, privacy: .public) observation(s)" + ) + await persist(incident.gap) + // Cleared after the marker is in the file, never before. The other + // order would leave a window in which the record says nothing is + // outstanding and the transcript does not carry it either, which is + // the one outcome that loses the incident rather than repeating it. + await persistence.noteOpenGapIncident(startingAt: nil) + } + /// Writes one gap marker to the transcript. /// /// - Parameter gap: The untranscribed stretch. @@ -1027,6 +1129,10 @@ final class MeetingRuntime { "Transcript persistence failed after \(self.transcriptSpanCount, privacy: .public) span(s)" ) persistenceState = .failed(message: message, layout: persistenceState.layout) + // Nothing more can reach the transcript, so an open incident is + // dropped rather than written: a write that is already failing cannot + // be made truthful by adding one more to it. + openGapIncident = nil try? await persistence.finishSession( endedAt: now(), outcome: .failed, diff --git a/ScribeKit/Models/TranscriptGap.swift b/ScribeKit/Models/TranscriptGap.swift index 7fe3741..625b3c0 100644 --- a/ScribeKit/Models/TranscriptGap.swift +++ b/ScribeKit/Models/TranscriptGap.swift @@ -11,6 +11,15 @@ import Foundation /// speech is worse than one that says where it stopped listening. The type is /// framework-independent and carries only what the pipeline actually knows — /// a position when there is one, and never an invented one. +/// +/// A gap describes one *incident* rather than one observation. A recogniser +/// that has fallen behind loses audio repeatedly for as long as it stays +/// behind, and each of those losses is an observation of the same condition; +/// ``TranscriptGapIncident`` accumulates them and produces a single gap when +/// the condition ends. That is why the affected range and the amount of audio +/// lost are two separate facts here: the range is when the incident was going +/// on, and ``duration`` is how much audio inside it was actually discarded. +/// They are equal only for a gap that lost every second of its own range. nonisolated struct TranscriptGap: Equatable, Sendable { /// Why the audio was not transcribed. @@ -24,15 +33,27 @@ nonisolated struct TranscriptGap: Equatable, Sendable { case recognizerRestarted } - /// Seconds from the start of the run to the start of the gap, when the - /// pipeline knows where it fell. + /// Seconds from the start of the meeting to the first audio the incident + /// affected, when the pipeline knows where it fell. /// /// `nil` means only the length is known. Dropped audio carries the time of /// the buffer that was discarded; time lost to a recogniser being rebuilt /// does not, because no audio clock was running to place it against. let startTime: Double? - /// How long the untranscribed stretch lasted, in seconds. + /// Seconds from the start of the meeting to the end of the last audio the + /// incident affected, when the pipeline knows it. + /// + /// With ``startTime`` this bounds the incident: nothing outside the range + /// was affected by it. It does **not** say that everything inside the + /// range was lost — audio between two discarded stretches may have been + /// transcribed normally, and ``duration`` is what says how much was not. + let endTime: Double? + + /// How much audio inside the incident was not transcribed, in seconds. + /// + /// This is a sum over distinct, non-overlapping stretches of discarded + /// audio, never a measure of the range they fell in. let duration: Double /// What caused it. @@ -41,16 +62,38 @@ nonisolated struct TranscriptGap: Equatable, Sendable { /// Creates a gap. /// /// - Parameters: - /// - startTime: Seconds from the start of the run, or `nil` when the - /// position is not known. + /// - startTime: Seconds from the start of the meeting to the first + /// affected audio, or `nil` when the position is not known. + /// - endTime: Seconds from the start of the meeting to the end of the + /// last affected audio, or `nil` when the incident has no known extent + /// beyond its start. /// - duration: How much audio was not transcribed, in seconds. /// - reason: What caused the gap. - init(startTime: Double? = nil, duration: Double, reason: Reason) { + init(startTime: Double? = nil, endTime: Double? = nil, duration: Double, reason: Reason) { self.startTime = startTime + self.endTime = endTime self.duration = duration self.reason = reason } + /// Seconds between the first and the last audio the incident affected, + /// when both ends are known. + var affectedSpan: Double? { + guard let startTime, let endTime else { return nil } + return max(0, endTime - startTime) + } + + /// Whether the gap covers a stretch of the meeting wide enough to state as + /// a range rather than as a position. + /// + /// The threshold is a whole second because that is the resolution a + /// transcript's clock times are written at: an incident narrower than one + /// second would be rendered as a range whose two ends are the same time, + /// which says less than naming the moment it happened. A wider one is + /// written as a range, and the range is always accompanied by how much + /// audio was actually lost, because the two are different quantities. + var spansRange: Bool { (affectedSpan ?? 0) >= 1 } + /// A phrase naming the cause, used in the transcript and the interface. var reasonDescription: String { switch reason { diff --git a/ScribeKit/Models/TranscriptGapIncident.swift b/ScribeKit/Models/TranscriptGapIncident.swift new file mode 100644 index 0000000..1af6861 --- /dev/null +++ b/ScribeKit/Models/TranscriptGapIncident.swift @@ -0,0 +1,124 @@ +// +// TranscriptGapIncident.swift +// ScribeKit +// + +import Foundation + +/// One continuing condition that is costing a meeting audio, accumulated from +/// the observations the pipeline reports about it. +/// +/// A recogniser that has fallen behind capture does not lose audio once. It +/// loses the oldest buffer in the backlog, then the next, for as long as it +/// stays behind, and the pipeline reports each accumulated fraction of a +/// second as it happens. Written out one by one those reports are a truthful +/// but unreadable document: a meeting that spent four minutes behind produces +/// hundreds of near-identical blockquotes. An incident is the same information +/// said once — when it began, when it stopped, and how much audio it cost. +/// +/// The type keeps two quantities that must never be confused. ``startTime`` +/// and ``endTime`` bound *when the condition was going on*; ``lostSeconds`` is +/// how much audio inside that range was actually discarded. Between two +/// discarded stretches the recogniser may have transcribed normally, so the +/// range is an upper bound on the damage and never a statement of it. +/// +/// ``lostSeconds`` is a sum, and it is only allowed to be one because of what +/// an observation means: the bounded backlog hands back each buffer it evicts +/// exactly once, so the stretches being added are distinct audio that cannot +/// overlap. Nothing here derives a length from the range, and nothing derives +/// a range from a length. +/// +/// The incident is a value type with no I/O. What decides where one incident +/// ends and the next begins is evidence from the audio pipeline rather than a +/// delay chosen for the interface: see ``DroppedAudio/closesIncident``. +nonisolated struct TranscriptGapIncident: Equatable, Sendable { + + /// What is causing the loss. Only ``TranscriptGap/Reason/audioDropped`` + /// accumulates; a recogniser restart is a single measured event, reported + /// on its own. + let reason: TranscriptGap.Reason + + /// Seconds from the start of the meeting to the first audio this incident + /// affected, when the pipeline knew where it fell. + private(set) var startTime: Double? + + /// Seconds from the start of the meeting to the end of the last audio this + /// incident affected, when the pipeline knew where it fell. + private(set) var endTime: Double? + + /// How much audio this incident has cost, in seconds. + private(set) var lostSeconds: Double + + /// How many observations have been folded into it, which is how many + /// blockquotes the transcript would have carried without coalescing. + private(set) var observationCount: Int + + /// Whether the evidence says the condition has ended. + private(set) var isClosed: Bool + + /// Opens an incident from the first observation of a condition. + /// + /// - Parameter drop: What the pipeline reported. + init(_ drop: DroppedAudio) { + reason = .audioDropped + startTime = drop.startTime + endTime = drop.endTime + lostSeconds = drop.seconds + observationCount = 1 + isClosed = drop.closesIncident + } + + /// Folds a further observation of the same condition into the incident. + /// + /// The range only ever widens and the loss only ever grows: an observation + /// reports audio the backlog has already discarded, so nothing it carries + /// can withdraw something an earlier one established. + /// + /// - Parameter drop: What the pipeline reported. + mutating func extend(with drop: DroppedAudio) { + if let start = drop.startTime { + startTime = startTime.map { min($0, start) } ?? start + } + if let end = drop.endTime { + endTime = endTime.map { max($0, end) } ?? end + } + lostSeconds += drop.seconds + observationCount += 1 + isClosed = isClosed || drop.closesIncident + } + + /// Whether this incident should absorb an observation rather than a new + /// incident being opened for it. + /// + /// The rule is the pipeline's own evidence and nothing else. An + /// observation that follows one which closed the incident belongs to a + /// separate condition, because closure means the backlog demonstrably + /// caught up in between; an observation that does not belongs to this one. + /// There is deliberately no wall-clock or interface-driven timer here: the + /// meeting's own clocks are not what decides whether recognition recovered. + /// + /// Boundaries the pipeline cannot see — a pause, a recogniser restart, the + /// end of the meeting — are enforced by whoever owns the incident closing + /// it explicitly, never by weakening this rule. + /// + /// - Parameter drop: What the pipeline reported. + /// - Returns: `true` when the observation continues this incident. + func accepts(_ drop: DroppedAudio) -> Bool { !isClosed } + + /// Whether anything has actually been lost yet. + /// + /// A report that only carries the news that recognition caught up closes + /// an incident without adding to it, and an incident that never held any + /// loss is not transcript material. + var isEmpty: Bool { lostSeconds <= 0 } + + /// The incident as the durable gap marker it becomes. + var gap: TranscriptGap { + TranscriptGap( + startTime: startTime, + endTime: endTime, + duration: lostSeconds, + reason: reason + ) + } +} diff --git a/ScribeKit/Models/TranscriptionEvent.swift b/ScribeKit/Models/TranscriptionEvent.swift index 4c406a7..2816075 100644 --- a/ScribeKit/Models/TranscriptionEvent.swift +++ b/ScribeKit/Models/TranscriptionEvent.swift @@ -32,39 +32,129 @@ nonisolated enum TranscriptionEvent: Equatable, Sendable { } } +/// One observation of audio the bounded backlog discarded because recognition +/// had fallen behind capture. +/// +/// An observation is not an incident. A recogniser that is behind keeps losing +/// audio for as long as it stays behind, and the pipeline reports what it has +/// lost as it goes rather than waiting for the end of a condition it cannot +/// predict the length of; ``TranscriptGapIncident`` is what turns a run of +/// these back into the single fact they describe. +/// +/// ``seconds`` is authoritative and may be summed. The backlog hands each +/// evicted buffer back exactly once, so two observations never describe the +/// same audio twice. ``startTime`` and ``endTime`` bound where the loss fell; +/// the audio between them was *not* necessarily all lost, because the +/// recogniser goes on consuming buffers while it is dropping others. +nonisolated struct DroppedAudio: Equatable, Sendable { + + /// How much audio was discarded, in seconds. Distinct audio in every + /// report, so a total over reports is a real total. + let seconds: Double + + /// Seconds from the start of the run to the first audio this report + /// covers, or `nil` when the buffer carried no usable timing. + let startTime: Double? + + /// Seconds from the start of the run to the end of the last audio this + /// report covers, or `nil` when the buffer carried no usable timing. + let endTime: Double? + + /// Whether the backlog has demonstrably caught up, so no further audio can + /// be lost to the condition this report belongs to. + /// + /// This is the evidence that separates one gap incident from the next, and + /// it is an observation of the backlog rather than a delay: the queue has + /// accepted a full backlog's worth of audio without having to evict any of + /// it, which it could not have done while still full. A report that closes + /// an incident may carry no audio at all — recognition catching up is news + /// whether or not anything was left unreported when it did. + let closesIncident: Bool + + /// Creates an observation. + /// + /// - Parameters: + /// - seconds: How much audio was discarded. + /// - startTime: Where the first discarded audio fell in the run. + /// - endTime: Where the last discarded audio ended in the run. + /// - closesIncident: Whether the backlog has caught up. + init( + seconds: Double, + startTime: Double? = nil, + endTime: Double? = nil, + closesIncident: Bool = false + ) { + self.seconds = seconds + self.startTime = startTime + self.endTime = endTime + self.closesIncident = closesIncident + } +} + /// A reason some audio was not, or may not have been, transcribed. nonisolated enum TranscriptionInterruption: Equatable, Sendable { /// Recognition fell behind capture and audio was discarded to keep memory - /// bounded; the transcript has a gap of this length. + /// bounded. /// - /// `startTime` is the position of the discarded audio in the run, in - /// seconds from its first frame, when the pipeline knows it. - case audioDropped(seconds: Double, startTime: Double? = nil) + /// One of these is an observation of a continuing condition, not a gap in + /// its own right: see ``DroppedAudio``. + case audioDropped(DroppedAudio) /// The recogniser stopped with an error, described by the system. case recognitionFailed(message: String) - /// A message suitable for display. - var message: String { + /// Reports discarded audio without naming the payload type. + /// + /// A convenience for the places that describe a single loss — the + /// recogniser's own reporting path and tests — so the common case reads as + /// one call rather than two. + /// + /// - Parameters: + /// - seconds: How much audio was discarded. + /// - startTime: Where the first discarded audio fell in the run. + /// - endTime: Where the last discarded audio ended in the run. + /// - closesIncident: Whether the backlog has caught up. + /// - Returns: The interruption. + static func audioDropped( + seconds: Double, + startTime: Double? = nil, + endTime: Double? = nil, + closesIncident: Bool = false + ) -> TranscriptionInterruption { + .audioDropped(DroppedAudio( + seconds: seconds, + startTime: startTime, + endTime: endTime, + closesIncident: closesIncident + )) + } + + /// The discarded audio this interruption reports, when it reports any. + var droppedAudio: DroppedAudio? { switch self { - case let .audioDropped(seconds, _): - String(format: "Recognition fell behind; %.1f s of audio was not transcribed.", seconds) - case let .recognitionFailed(message): - "Recognition stopped: \(message)" + case let .audioDropped(drop): drop + case .recognitionFailed: nil } } - /// The interruption expressed as durable transcript material. + /// A message suitable for display, or `nil` when the interruption carries + /// nothing to tell the user about. /// - /// A recogniser that stopped is not itself a gap: the time it was down is - /// measured by whoever restarts it and reported separately, so nothing - /// here claims a length it does not know. - var gap: TranscriptGap? { + /// A report that only says recognition caught up is one of those: it + /// closes an incident and has no loss of its own to describe, and showing + /// it as an interruption would put "0.0 s of audio was not transcribed" on + /// the screen at the moment the trouble ended. + var message: String? { switch self { - case let .audioDropped(seconds, startTime): - TranscriptGap(startTime: startTime, duration: seconds, reason: .audioDropped) - case .recognitionFailed: - nil + case let .audioDropped(drop): + drop.seconds > 0 + ? String( + format: "Recognition fell behind; %.1f s of audio was not transcribed.", + drop.seconds + ) + : nil + case let .recognitionFailed(message): + "Recognition stopped: \(message)" } } } diff --git a/ScribeKit/Persistence/TranscriptMarkdownFormatter.swift b/ScribeKit/Persistence/TranscriptMarkdownFormatter.swift index 6c62bfe..0db3975 100644 --- a/ScribeKit/Persistence/TranscriptMarkdownFormatter.swift +++ b/ScribeKit/Persistence/TranscriptMarkdownFormatter.swift @@ -139,13 +139,36 @@ nonisolated struct TranscriptMarkdownFormatter: Equatable, Sendable { /// timeline rather than speech, and a gap whose position is unknown has no /// minute to head. /// + /// One marker describes one incident, however many separate losses the + /// pipeline observed while it was going on. Which of the three sentences + /// is written follows from what the incident actually knows: + /// + /// - No position at all — time lost while the recogniser was being rebuilt + /// — states the length and nothing else. + /// - A position but no width worth stating names the moment it happened. + /// - A width of a second or more is written as a range, and the range is + /// stated *beside* the amount of audio lost rather than instead of it. + /// The two are different quantities: the range is how long the meeting + /// was in trouble, and the seconds are how much audio that cost. Audio + /// inside the range was still being transcribed between the losses, so + /// nothing here may be read as saying the whole range is missing. + /// /// - Parameter gap: The untranscribed stretch. /// - Returns: Markdown ending in a blank line. func gap(_ gap: TranscriptGap) -> String { let seconds = String(format: "%.1f", gap.duration) - let position = gap.startTime.map { " around \(clock(wallClock(offset: $0), includingSeconds: true))" } ?? "" - return "> **Transcription gap:** approximately \(seconds) seconds of audio" - + "\(position) was not transcribed; \(gap.reasonDescription).\n\n" + var text = "> **Transcription gap:** approximately \(seconds) seconds of audio" + if gap.spansRange, let startTime = gap.startTime, let endTime = gap.endTime { + text += " was not transcribed between " + text += clock(wallClock(offset: startTime), includingSeconds: true) + text += " and \(clock(wallClock(offset: endTime), includingSeconds: true))" + text += "; \(gap.reasonDescription).\n\n" + return text + } + if let startTime = gap.startTime { + text += " around \(clock(wallClock(offset: startTime), includingSeconds: true))" + } + return text + " was not transcribed; \(gap.reasonDescription).\n\n" } /// The closing block, appended once when the session ends. @@ -227,6 +250,29 @@ nonisolated struct TranscriptMarkdownFormatter: Equatable, Sendable { + "so nothing was captured or transcribed after this point.\n\n" } + /// The note recovery appends for a gap incident that was still open when + /// ScribeKit stopped. + /// + /// A meeting killed mid-incident knows where the trouble started and + /// nothing else about it: not when it ended, because it was still going + /// on, and not how much audio it cost in total, because the losses were + /// still being counted. The note therefore states the one fact the session + /// record carried and refuses the other two, rather than closing the range + /// at the moment the interruption happened to be noticed. + /// + /// - Parameters: + /// - startedAt: The wall-clock moment the open incident began, as the + /// session record stored it. + /// - timeZone: The zone the time is written in. + /// - Returns: Markdown ending in a blank line. + static func unfinishedGapNotice(startedAt: Date, timeZone: TimeZone = .current) -> String { + let formatter = TranscriptMarkdownFormatter(startedAt: startedAt, timeZone: timeZone) + return "> **Transcription gap:** audio was not being fully transcribed from approximately " + + "\(formatter.clock(startedAt, includingSeconds: true)) onwards, because recognition " + + "fell behind capture. ScribeKit stopped before that ended, so how long it lasted and " + + "how much audio it cost are not known.\n\n" + } + /// The note recovery appends to a transcript whose meeting never finished. /// /// Every clause is something ScribeKit knows. It does not state when the diff --git a/ScribeKit/Transcription/AppleSpeechTranscriber.swift b/ScribeKit/Transcription/AppleSpeechTranscriber.swift index 8a41557..c207c17 100644 --- a/ScribeKit/Transcription/AppleSpeechTranscriber.swift +++ b/ScribeKit/Transcription/AppleSpeechTranscriber.swift @@ -102,9 +102,7 @@ actor AppleSpeechTranscriber: SpeechTranscribing { nonisolated func consume(_ buffer: CapturedPCMBuffer) { guard let input = input.withLock({ $0 }) else { return } if let dropped = input.append(buffer) { - publisher.publish(.interrupted( - .audioDropped(seconds: dropped.seconds, startTime: dropped.startTime) - )) + publisher.publish(.interrupted(.audioDropped(dropped))) } } @@ -203,9 +201,7 @@ actor AppleSpeechTranscriber: SpeechTranscribing { input.withLock { $0 = nil } if let dropped = run.input.takeUnreportedDrop() { - publisher.publish(.interrupted( - .audioDropped(seconds: dropped.seconds, startTime: dropped.startTime) - )) + publisher.publish(.interrupted(.audioDropped(dropped))) } run.input.close() diff --git a/ScribeKit/Transcription/TranscriptionAudioInput.swift b/ScribeKit/Transcription/TranscriptionAudioInput.swift index 3cbf061..45fb385 100644 --- a/ScribeKit/Transcription/TranscriptionAudioInput.swift +++ b/ScribeKit/Transcription/TranscriptionAudioInput.swift @@ -24,28 +24,23 @@ import Synchronization /// audio, not against when a result happened to arrive, and audio dropped from /// the backlog leaves a real gap in that timeline rather than sliding /// everything after it. +/// +/// Losses are published as observations of a continuing condition rather than +/// as separate gaps. A recogniser that is behind keeps losing audio until it is +/// not, so what the input reports is how much it has lost so far and — +/// separately, and from the backlog's own behaviour rather than from a clock — +/// the moment the backlog has caught up and the run of losses is over. Turning +/// a run of observations back into the single incident they describe is +/// ``TranscriptGapIncident``'s job, not this one's. nonisolated final class TranscriptionAudioInput: Sendable { - /// Audio the backlog discarded, and where in the run it fell. - /// - /// The position is the start time of the oldest buffer discarded since the - /// last report, taken from the buffer itself rather than from a clock, so - /// a gap is placed where the audio was and not where the report arrived. - /// It is optional because the position is only as good as the timing the - /// buffer carried. - struct DroppedAudio: Equatable, Sendable { - /// How many seconds of audio were discarded. - let seconds: Double - - /// Seconds from the start of the run to the first discarded audio. - let startTime: Double? - } - - /// Dropped audio worth reporting, in seconds. + /// How much unreported loss is worth publishing on its own. /// /// Losses are accumulated and reported once they add up, so a recogniser /// that is badly behind produces a readable statement rather than one - /// event per buffer. + /// event per buffer. It is a reporting rate, not an incident boundary: + /// what separates one gap incident from the next is + /// ``DroppedAudio/closesIncident``. private static let reportingThreshold = 0.5 /// The backlog the recogniser reads from. @@ -57,7 +52,18 @@ nonisolated final class TranscriptionAudioInput: Sendable { var droppedSeconds: Double = 0 var unreportedSeconds: Double = 0 var unreportedStart: Double? + var unreportedEnd: Double? var isOpen = true + + /// Whether audio has been evicted since the last time the backlog was + /// seen to have caught up. While this is true the losses being + /// accumulated all belong to one incident. + var isBehind = false + + /// Buffers accepted since the last eviction. The backlog catching up + /// is measured in these rather than in seconds, because the queue's + /// capacity is what a full recovery has to be measured against. + var acceptedSinceEviction = 0 } private let state: Mutex @@ -87,9 +93,9 @@ nonisolated final class TranscriptionAudioInput: Sendable { /// allocates beyond one converted buffer. /// /// - Parameter buffer: Audio as the capture system delivered it. - /// - Returns: The unreported drop, once it is worth reporting: how many - /// seconds were lost and where the loss began, in seconds from the start - /// of the run. `nil` when there is nothing worth reporting. + /// - Returns: An observation worth publishing: unreported loss once it has + /// added up, or the news that the backlog has caught up and the run of + /// losses is therefore over. `nil` when there is nothing to report. func append(_ buffer: CapturedPCMBuffer) -> DroppedAudio? { state.withLock { state -> DroppedAudio? in guard state.isOpen else { return nil } @@ -105,39 +111,84 @@ nonisolated final class TranscriptionAudioInput: Sendable { ) let evicted = queue.append(AnalyzerInput(buffer: converted, bufferStartTime: startTime)) - guard let evicted, outputSampleRate > 0 else { return nil } + guard let evicted else { + return Self.noteAccepted(in: &state, capacity: queue.capacity) + } + guard outputSampleRate > 0 else { return nil } + state.isBehind = true + state.acceptedSinceEviction = 0 let lost = Double(evicted.buffer.frameLength) / outputSampleRate state.droppedSeconds += lost state.unreportedSeconds += lost - if state.unreportedStart == nil, let evictedStart = evicted.bufferStartTime, evictedStart.isNumeric { - state.unreportedStart = evictedStart.seconds + if let evictedStart = evicted.bufferStartTime, evictedStart.isNumeric { + if state.unreportedStart == nil { state.unreportedStart = evictedStart.seconds } + state.unreportedEnd = evictedStart.seconds + lost } guard state.unreportedSeconds >= Self.reportingThreshold else { return nil } - return Self.takeDrop(from: &state) + return Self.takeDrop(from: &state, closesIncident: false) } } + /// Records a buffer the backlog had room for, and reports the backlog + /// catching up when it finally has. + /// + /// The evidence is the queue's own capacity. Appending to a full queue + /// always evicts, so a run of appends that evicted nothing is a run during + /// which the queue was never full; once that run is as long as the whole + /// backlog, the recogniser has absorbed everything it had fallen behind on + /// and the incident is over. Nothing here consults a clock: a wall-clock + /// delay would be a guess about the recogniser, and this is a measurement + /// of it. + /// + /// - Parameters: + /// - state: The input's state, with the lock held. + /// - capacity: The backlog's capacity. + /// - Returns: The closing observation, or `nil` when the run of losses is + /// not over or there was none. + private static func noteAccepted(in state: inout State, capacity: Int) -> DroppedAudio? { + guard state.isBehind else { return nil } + state.acceptedSinceEviction += 1 + guard state.acceptedSinceEviction >= capacity else { return nil } + return takeDrop(from: &state, closesIncident: true) + } + /// Takes any dropped audio that has not been reported yet. /// + /// Called as a run ends, so whatever comes back closes its incident: no + /// further audio can reach a recogniser that is being shut down. + /// /// - Returns: The unreported drop, or `nil` when there is none. func takeUnreportedDrop() -> DroppedAudio? { state.withLock { state in guard state.unreportedSeconds > 0 else { return nil } - return Self.takeDrop(from: &state) + return Self.takeDrop(from: &state, closesIncident: true) } } /// Empties the unreported drop and returns it. /// - /// - Parameter state: The input's state, with the lock held. - /// - Returns: The drop that was pending. - private static func takeDrop(from state: inout State) -> DroppedAudio { + /// - Parameters: + /// - state: The input's state, with the lock held. + /// - closesIncident: Whether the backlog has caught up, or the run is + /// ending, so nothing further can be lost to this incident. + /// - Returns: The observation that was pending. + private static func takeDrop(from state: inout State, closesIncident: Bool) -> DroppedAudio { defer { state.unreportedSeconds = 0 state.unreportedStart = nil + state.unreportedEnd = nil + if closesIncident { + state.isBehind = false + state.acceptedSinceEviction = 0 + } } - return DroppedAudio(seconds: state.unreportedSeconds, startTime: state.unreportedStart) + return DroppedAudio( + seconds: state.unreportedSeconds, + startTime: state.unreportedStart, + endTime: state.unreportedEnd, + closesIncident: closesIncident + ) } /// Stops accepting audio and ends the recogniser's input sequence. From 0a5f2dea6130c0274b71f2699749cbcd14f09529 Mon Sep 17 00:00:00 2001 From: Quang <20378quang@gmail.com> Date: Mon, 21 Sep 2026 12:34:06 -0400 Subject: [PATCH 2/4] fix: record a gap incident that was open when ScribeKit stopped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A marker's range is not known until the incident ends, so an incident held only in memory would go with the process that was killed during it. The session record now carries openGapStartedAt: set the moment an incident opens, and cleared after the marker has reached the transcript — that order, because the failure window then repeats a notice rather than losing one. Recovery states the start and refuses the rest: the end and the total were still being measured, and either would be invented. The field is additive and optional, like pausedAt, so the record's schema version is unchanged and earlier sessions read exactly as before. Noting one never fails a meeting whose transcript is intact. --- .../Persistence/MarkdownTranscriptStore.swift | 17 +++++++ .../Persistence/SessionRecoveryMetadata.swift | 50 +++++++++++++++++-- .../Persistence/SessionRecoveryService.swift | 26 ++++++++-- .../Persistence/TranscriptPersisting.swift | 19 +++++++ 4 files changed, 104 insertions(+), 8 deletions(-) diff --git a/ScribeKit/Persistence/MarkdownTranscriptStore.swift b/ScribeKit/Persistence/MarkdownTranscriptStore.swift index abc3c2e..381c9c8 100644 --- a/ScribeKit/Persistence/MarkdownTranscriptStore.swift +++ b/ScribeKit/Persistence/MarkdownTranscriptStore.swift @@ -241,6 +241,23 @@ actor MarkdownTranscriptStore: TranscriptPersisting { current = session } + func noteOpenGapIncident(startingAt startTime: Double?) async { + guard var session = current else { return } + // The record stores the moment on the wall clock rather than the media + // offset, because whoever reads it back — a later launch, after the + // process that knew this meeting's pauses is gone — has no way to map + // one onto the other. The formatter here still does. + let startedAt = startTime.map { session.formatter.wallClock(offset: $0) } + guard startedAt != session.metadata.openGapStartedAt else { return } + session.metadata = session.metadata.notingOpenGap(startedAt: startedAt) + current = session + do { + try recoveryStore.writeMetadata(session.metadata, to: session.layout) + } catch { + ScribeKitLog.persistence.error("Open transcription gap not recorded in the session record") + } + } + func recordPause(at date: Date, capturedDuration: Double) async throws { guard var session = current else { throw TranscriptPersistenceError(.noSessionInProgress) } diff --git a/ScribeKit/Persistence/SessionRecoveryMetadata.swift b/ScribeKit/Persistence/SessionRecoveryMetadata.swift index 499d319..44f87ca 100644 --- a/ScribeKit/Persistence/SessionRecoveryMetadata.swift +++ b/ScribeKit/Persistence/SessionRecoveryMetadata.swift @@ -58,9 +58,9 @@ nonisolated enum SessionRecoveryStatus: String, Codable, Sendable, CaseIterable, /// before anything else is interpreted, so a file written by a later ScribeKit /// is refused rather than misread as this one. /// -/// ``audioRetention``, ``audioPath``, ``pausedAt`` and ``capturedDuration`` -/// were added after version 1 was in use and the version was deliberately not -/// raised. All four are optional and additive: +/// ``audioRetention``, ``audioPath``, ``pausedAt``, ``capturedDuration`` and +/// ``openGapStartedAt`` were added after version 1 was in use and the version +/// was deliberately not raised. All are optional and additive: /// a record written before they existed decodes with them absent, which is the /// truth about a session that kept no audio, and a build that has never heard /// of them ignores the extra keys. Raising the version would have made @@ -136,6 +136,22 @@ nonisolated struct SessionRecoveryMetadata: Codable, Equatable, Sendable { /// when it was last written. let pausedAt: Date? + /// When a transcription-gap incident that had not finished began. + /// + /// A run of recognition-backpressure losses is summarised into a single + /// gap marker when it ends, so while one is going on the transcript does + /// not yet carry it. This is what keeps that honest across a ScribeKit + /// that never gets to finish: it is set when an incident opens and cleared + /// when the marker has been written, so a record found in progress with + /// this present describes a meeting that was losing audio when it stopped. + /// + /// Only the start is stored. The end and the total were still being + /// measured, and a record that guessed at them would be inventing the two + /// facts the incident had not established yet. `nil` in every record + /// written before gap incidents existed, and in every meeting that was not + /// in the middle of one. + let openGapStartedAt: Date? + /// Seconds of audio the meeting had captured when the record was last /// written. /// @@ -164,6 +180,7 @@ nonisolated struct SessionRecoveryMetadata: Codable, Equatable, Sendable { /// - endedAt: When ScribeKit closed the session, if it did. /// - interruptedAt: When ScribeKit recorded an interruption, if it has. /// - pausedAt: When the meeting was paused, while it still is. + /// - openGapStartedAt: When an unfinished gap incident began. /// - capturedDuration: Seconds of audio captured so far. init( schemaVersion: Int = SessionRecoveryMetadata.currentSchemaVersion, @@ -179,6 +196,7 @@ nonisolated struct SessionRecoveryMetadata: Codable, Equatable, Sendable { endedAt: Date? = nil, interruptedAt: Date? = nil, pausedAt: Date? = nil, + openGapStartedAt: Date? = nil, capturedDuration: Double? = nil ) { self.schemaVersion = schemaVersion @@ -194,6 +212,7 @@ nonisolated struct SessionRecoveryMetadata: Codable, Equatable, Sendable { self.endedAt = endedAt self.interruptedAt = interruptedAt self.pausedAt = pausedAt + self.openGapStartedAt = openGapStartedAt self.capturedDuration = capturedDuration } @@ -222,6 +241,21 @@ nonisolated struct SessionRecoveryMetadata: Codable, Equatable, Sendable { ) } + /// The same record with an unfinished gap incident noted or cleared. + /// + /// - Parameter startedAt: When the open incident began, or `nil` once its + /// marker has been written to the transcript and there is nothing + /// outstanding to report. + /// - Returns: A copy carrying the new open-gap state. + func notingOpenGap(startedAt: Date?) -> SessionRecoveryMetadata { + copy( + status: status, + endedAt: endedAt, + interruptedAt: interruptedAt, + openGapStartedAt: .some(startedAt) + ) + } + /// The same record marked as closed by ScribeKit. /// /// - Parameters: @@ -239,7 +273,11 @@ nonisolated struct SessionRecoveryMetadata: Codable, Equatable, Sendable { at endedAt: Date, capturedDuration: Double? = nil ) -> SessionRecoveryMetadata { - // A closed meeting is not paused, whatever it was doing a moment ago. + // A closed meeting is not paused, whatever it was doing a moment ago, + // and it is not in the middle of a gap incident either: every path + // that closes a session flushes an open one to the transcript first, + // so leaving that field set would have recovery report a gap the + // document already carries. // An interruption ScribeKit closed itself is dated: unlike one found // after a relaunch, this process was running and watched it happen. copy( @@ -247,6 +285,7 @@ nonisolated struct SessionRecoveryMetadata: Codable, Equatable, Sendable { endedAt: endedAt, interruptedAt: outcome == .interrupted ? endedAt : interruptedAt, pausedAt: .some(nil), + openGapStartedAt: .some(nil), capturedDuration: capturedDuration ) } @@ -289,6 +328,7 @@ nonisolated struct SessionRecoveryMetadata: Codable, Equatable, Sendable { /// - interruptedAt: The new interruption time. /// - pausedAt: The new pause time, wrapped once so that `.some(nil)` /// clears it and an omitted argument keeps the current one. + /// - openGapStartedAt: The new open-gap time, wrapped the same way. /// - capturedDuration: The new captured duration. Defaults to the /// current one. /// - Returns: The copy. @@ -297,6 +337,7 @@ nonisolated struct SessionRecoveryMetadata: Codable, Equatable, Sendable { endedAt: Date?, interruptedAt: Date?, pausedAt: Date?? = nil, + openGapStartedAt: Date?? = nil, capturedDuration: Double? = nil ) -> SessionRecoveryMetadata { SessionRecoveryMetadata( @@ -313,6 +354,7 @@ nonisolated struct SessionRecoveryMetadata: Codable, Equatable, Sendable { endedAt: endedAt, interruptedAt: interruptedAt, pausedAt: pausedAt ?? self.pausedAt, + openGapStartedAt: openGapStartedAt ?? self.openGapStartedAt, capturedDuration: capturedDuration ?? self.capturedDuration ) } diff --git a/ScribeKit/Persistence/SessionRecoveryService.swift b/ScribeKit/Persistence/SessionRecoveryService.swift index c29b6e8..e96f576 100644 --- a/ScribeKit/Persistence/SessionRecoveryService.swift +++ b/ScribeKit/Persistence/SessionRecoveryService.swift @@ -252,6 +252,14 @@ actor SessionRecoveryService { /// longer marked in progress — because another ScribeKit already handled /// it — is returned unchanged rather than annotated again. /// + /// A record that says a transcription-gap incident was still open when + /// ScribeKit stopped is reported too, because the marker summarising that + /// incident never reached the document. Nothing is invented for it: the + /// note states when the trouble began and says plainly that its end and + /// its cost are not known. ``SessionRecoveryMetadata/markingInterruption(recordedAt:)`` + /// is what stops it being said twice — the record is no longer in progress + /// afterwards, so a second pass adds nothing. + /// /// - Parameters: /// - candidate: The unfinished session, from ``scan(_:)``. /// - date: When the interruption is being recorded, which is now. It is @@ -272,10 +280,20 @@ actor SessionRecoveryService { let updated = current.markingInterruption(recordedAt: date) try store.writeMetadata(updated, to: candidate.layout) - try store.appendToTranscript( - TranscriptMarkdownFormatter.interruptionNotice(recordedAt: date, timeZone: timeZone), - at: transcriptURL - ) + // A gap incident the meeting was in the middle of is stated before + // the interruption note, because that is where it happened: it + // began while the meeting was still running and the marker for it + // never got written. It is one append with the note, so a + // transcript never gains half of this. + var text = "" + if let gapStartedAt = current.openGapStartedAt { + text += TranscriptMarkdownFormatter.unfinishedGapNotice( + startedAt: gapStartedAt, + timeZone: timeZone + ) + } + text += TranscriptMarkdownFormatter.interruptionNotice(recordedAt: date, timeZone: timeZone) + try store.appendToTranscript(text, at: transcriptURL) ScribeKitLog.recovery.info("Interruption recorded for an unfinished session") return updated } diff --git a/ScribeKit/Persistence/TranscriptPersisting.swift b/ScribeKit/Persistence/TranscriptPersisting.swift index 2c42d51..8381fa2 100644 --- a/ScribeKit/Persistence/TranscriptPersisting.swift +++ b/ScribeKit/Persistence/TranscriptPersisting.swift @@ -58,6 +58,25 @@ nonisolated protocol TranscriptPersisting: Sendable { /// or the write fails. func recordGap(_ gap: TranscriptGap) async throws + /// Notes in the session record that a transcription-gap incident is open, + /// or that the one that was open has been written to the transcript. + /// + /// A run of recognition-backpressure losses becomes one gap marker, and + /// that marker cannot be written until the incident ends. This is what + /// keeps the durable artifacts honest in the meantime: a ScribeKit that is + /// killed mid-incident leaves a record saying a gap had started, so + /// recovery can say so instead of the incident vanishing with the process. + /// + /// It is deliberately not throwing. The record is bookkeeping beside a + /// transcript that is intact either way, and failing a meeting — or + /// worsening a backlog the pipeline is already struggling with — over a + /// note about a gap would be the wrong trade. A failure is logged. + /// + /// - Parameter startTime: Seconds of captured audio from the start of the + /// meeting to the first audio the open incident affected, or `nil` once + /// the incident's marker has been written and nothing is outstanding. + func noteOpenGapIncident(startingAt startTime: Double?) async + /// Records that the user paused the meeting. /// /// The session stays open. The marker is a structural remark rather than From 2105808437c21455d0da4e9969a33593b16f078d Mon Sep 17 00:00:00 2001 From: Quang <20378quang@gmail.com> Date: Mon, 21 Sep 2026 12:34:06 -0400 Subject: [PATCH 3/4] test: cover gap-incident coalescing, boundaries and recovery Adds the regression the interval exists for: 430 half-second observations over three and a half minutes, through the real Markdown writer to a real file, leaving one blockquote stating the range and the audio lost. Also covers an isolated short gap, separate incidents with speech between them, pause and recogniser-restart boundaries, a stop with an incident still open, a marker that cannot be written at stop, the backlog-recovery evidence in TranscriptionAudioInput, the range and point wordings across a minute boundary and noon, and the open-incident record and its recovery note. --- .../FakeTranscriptPersistence.swift | 20 + .../MarkdownTranscriptStoreTests.swift | 93 +++++ ScribeKitTests/MeetingPauseResumeTests.swift | 7 +- ScribeKitTests/MeetingReliabilityTests.swift | 10 +- ScribeKitTests/MeetingRuntimeTests.swift | 14 +- .../MeetingTranscriptionGapTests.swift | 383 ++++++++++++++++++ ScribeKitTests/ReliabilityHarness.swift | 35 ++ .../SessionRecoveryMetadataTests.swift | 63 +++ .../SessionRecoveryServiceTests.swift | 56 +++ .../TranscriptGapIncidentTests.swift | 149 +++++++ .../TranscriptMarkdownFormatterTests.swift | 151 +++++++ ScribeKitTests/TranscriptSegmentTests.swift | 9 +- .../TranscriptionAudioInputTests.swift | 77 +++- 13 files changed, 1059 insertions(+), 8 deletions(-) create mode 100644 ScribeKitTests/MeetingTranscriptionGapTests.swift create mode 100644 ScribeKitTests/TranscriptGapIncidentTests.swift diff --git a/ScribeKitTests/FakeTranscriptPersistence.swift b/ScribeKitTests/FakeTranscriptPersistence.swift index addd5db..9873f44 100644 --- a/ScribeKitTests/FakeTranscriptPersistence.swift +++ b/ScribeKitTests/FakeTranscriptPersistence.swift @@ -20,6 +20,7 @@ nonisolated final class FakeTranscriptPersistence: TranscriptPersisting, @unchec case started(directory: URL) case segment(TranscriptSegment) case gap(TranscriptGap) + case openGapNoted(startTime: Double?) case paused(capturedDuration: Double) case resumed(capturedDuration: Double) case finished(SessionCompletionOutcome) @@ -58,6 +59,18 @@ nonisolated final class FakeTranscriptPersistence: TranscriptPersisting, @unchec entries.compactMap { if case let .gap(gap) = $0 { gap } else { nil } } } + /// Every open-gap note the runtime made, in order: the media offset of the + /// incident that is outstanding, or `nil` once one has been written out. + /// + /// Unlike the real store this records every call rather than only the ones + /// that change the record, so a test can see what the runtime asked for. + var openGapNotes: [Double?] { + entries.compactMap { if case let .openGapNoted(startTime) = $0 { startTime } else { nil } } + } + + /// Whether the session record currently says an incident is outstanding. + var hasOpenGapNote: Bool { openGapNotes.last.flatMap { $0 } != nil } + /// Thrown by the next `startSession`, when set. func failStart(with error: TranscriptPersistenceError) { state.withLock { $0.startError = error } @@ -118,6 +131,13 @@ nonisolated final class FakeTranscriptPersistence: TranscriptPersisting, @unchec try append(.gap(gap), isFinalized: true) } + func noteOpenGapIncident(startingAt startTime: Double?) async { + state.withLock { state in + guard state.isOpen else { return } + state.entries.append(.openGapNoted(startTime: startTime)) + } + } + func recordPause(at date: Date, capturedDuration: Double) async throws { try appendPauseEntry(.paused(capturedDuration: capturedDuration)) } diff --git a/ScribeKitTests/MarkdownTranscriptStoreTests.swift b/ScribeKitTests/MarkdownTranscriptStoreTests.swift index fdc78c9..658bb24 100644 --- a/ScribeKitTests/MarkdownTranscriptStoreTests.swift +++ b/ScribeKitTests/MarkdownTranscriptStoreTests.swift @@ -493,6 +493,99 @@ struct MarkdownTranscriptStoreTests { )) } + @Test("An incident spanning minutes is one blockquote stating a range and a loss") + func rangeGapIsWrittenOnce() async throws { + let (store, fileStore, _) = makeStore() + _ = try await start(store) + let before = fileStore.file.text + + try await store.recordGap(TranscriptGap( + startTime: 2_280, + endTime: 2_483, + duration: 37.5, + reason: .audioDropped + )) + + let written = String(fileStore.file.text.dropFirst(before.count)) + #expect(written == """ + > **Transcription gap:** approximately 37.5 seconds of audio was not transcribed \ + between 11:38:00 AM and 11:41:23 AM; recognition fell behind capture. + + + """) + // One blockquote, not one per observation. + #expect(written.components(separatedBy: "> **Transcription gap:**").count == 2) + } + + @Test("An open incident is recorded on the wall clock, and cleared once it is written") + func openIncidentIsRecorded() async throws { + let recoveryStore = FakeSessionRecoveryStore() + let (store, fileStore, _) = makeStore(recoveryStore: recoveryStore) + let layout = try await start(store) + + await store.noteOpenGapIncident(startingAt: 2_280) + + let open = try #require(recoveryStore.storedMetadata(in: layout.directory)) + #expect(open.openGapStartedAt == startedAt.addingTimeInterval(2_280)) + #expect(open.status == .inProgress) + // The record is bookkeeping: the transcript has not been touched. + #expect(!fileStore.file.text.contains("Transcription gap")) + + try await store.recordGap(TranscriptGap( + startTime: 2_280, + endTime: 2_483, + duration: 37.5, + reason: .audioDropped + )) + await store.noteOpenGapIncident(startingAt: nil) + + let cleared = try #require(recoveryStore.storedMetadata(in: layout.directory)) + #expect(cleared.openGapStartedAt == nil) + #expect(fileStore.file.text.contains("Transcription gap")) + } + + @Test("An open incident's wall-clock start accounts for time the meeting was paused") + func openIncidentStartAccountsForPauses() async throws { + let recoveryStore = FakeSessionRecoveryStore() + let (store, _, _) = makeStore(recoveryStore: recoveryStore) + let layout = try await start(store) + try await store.recordPause(at: startedAt.addingTimeInterval(40), capturedDuration: 40) + try await store.recordResume(at: startedAt.addingTimeInterval(340), capturedDuration: 40) + + await store.noteOpenGapIncident(startingAt: 50) + + let record = try #require(recoveryStore.storedMetadata(in: layout.directory)) + // Media offset 50 is ten seconds after the resume, which is five + // minutes later on the wall clock than a meeting that never paused. + #expect(record.openGapStartedAt == startedAt.addingTimeInterval(350)) + } + + @Test("A session that finishes is not recorded as still having a gap open") + func finishedSessionClearsTheOpenIncident() async throws { + let recoveryStore = FakeSessionRecoveryStore() + let (store, _, _) = makeStore(recoveryStore: recoveryStore) + let layout = try await start(store) + await store.noteOpenGapIncident(startingAt: 12) + + try await store.finishSession(endedAt: startedAt.addingTimeInterval(600)) + + let record = try #require(recoveryStore.storedMetadata(in: layout.directory)) + #expect(record.status == .completed) + #expect(record.openGapStartedAt == nil) + } + + @Test("Noting the same open incident twice writes the record once") + func openIncidentIsNotRewrittenPerObservation() async throws { + let recoveryStore = FakeSessionRecoveryStore() + let (store, _, _) = makeStore(recoveryStore: recoveryStore) + _ = try await start(store) + let writesAfterStart = recoveryStore.writes.count + + for _ in 0..<50 { await store.noteOpenGapIncident(startingAt: 12) } + + #expect(recoveryStore.writes.count == writesAfterStart + 1) + } + @Test("Appending without a session is refused") func appendNeedsASession() async { let (store, _, _) = makeStore() diff --git a/ScribeKitTests/MeetingPauseResumeTests.swift b/ScribeKitTests/MeetingPauseResumeTests.swift index 69f5c85..7cb6f78 100644 --- a/ScribeKitTests/MeetingPauseResumeTests.swift +++ b/ScribeKitTests/MeetingPauseResumeTests.swift @@ -248,7 +248,12 @@ struct MeetingPauseResumeTests { await meeting.runtime.pause() await meeting.runtime.resume() - meeting.transcriber.emit(.interrupted(.audioDropped(seconds: 0.6, startTime: 2))) + meeting.transcriber.emit(.interrupted(.audioDropped( + seconds: 0.6, + startTime: 2, + endTime: 2.6, + closesIncident: true + ))) _ = await wait { meeting.persistence.entries.contains { if case .gap = $0 { true } else { false } } } diff --git a/ScribeKitTests/MeetingReliabilityTests.swift b/ScribeKitTests/MeetingReliabilityTests.swift index a23cfe0..e3e4f55 100644 --- a/ScribeKitTests/MeetingReliabilityTests.swift +++ b/ScribeKitTests/MeetingReliabilityTests.swift @@ -171,7 +171,15 @@ struct MeetingReliabilityTests { harness.emitFinal("Never saved.") case .gap: harness.persistence.failAppends(with: TranscriptPersistenceError(.writeFailed)) - harness.transcriber.emit(.interrupted(.audioDropped(seconds: 0.8, startTime: 3))) + // The marker is written when the incident closes, so the report + // that carries the evidence of recognition catching up is the one + // that reaches the failing writer. + harness.transcriber.emit(.interrupted(.audioDropped( + seconds: 0.8, + startTime: 3, + endTime: 3.8, + closesIncident: true + ))) case .pauseMarker: harness.persistence.failPauseMarkers(with: TranscriptPersistenceError(.writeFailed)) await harness.runtime.pause() diff --git a/ScribeKitTests/MeetingRuntimeTests.swift b/ScribeKitTests/MeetingRuntimeTests.swift index 3b77902..cf7d96b 100644 --- a/ScribeKitTests/MeetingRuntimeTests.swift +++ b/ScribeKitTests/MeetingRuntimeTests.swift @@ -618,10 +618,20 @@ struct MeetingRuntimeTests { let (model, _, transcriber, persistence) = await makeMeeting() await model.start(request([meet])) - transcriber.emit(.interrupted(.audioDropped(seconds: 0.8, startTime: 12.5))) + transcriber.emit(.interrupted(.audioDropped( + seconds: 0.8, + startTime: 12.5, + endTime: 13.3, + closesIncident: true + ))) #expect(await wait { persistence.gaps.count == 1 }) - #expect(persistence.gaps.first == TranscriptGap(startTime: 12.5, duration: 0.8, reason: .audioDropped)) + #expect(persistence.gaps.first == TranscriptGap( + startTime: 12.5, + endTime: 13.3, + duration: 0.8, + reason: .audioDropped + )) } @Test("Time lost to a recogniser restart is written as a gap with no invented position") diff --git a/ScribeKitTests/MeetingTranscriptionGapTests.swift b/ScribeKitTests/MeetingTranscriptionGapTests.swift new file mode 100644 index 0000000..e51cf6f --- /dev/null +++ b/ScribeKitTests/MeetingTranscriptionGapTests.swift @@ -0,0 +1,383 @@ +// +// MeetingTranscriptionGapTests.swift +// ScribeKitTests +// + +import Foundation +import Testing +@testable import ScribeKit + +/// How a running meeting turns a stream of dropped-audio observations into +/// transcript material. +/// +/// The condition these cover is the one a real meeting produced: recognition +/// fell behind capture and stayed behind for minutes, and the pipeline +/// reported roughly half a second of lost audio every half second for the +/// whole of it. Written out one by one those reports buried the meeting under +/// hundreds of near-identical blockquotes, which is truthful and unreadable. +/// What is checked here is that the document says the same thing once, that it +/// keeps saying separate things separately, and that nothing about the +/// durability of finalised speech changed to buy it. +@MainActor +@Suite("Meeting transcription gaps") +struct MeetingTranscriptionGapTests { + + // MARK: - Coalescing + + @Test("A sustained backlog leaves one gap marker rather than hundreds") + func sustainedBacklogIsOneMarker() async throws { + let harness = ReliabilityHarness() + await harness.start() + harness.deliver(seconds: 1, count: 600) + + // 430 observations of half a second each, half a second apart: three + // and a half minutes of a recogniser that never caught up. This is the + // shape of the failure the interval exists to fix. + for step in 0..<430 { + let start = 600 + Double(step) * 0.5 + harness.dropAudio(seconds: 0.5, from: start, to: start + 0.5) + } + harness.reportRecognitionCaughtUp() + + #expect(await harness.wait { harness.writtenGaps.count == 1 }) + let gap = try #require(harness.writtenGaps.first) + #expect(gap.reason == .audioDropped) + #expect(gap.startTime == 600) + #expect(gap.endTime == 815) + #expect(abs(gap.duration - 215) < 0.001) + #expect(gap.spansRange) + #expect(harness.runtime.gapCount == 1) + // Every note names the same moment, so the record's own comparison + // turns minutes of observations into one write. + let outstanding = harness.persistence.openGapNotes.compactMap { $0 } + #expect(!outstanding.isEmpty) + #expect(Set(outstanding) == [600]) + } + + @Test("An isolated short gap is still written, as a position rather than a range") + func isolatedGapIsWritten() async throws { + let harness = ReliabilityHarness() + await harness.start() + harness.deliver(seconds: 1, count: 60) + + harness.dropAudio(seconds: 0.5, from: 30, to: 30.5, closesIncident: true) + + #expect(await harness.wait { harness.writtenGaps.count == 1 }) + let gap = try #require(harness.writtenGaps.first) + #expect(gap.duration == 0.5) + #expect(!gap.spansRange) + } + + @Test("Two incidents with recognition working in between stay two incidents") + func separateIncidentsStaySeparate() async { + let harness = ReliabilityHarness() + await harness.start() + harness.deliver(seconds: 1, count: 300) + + harness.dropAudio(seconds: 0.5, from: 10, to: 10.5) + harness.dropAudio(seconds: 0.5, from: 10.5, to: 11, closesIncident: true) + #expect(await harness.wait { harness.writtenGaps.count == 1 }) + + harness.emitFinal("Recognition was working again here.") + #expect(await harness.waitForSegments(1)) + + harness.dropAudio(seconds: 0.5, from: 200, to: 200.5) + harness.dropAudio(seconds: 0.5, from: 200.5, to: 201, closesIncident: true) + #expect(await harness.wait { harness.writtenGaps.count == 2 }) + + #expect(harness.writtenGaps.map(\.startTime) == [10, 200]) + #expect(harness.writtenGaps.map(\.endTime) == [11, 201]) + // The speech that was transcribed between them sits between them in + // the document, because every marker is appended where it belongs. + let order: [String] = harness.persistence.entries.compactMap { entry in + switch entry { + case .gap: "gap" + case .segment: "span" + default: nil + } + } + #expect(order == ["gap", "span", "gap"]) + } + + @Test("Speech finalised during an incident still reaches the file while it is open") + func speechDuringAnIncidentIsStillDurable() async { + let harness = ReliabilityHarness() + await harness.start() + harness.deliver(seconds: 1, count: 120) + + for step in 0..<40 { + let start = 10 + Double(step) * 0.5 + harness.dropAudio(seconds: 0.5, from: start, to: start + 0.5) + } + harness.emitFinal("Heard in the middle of the trouble.") + #expect(await harness.waitForSegments(1)) + #expect(harness.writtenGaps.isEmpty) + + harness.reportRecognitionCaughtUp() + #expect(await harness.wait { harness.writtenGaps.count == 1 }) + #expect(harness.persistence.segments.map(\.text) == ["Heard in the middle of the trouble."]) + } + + // MARK: - Semantic boundaries + + @Test("A pause closes the incident that was open and never reaches across it") + func pauseClosesTheIncident() async throws { + let harness = ReliabilityHarness() + await harness.start() + harness.deliver(seconds: 1, count: 60) + harness.dropAudio(seconds: 0.5, from: 20, to: 20.5) + harness.dropAudio(seconds: 0.5, from: 21, to: 21.5) + + await harness.pause(for: 600) + #expect(await harness.wait { harness.writtenGaps.count == 1 }) + await harness.resume() + harness.deliver(seconds: 1, count: 30) + harness.dropAudio(seconds: 0.5, from: 70, to: 70.5, closesIncident: true) + + #expect(await harness.wait { harness.writtenGaps.count == 2 }) + let first = try #require(harness.writtenGaps.first) + let second = try #require(harness.writtenGaps.last) + // Media time, so the ten minutes the meeting spent paused are in + // neither incident: the first ends before the pause and the second + // begins after it. + #expect(first.endTime == 21.5) + #expect(second.startTime == 70) + #expect(abs(first.duration - 1) < 0.001) + #expect(second.duration == 0.5) + // The pause marker sits between the two, where the pause happened. + let order: [String] = harness.persistence.entries.compactMap { entry in + switch entry { + case .gap: "gap" + case .paused: "pause" + case .resumed: "resume" + default: nil + } + } + #expect(order == ["gap", "pause", "resume", "gap"]) + } + + @Test("A recogniser restart closes the incident before its own gap is written") + func restartClosesTheIncident() async { + let harness = ReliabilityHarness() + await harness.start() + harness.deliver(seconds: 1, count: 60) + harness.dropAudio(seconds: 0.5, from: 20, to: 20.5) + harness.dropAudio(seconds: 0.5, from: 21, to: 21.5) + + #expect(await harness.failRecognition()) + + #expect(await harness.wait { harness.writtenGaps.count == 2 }) + #expect(harness.writtenGaps.map(\.reason) == [.audioDropped, .recognizerRestarted]) + #expect(harness.writtenGaps.last?.startTime == nil) + } + + @Test("Stopping with an incident open writes it once, before the session closes") + func stopFlushesTheOpenIncident() async throws { + let harness = ReliabilityHarness() + await harness.start() + harness.deliver(seconds: 1, count: 60) + for step in 0..<20 { + let start = 30 + Double(step) * 0.5 + harness.dropAudio(seconds: 0.5, from: start, to: start + 0.5) + } + #expect(harness.writtenGaps.isEmpty) + + await harness.stop() + + #expect(harness.writtenGaps.count == 1) + let gap = try #require(harness.writtenGaps.first) + #expect(gap.startTime == 30) + #expect(gap.endTime == 40) + #expect(abs(gap.duration - 10) < 0.001) + // The marker is in the document before the session was recorded as + // finished, not after it. + let indexOfGap = harness.persistence.entries.firstIndex { if case .gap = $0 { true } else { false } } + let indexOfFinish = harness.persistence.entries.firstIndex { + if case .finished = $0 { true } else { false } + } + #expect(indexOfGap != nil && indexOfFinish != nil && indexOfGap! < indexOfFinish!) + #expect(harness.persistence.outcomes == [.completed]) + } + + @Test("Capture ending by itself closes the incident that was open") + func captureInterruptionClosesTheIncident() async { + let harness = ReliabilityHarness() + await harness.start() + harness.deliver(seconds: 1, count: 60) + harness.dropAudio(seconds: 0.5, from: 40, to: 40.5) + + harness.capturer.interrupt(.interrupted("Meet quit")) + + #expect(await harness.wait { !harness.runtime.isRunning }) + #expect(harness.writtenGaps.count == 1) + #expect(harness.persistence.outcomes == [.interrupted]) + } + + @Test("A marker that cannot be written at stop fails the transcript rather than being claimed") + func aFailedFlushAtStopIsReported() async { + let harness = ReliabilityHarness() + await harness.start() + harness.deliver(seconds: 1, count: 60) + harness.emitFinal("Durable before the failure.") + #expect(await harness.waitForSegments(1)) + harness.dropAudio(seconds: 0.5, from: 30, to: 30.5) + #expect(await harness.wait { harness.persistence.hasOpenGapNote }) + + harness.persistence.failAppends(with: TranscriptPersistenceError(.writeFailed)) + await harness.stop() + + // The marker did not reach the file, and nothing pretends it did: the + // transcript is reported as failed and the meeting is not a completion. + #expect(harness.writtenGaps.isEmpty) + #expect(harness.runtime.persistenceState.failureMessage != nil) + #expect(!harness.persistence.outcomes.contains(.completed)) + #expect(harness.persistence.segments.map(\.text) == ["Durable before the failure."]) + #expect(!harness.persistence.isOpen) + } + + // MARK: - Surviving a ScribeKit that never finishes + + @Test("An open incident is noted in the session record and cleared when it is written") + func openIncidentIsRecorded() async { + let harness = ReliabilityHarness() + await harness.start() + harness.deliver(seconds: 1, count: 60) + + harness.dropAudio(seconds: 0.5, from: 12, to: 12.5) + #expect(await harness.wait { harness.persistence.hasOpenGapNote }) + // The record says where the trouble started and nothing else: the end + // and the total were still being measured. + #expect(harness.persistence.openGapNotes == [12]) + + harness.dropAudio(seconds: 0.5, from: 13, to: 13.5, closesIncident: true) + #expect(await harness.wait { harness.writtenGaps.count == 1 }) + #expect(!harness.persistence.hasOpenGapNote) + #expect(harness.persistence.openGapNotes.count == 2) + } + + @Test("A meeting that finishes leaves nothing outstanding for recovery to report") + func finishedMeetingLeavesNoOpenIncident() async { + let harness = ReliabilityHarness() + await harness.start() + harness.deliver(seconds: 1, count: 60) + harness.dropAudio(seconds: 0.5, from: 12, to: 12.5) + #expect(await harness.wait { harness.persistence.hasOpenGapNote }) + + await harness.stop() + + #expect(harness.writtenGaps.count == 1) + #expect(!harness.persistence.hasOpenGapNote) + // Exactly one marker, and no second one from the flush at the + // boundary: closing an incident twice would be a duplicate notice. + #expect(harness.persistence.entries.filter { if case .gap = $0 { true } else { false } }.count == 1) + } + + @Test("Recognition catching up with nothing outstanding writes no marker at all") + func catchingUpAloneWritesNothing() async { + let harness = ReliabilityHarness() + await harness.start() + harness.deliver(seconds: 1, count: 10) + + harness.reportRecognitionCaughtUp() + harness.emitFinal("Nothing was ever lost here.") + + #expect(await harness.waitForSegments(1)) + #expect(harness.writtenGaps.isEmpty) + #expect(harness.persistence.openGapNotes.isEmpty) + #expect(harness.runtime.gapCount == 0) + } + + // MARK: - The real failure, end to end + + @Test("Hundreds of half-second observations leave one blockquote in the file on disk") + func sustainedBacklogIsOneBlockquoteInTheDocument() async throws { + // The real writers against a real file: what a reader would actually + // open after a meeting that spent three and a half minutes behind. + let root = URL.temporaryDirectory.appending( + path: "scribekit-gap-\(UUID().uuidString)", + directoryHint: .isDirectory + ) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: root) } + + let zone = TimeZone(identifier: "America/New_York")! + let clock = ReliabilityHarness.Clock(start: Date(timeIntervalSinceReferenceDate: 0)) + let transcriber = FakeSpeechTranscriber() + let runtime = MeetingRuntime( + monitor: AudioCaptureActivityMonitor(minimumPublishInterval: .zero), + transcriber: transcriber, + persistence: MarkdownTranscriptStore(access: FakeSecurityScopedAccess(), timeZone: zone), + audio: FakeAudioRetention(), + elapsed: MeetingElapsedClock(now: { clock.now }, interval: nil), + processActivity: FakeMeetingActivity(), + now: { clock.now }, + makeCapturer: { FakeCapturer(consumer: $0) } + ) + await runtime.prepare() + await runtime.start(MeetingStartRequest( + title: "Gap Regression", + sources: [ReliabilityHarness.meet], + destination: root, + audioRetention: .none + )) + let layout = try #require(runtime.persistenceState.layout) + + transcriber.emit(.final(TranscriptSegment( + text: "Before the trouble started.", + startTime: 0, + endTime: 2, + state: .final, + localeIdentifier: "en-US" + ))) + for step in 0..<430 { + let start = 600 + Double(step) * 0.5 + transcriber.emit(.interrupted(.audioDropped( + seconds: 0.5, + startTime: start, + endTime: start + 0.5 + ))) + } + transcriber.emit(.final(TranscriptSegment( + text: "After it ended.", + startTime: 820, + endTime: 822, + state: .final, + localeIdentifier: "en-US" + ))) + await runtime.stop() + + let markdown = try String(contentsOf: layout.transcriptURL, encoding: .utf8) + let notices = markdown.components(separatedBy: "> **Transcription gap:**").count - 1 + #expect(notices == 1) + #expect(markdown.contains( + "> **Transcription gap:** approximately 215.0 seconds of audio was not transcribed " + + "between 7:10:00 PM and 7:13:35 PM; recognition fell behind capture." + )) + // The speech either side of it is untouched and still in order. + let document = TranscriptDocument.parse(markdown) + #expect(document.spans.map(\.text) == ["Before the trouble started.", "After it ended."]) + #expect(markdown.contains("**Ended:**")) + } + + // MARK: - What the interface is told + + @Test("Every observation still counts towards the audio the meeting could not transcribe") + func liveTotalCountsEveryObservation() async { + let harness = ReliabilityHarness() + await harness.start() + harness.deliver(seconds: 1, count: 60) + + for step in 0..<20 { + let start = 10 + Double(step) * 0.5 + harness.dropAudio(seconds: 0.5, from: start, to: start + 0.5) + } + + #expect(await harness.wait { harness.runtime.transcript.untranscribedSeconds >= 10 }) + #expect(abs(harness.runtime.transcript.untranscribedSeconds - 10) < 0.001) + // The live total is the sum of the observations; the document says the + // same number once, when the incident closes. + harness.reportRecognitionCaughtUp() + #expect(await harness.wait { harness.writtenGaps.count == 1 }) + #expect(abs((harness.writtenGaps.first?.duration ?? 0) - 10) < 0.001) + } +} diff --git a/ScribeKitTests/ReliabilityHarness.swift b/ScribeKitTests/ReliabilityHarness.swift index 914e463..ac892b2 100644 --- a/ScribeKitTests/ReliabilityHarness.swift +++ b/ScribeKitTests/ReliabilityHarness.swift @@ -215,6 +215,41 @@ final class ReliabilityHarness { return absolute } + /// Reports audio the bounded backlog discarded, the way a saturated + /// recognition run reports it. + /// + /// The offsets are given on the meeting's own timeline and converted to + /// the run-relative form a recogniser publishes, so a test states where + /// the audio was rather than where the current run happens to count from. + /// + /// - Parameters: + /// - seconds: How much audio the observation says was discarded. + /// - from: Where the first discarded audio fell, on the meeting's + /// timeline. + /// - to: Where the last discarded audio ended, on the meeting's timeline. + /// - closesIncident: Whether the backlog has caught up. + func dropAudio( + seconds: Double, + from start: Double, + to end: Double, + closesIncident: Bool = false + ) { + transcriber.emit(.interrupted(.audioDropped( + seconds: seconds, + startTime: start - runOrigin, + endTime: end - runOrigin, + closesIncident: closesIncident + ))) + } + + /// Reports that the backlog has caught up, with nothing left unreported. + func reportRecognitionCaughtUp() { + transcriber.emit(.interrupted(.audioDropped(seconds: 0, closesIncident: true))) + } + + /// The gap markers the writer accepted, in order. + var writtenGaps: [TranscriptGap] { persistence.gaps } + /// Reports the recogniser stopping by itself and waits for whatever the /// runtime decides to do about it. /// diff --git a/ScribeKitTests/SessionRecoveryMetadataTests.swift b/ScribeKitTests/SessionRecoveryMetadataTests.swift index 3e44e71..3c2d424 100644 --- a/ScribeKitTests/SessionRecoveryMetadataTests.swift +++ b/ScribeKitTests/SessionRecoveryMetadataTests.swift @@ -195,6 +195,69 @@ struct SessionRecoveryMetadataTests { #expect(decoded.status == .completed) } + // MARK: - Unfinished gap incidents + + @Test("An open gap incident survives a round trip and states only its start") + func openGapRoundTrips() throws { + let gapStartedAt = startedAt.addingTimeInterval(2_280) + let record = metadata().notingOpenGap(startedAt: gapStartedAt) + + let restored = try SessionRecoveryMetadata.decoded(from: record.encoded()) + + #expect(restored.openGapStartedAt == gapStartedAt) + #expect(restored.status == .inProgress) + #expect(restored == record) + } + + @Test("A record written before gap incidents existed carries no such key and still decodes") + func recordWithoutOpenGapDecodes() throws { + let record = metadata() + let json = try #require(String(data: try record.encoded(), encoding: .utf8)) + + #expect(!json.contains("openGapStartedAt")) + #expect(try SessionRecoveryMetadata.decoded(from: record.encoded()).openGapStartedAt == nil) + } + + @Test("An incident whose marker has been written is cleared from the record") + func openGapIsCleared() { + let record = metadata().notingOpenGap(startedAt: startedAt.addingTimeInterval(60)) + + #expect(record.notingOpenGap(startedAt: nil).openGapStartedAt == nil) + } + + @Test("Closing a session leaves nothing outstanding for recovery to report") + func closingClearsTheOpenGap() { + let record = metadata().notingOpenGap(startedAt: startedAt.addingTimeInterval(60)) + + let closed = record.closed(.completed, at: startedAt.addingTimeInterval(600)) + + #expect(closed.openGapStartedAt == nil) + #expect(closed.status == .completed) + } + + @Test("Noting an incident changes nothing else about the record") + func notingAnOpenGapChangesNothingElse() { + let record = metadata().pausing(at: startedAt.addingTimeInterval(30), capturedDuration: 30) + + let noted = record.notingOpenGap(startedAt: startedAt.addingTimeInterval(60)) + + #expect(noted.status == record.status) + #expect(noted.pausedAt == record.pausedAt) + #expect(noted.capturedDuration == record.capturedDuration) + #expect(noted.endedAt == record.endedAt) + #expect(noted.interruptedAt == record.interruptedAt) + } + + @Test("An interruption found after a relaunch keeps the gap the meeting was in the middle of") + func interruptionKeepsTheOpenGap() { + let gapStartedAt = startedAt.addingTimeInterval(60) + let record = metadata().notingOpenGap(startedAt: gapStartedAt) + + let marked = record.markingInterruption(recordedAt: startedAt.addingTimeInterval(9_000)) + + #expect(marked.openGapStartedAt == gapStartedAt) + } + @Test("The transcript is found relative to the directory the record was read from") func transcriptIsResolvedRelatively() { let moved = URL(filePath: "/Volumes/Backup/2026-08-31-training", directoryHint: .isDirectory) diff --git a/ScribeKitTests/SessionRecoveryServiceTests.swift b/ScribeKitTests/SessionRecoveryServiceTests.swift index 59ef687..01921e0 100644 --- a/ScribeKitTests/SessionRecoveryServiceTests.swift +++ b/ScribeKitTests/SessionRecoveryServiceTests.swift @@ -386,6 +386,62 @@ struct SessionRecoveryServiceTests { #expect(!note.contains("Ended")) } + @Test("A gap that was still open when ScribeKit stopped is reported, before the interruption note") + func openGapIsReported() async throws { + let store = FakeSessionRecoveryStore() + let session = directory("2026-08-31-training") + let record = metadata(title: "Training", status: .inProgress) + .notingOpenGap(startedAt: startedAt.addingTimeInterval(2_280)) + try store.addSession(session, in: destination, metadata: record) + let (service, _) = makeService(store) + let candidate = try #require(try await service.scan(destination).candidates.first) + + _ = try await service.recordInterruption(for: candidate, at: startedAt.addingTimeInterval(9_000)) + let note = try #require(store.appends.first?.text) + + // One append, so a transcript never gains half of this. + #expect(store.appends.count == 1) + #expect(note.hasPrefix("> **Transcription gap:**")) + #expect(note.contains("from approximately 11:38:00 AM onwards")) + #expect(note.contains("Session interrupted.")) + #expect(note.range(of: "Transcription gap")!.lowerBound + < note.range(of: "Session interrupted.")!.lowerBound) + // Neither the end of the incident nor its cost is invented. + #expect(note.contains("how much audio it cost are not known")) + #expect(!note.contains("seconds of audio")) + } + + @Test("A session with no open gap gains no gap notice") + func noOpenGapMeansNoGapNotice() async throws { + let store = FakeSessionRecoveryStore() + let session = directory("2026-08-31-training") + try store.addSession(session, in: destination, metadata: metadata(title: "Training", status: .inProgress)) + let (service, _) = makeService(store) + let candidate = try #require(try await service.scan(destination).candidates.first) + + _ = try await service.recordInterruption(for: candidate, at: startedAt) + + #expect(store.appends.count == 1) + #expect(store.appends.first?.text.contains("Transcription gap") == false) + } + + @Test("An open gap is reported once, not again on a second pass") + func openGapIsNotReportedTwice() async throws { + let store = FakeSessionRecoveryStore() + let session = directory("2026-08-31-training") + let record = metadata(title: "Training", status: .inProgress) + .notingOpenGap(startedAt: startedAt.addingTimeInterval(60)) + try store.addSession(session, in: destination, metadata: record) + let (service, _) = makeService(store) + let candidate = try #require(try await service.scan(destination).candidates.first) + + _ = try await service.recordInterruption(for: candidate, at: startedAt.addingTimeInterval(600)) + _ = try await service.recordInterruption(for: candidate, at: startedAt.addingTimeInterval(700)) + + #expect(store.appends.count == 1) + #expect(store.appends.filter { $0.text.contains("Transcription gap") }.count == 1) + } + @Test("A session already recorded as interrupted is not annotated a second time") func recordingIsNotRepeated() async throws { let store = FakeSessionRecoveryStore() diff --git a/ScribeKitTests/TranscriptGapIncidentTests.swift b/ScribeKitTests/TranscriptGapIncidentTests.swift new file mode 100644 index 0000000..d432ab1 --- /dev/null +++ b/ScribeKitTests/TranscriptGapIncidentTests.swift @@ -0,0 +1,149 @@ +// +// TranscriptGapIncidentTests.swift +// ScribeKitTests +// + +import Foundation +import Testing +@testable import ScribeKit + +@Suite("Transcription gap incidents") +struct TranscriptGapIncidentTests { + + /// One observation of the shape a saturated backlog reports. + /// + /// - Parameters: + /// - seconds: How much audio was discarded. + /// - start: Where the first discarded audio fell. + /// - end: Where the last discarded audio ended. + /// - closes: Whether the backlog has caught up. + /// - Returns: The observation. + private func drop( + _ seconds: Double, + from start: Double, + to end: Double, + closes: Bool = false + ) -> DroppedAudio { + DroppedAudio(seconds: seconds, startTime: start, endTime: end, closesIncident: closes) + } + + // MARK: - One incident + + @Test("A single short loss is one incident that states a position, not a range") + func isolatedGapIsAPosition() { + var incident = TranscriptGapIncident(drop(0.5, from: 120, to: 120.5, closes: true)) + + #expect(incident.isClosed) + #expect(incident.observationCount == 1) + #expect(incident.lostSeconds == 0.5) + let gap = incident.gap + #expect(gap.startTime == 120) + #expect(gap.endTime == 120.5) + #expect(gap.duration == 0.5) + #expect(gap.reason == .audioDropped) + #expect(!gap.spansRange) + // A closing report that carries no audio adds none. + incident.extend(with: DroppedAudio(seconds: 0, closesIncident: true)) + #expect(incident.lostSeconds == 0.5) + #expect(incident.gap.duration == 0.5) + } + + @Test("Hundreds of adjacent half-second losses collapse into one incident") + func adjacentObservationsCollapse() { + // The pattern a real meeting produced: a report every half second of + // captured audio, for a little over three and a half minutes. + var incident = TranscriptGapIncident(drop(0.5, from: 600, to: 600.5)) + for step in 1..<430 { + let start = 600 + Double(step) * 0.5 + #expect(incident.accepts(drop(0.5, from: start, to: start + 0.5))) + incident.extend(with: drop(0.5, from: start, to: start + 0.5)) + } + + #expect(incident.observationCount == 430) + #expect(!incident.isClosed) + let gap = incident.gap + #expect(gap.startTime == 600) + #expect(gap.endTime == 815) + #expect(abs(gap.duration - 215) < 0.000_001) + #expect(gap.spansRange) + } + + @Test("The range affected and the audio lost stay separate quantities") + func rangeIsNotTheLoss() { + // Recognition kept transcribing between the losses: three seconds of + // audio went missing out of a two-minute stretch of trouble. + var incident = TranscriptGapIncident(drop(1, from: 30, to: 31)) + incident.extend(with: drop(1, from: 90, to: 91)) + incident.extend(with: drop(1, from: 149, to: 150)) + + let gap = incident.gap + #expect(gap.duration == 3) + #expect(gap.affectedSpan == 120) + #expect(gap.spansRange) + // Nothing derives one from the other in either direction. + #expect(gap.duration != gap.affectedSpan) + } + + @Test("An observation widens the incident and never narrows it") + func extendOnlyWidens() { + var incident = TranscriptGapIncident(drop(0.5, from: 50, to: 50.5)) + incident.extend(with: drop(0.5, from: 40, to: 40.5)) + incident.extend(with: drop(0.5, from: 60, to: 60.5)) + + #expect(incident.startTime == 40) + #expect(incident.endTime == 60.5) + #expect(abs(incident.lostSeconds - 1.5) < 0.000_001) + } + + // MARK: - Separate incidents + + @Test("An incident the pipeline closed takes nothing further") + func closedIncidentAcceptsNothing() { + var incident = TranscriptGapIncident(drop(0.5, from: 10, to: 10.5)) + #expect(incident.accepts(drop(0.5, from: 11, to: 11.5))) + + incident.extend(with: drop(0.5, from: 11, to: 11.5, closes: true)) + + #expect(incident.isClosed) + // Even audio that fell immediately afterwards belongs to a new + // incident: the backlog demonstrably caught up in between, which is + // the whole of the evidence the rule rests on. + #expect(!incident.accepts(drop(0.5, from: 12, to: 12.5))) + } + + @Test("A report that only says recognition caught up carries no loss of its own") + func closingReportWithoutLossIsEmpty() { + let incident = TranscriptGapIncident(DroppedAudio(seconds: 0, closesIncident: true)) + + #expect(incident.isEmpty) + #expect(incident.isClosed) + } + + @Test("An observation with no usable position leaves the range it found") + func positionlessObservationKeepsTheRange() { + var incident = TranscriptGapIncident(drop(0.5, from: 10, to: 10.5)) + incident.extend(with: DroppedAudio(seconds: 0.5)) + + #expect(incident.startTime == 10) + #expect(incident.endTime == 10.5) + #expect(abs(incident.lostSeconds - 1) < 0.000_001) + } + + // MARK: - Rendering decisions + + @Test("A gap narrower than a written second names a moment rather than a range") + func narrowGapIsNotARange() { + let gap = TranscriptGap(startTime: 42, endTime: 42.8, duration: 0.8, reason: .audioDropped) + #expect(!gap.spansRange) + #expect(abs((gap.affectedSpan ?? 0) - 0.8) < 0.000_001) + } + + @Test("A gap with no position claims neither a range nor a span") + func positionlessGapClaimsNothing() { + let gap = TranscriptGap(duration: 2.35, reason: .recognizerRestarted) + #expect(!gap.spansRange) + #expect(gap.affectedSpan == nil) + #expect(gap.startTime == nil) + #expect(gap.endTime == nil) + } +} diff --git a/ScribeKitTests/TranscriptMarkdownFormatterTests.swift b/ScribeKitTests/TranscriptMarkdownFormatterTests.swift index 9f91c37..246ad2d 100644 --- a/ScribeKitTests/TranscriptMarkdownFormatterTests.swift +++ b/ScribeKitTests/TranscriptMarkdownFormatterTests.swift @@ -183,6 +183,157 @@ struct TranscriptMarkdownFormatterTests { """) } + @Test("A gap incident spanning minutes is written once as a range") + func rangeGapIsFormatted() { + let formatter = makeFormatter() + + // The real failure: recognition fell behind at 11:38:00 and had not + // caught up by 11:41:23, and 37.5 seconds of audio went missing in + // between. Both facts are stated, and neither stands in for the other. + let block = formatter.gap(TranscriptGap( + startTime: 2_280, + endTime: 2_483, + duration: 37.5, + reason: .audioDropped + )) + + #expect(block == """ + > **Transcription gap:** approximately 37.5 seconds of audio was not transcribed \ + between 11:38:00 AM and 11:41:23 AM; recognition fell behind capture. + + + """) + } + + @Test("A range says how much audio was lost, never that the whole range was") + func rangeDoesNotClaimTheWholeSpan() { + let formatter = makeFormatter() + + let block = formatter.gap(TranscriptGap( + startTime: 2_280, + endTime: 2_483, + duration: 37.5, + reason: .audioDropped + )) + + // 203 seconds of meeting, 37.5 seconds of audio. The sentence carries + // the second number and never the first. + #expect(block.contains("37.5 seconds of audio was not transcribed between")) + #expect(!block.contains("203")) + #expect(!block.contains("every")) + } + + @Test("A gap incident that crosses a minute boundary states both minutes") + func rangeGapCrossesAMinute() { + let formatter = makeFormatter() + + let block = formatter.gap(TranscriptGap( + startTime: 116, + endTime: 124, + duration: 5, + reason: .audioDropped + )) + + #expect(block.contains("between 11:01:56 AM and 11:02:04 AM")) + } + + @Test("A gap incident that runs through noon states the period on both ends") + func rangeGapCrossesNoon() { + // A meeting that began at 11:00 AM: the incident starts two seconds + // before noon and ends four seconds after it. + let formatter = makeFormatter() + + let block = formatter.gap(TranscriptGap( + startTime: 3_598, + endTime: 3_604, + duration: 3, + reason: .audioDropped + )) + + #expect(block.contains("between 11:59:58 AM and 12:00:04 PM")) + } + + @Test("A gap incident narrower than a written second keeps the concise wording") + func narrowGapStaysAPosition() { + let formatter = makeFormatter() + + let block = formatter.gap(TranscriptGap( + startTime: 271.4, + endTime: 272.2, + duration: 0.8, + reason: .audioDropped + )) + + #expect(block == """ + > **Transcription gap:** approximately 0.8 seconds of audio around 11:04:31 AM \ + was not transcribed; recognition fell behind capture. + + + """) + #expect(!block.contains("between")) + } + + @Test("A gap range written after a resume reads on the wall clock the meeting kept") + func rangeGapAfterAResume() { + var formatter = makeFormatter() + // 100 seconds captured, then ten minutes paused, then capture resumes. + formatter.resume(at: startedAt.addingTimeInterval(100 + 600), capturedDuration: 100) + + let block = formatter.gap(TranscriptGap( + startTime: 110, + endTime: 130, + duration: 12, + reason: .audioDropped + )) + + // Media offsets 110 and 130 are ten minutes later on the wall clock + // than they would be in a meeting that never paused, and the pause + // itself is in neither the range nor the twelve seconds. + #expect(block.contains("between 11:11:50 AM and 11:12:10 AM")) + #expect(block.contains("approximately 12.0 seconds")) + } + + @Test("A range gap does not open a minute heading either") + func rangeGapDoesNotDisturbMinuteHeadings() { + var formatter = makeFormatter() + _ = formatter.finalSegment(segment("One.", start: 0)) + + _ = formatter.gap(TranscriptGap(startTime: 100, endTime: 200, duration: 60, reason: .audioDropped)) + let next = formatter.finalSegment(segment("Two.", start: 30)) + + #expect(!next.contains("###")) + } + + // MARK: - Unfinished incidents + + @Test("A gap still open when ScribeKit stopped states its start and invents no end") + func unfinishedGapNoticeInventsNothing() { + let notice = TranscriptMarkdownFormatter.unfinishedGapNotice( + startedAt: startedAt.addingTimeInterval(2_280), + timeZone: zone + ) + + #expect(notice == """ + > **Transcription gap:** audio was not being fully transcribed from approximately \ + 11:38:00 AM onwards, because recognition fell behind capture. ScribeKit stopped \ + before that ended, so how long it lasted and how much audio it cost are not known. + + + """) + #expect(!notice.contains("seconds of audio")) + #expect(notice.hasPrefix("> ")) + } + + @Test("An unfinished gap that began in the afternoon says so") + func unfinishedGapNoticeStatesItsPeriod() { + let notice = TranscriptMarkdownFormatter.unfinishedGapNotice( + startedAt: startedAt.addingTimeInterval(3_604), + timeZone: zone + ) + + #expect(notice.contains("12:00:04 PM")) + } + @Test("A gap does not open a minute heading of its own") func gapDoesNotDisturbMinuteHeadings() { var formatter = makeFormatter() diff --git a/ScribeKitTests/TranscriptSegmentTests.swift b/ScribeKitTests/TranscriptSegmentTests.swift index d890b72..f71f2e7 100644 --- a/ScribeKitTests/TranscriptSegmentTests.swift +++ b/ScribeKitTests/TranscriptSegmentTests.swift @@ -58,7 +58,12 @@ struct TranscriptSegmentTests { @Test("An interruption explains itself in the terms the user needs") func interruptionsExplainThemselves() { - #expect(TranscriptionInterruption.audioDropped(seconds: 1.5).message.contains("1.5")) - #expect(TranscriptionInterruption.recognitionFailed(message: "boom").message.contains("boom")) + #expect(TranscriptionInterruption.audioDropped(seconds: 1.5).message?.contains("1.5") == true) + #expect(TranscriptionInterruption.recognitionFailed(message: "boom").message?.contains("boom") == true) + } + + @Test("A report that only says recognition caught up has nothing to tell the user") + func caughtUpReportHasNoMessage() { + #expect(TranscriptionInterruption.audioDropped(seconds: 0, closesIncident: true).message == nil) } } diff --git a/ScribeKitTests/TranscriptionAudioInputTests.swift b/ScribeKitTests/TranscriptionAudioInputTests.swift index 3a55991..3859651 100644 --- a/ScribeKitTests/TranscriptionAudioInputTests.swift +++ b/ScribeKitTests/TranscriptionAudioInputTests.swift @@ -63,7 +63,7 @@ struct TranscriptionAudioInputTests { func reportsDroppedAudioInAggregate() { let input = TranscriptionAudioInput(outputFormat: recogniserFormat, capacity: 2) - var reports: [TranscriptionAudioInput.DroppedAudio] = [] + var reports: [DroppedAudio] = [] for _ in 0..<100 { if let dropped = input.append(captured()) { reports.append(dropped) } } @@ -77,7 +77,7 @@ struct TranscriptionAudioInputTests { func reportsWhereAudioWasLost() { let input = TranscriptionAudioInput(outputFormat: recogniserFormat, capacity: 1) - var first: TranscriptionAudioInput.DroppedAudio? + var first: DroppedAudio? for _ in 0..<100 where first == nil { first = input.append(captured()) } @@ -88,6 +88,79 @@ struct TranscriptionAudioInputTests { #expect((first?.seconds ?? 0) >= 0.5) } + @Test("Reports made while the backlog is still full do not close an incident") + func sustainedBacklogReportsStayOpen() { + let input = TranscriptionAudioInput(outputFormat: recogniserFormat, capacity: 2) + + var reports: [DroppedAudio] = [] + for _ in 0..<100 { + if let dropped = input.append(captured()) { reports.append(dropped) } + } + + #expect(reports.count > 1) + // The queue never had room, so nothing here says recognition caught + // up: one incident is going on, however many reports describe it. + #expect(reports.allSatisfy { !$0.closesIncident }) + } + + @Test("A report says where the audio it covers began and ended") + func reportsCarryTheirRange() { + let input = TranscriptionAudioInput(outputFormat: recogniserFormat, capacity: 1) + + var first: DroppedAudio? + for _ in 0..<100 where first == nil { + first = input.append(captured()) + } + + let report = first + #expect(report?.startTime == 0) + let start = report?.startTime ?? 0 + let end = report?.endTime ?? 0 + #expect(end > start) + // The batch is contiguous audio, so its width is the audio it lost, + // to within the part of a buffer the resampler holds back when a run + // starts. Nothing widens the range beyond what was actually dropped. + #expect(end - start >= (report?.seconds ?? 0)) + #expect((end - start) - (report?.seconds ?? 0) < 0.02) + } + + @Test("The backlog absorbing a full capacity without evicting closes the incident") + func recoveryClosesTheIncident() async { + // A queue of four that is filled and overrun, and then a recogniser + // that catches up: the evidence is the queue's own behaviour, so the + // test drives the consumer rather than waiting on a clock. + let capacity = 4 + let input = TranscriptionAudioInput(outputFormat: recogniserFormat, capacity: capacity) + var reports: [DroppedAudio] = [] + for _ in 0..<60 { + if let dropped = input.append(captured()) { reports.append(dropped) } + } + #expect(!reports.isEmpty) + #expect(reports.allSatisfy { !$0.closesIncident }) + + // The recogniser takes everything that was waiting: from here the + // queue has room, so no further append evicts anything. + var iterator = input.queue.makeAsyncIterator() + while input.queue.count > 0 { _ = await iterator.next() } + + var closing: DroppedAudio? + for _ in 0.. Date: Mon, 21 Sep 2026 12:34:06 -0400 Subject: [PATCH 4/4] docs: describe transcription gaps as incidents rather than observations Explains that related recognition-backpressure losses are summarised into one incident with one marker, that the range and the audio lost are separate quantities, and that an incident still open when ScribeKit stops is recorded and reported by recovery without inventing its end or its cost. --- CHANGELOG.md | 32 +++++ CONTEXT.md | 168 ++++++++++++++++++++------ docs/internals/on-device-speech.md | 13 ++ docs/reference/limitations.md | 4 + docs/reference/transcript-format.md | 25 ++++ docs/reliability/crash-recovery.md | 16 +++ docs/reliability/failure-semantics.md | 20 ++- docs/using/live-transcription.md | 29 +++++ docs/using/recovery.md | 9 ++ 9 files changed, 279 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4c4ca5..8cf571d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,38 @@ All notable changes to this project are documented in this file. +## Unreleased + +### Changed + +- **A sustained transcription gap is one marker, not hundreds.** Recognition + falling behind capture loses audio repeatedly for as long as it stays behind, + and each loss was previously written into `transcript.md` as its own + blockquote — roughly one every half second, for as long as the condition + lasted. Those observations are now accumulated into a single incident and + written once, when the incident ends. +- **A gap marker distinguishes the range from the loss.** An incident a second + or more wide is written as `approximately N seconds of audio was not + transcribed between and `, stating how long the meeting was in + trouble and, separately, how much audio that cost. The total is a sum over + distinct discarded audio and is never derived from the range; a shorter + incident keeps the existing wording naming the moment it fell at, and a gap + with no known position still states its length alone. No loss is dropped and + no failure reporting is weakened. + +### Added + +- **Incidents are separated by evidence rather than by a delay.** A new + incident begins once the bounded backlog has demonstrably caught up, and a + pause, a recogniser restart, capture ending by itself, a stop or a + persistence failure each close the incident that was open, so two separate + spells of trouble stay two markers. +- **An unfinished incident survives a ScribeKit that does not.** The session + record notes when an open incident began, and recovery says so in the + transcript — stating the start and refusing to invent the end or the total, + neither of which had been measured. The field is additive and the record's + schema version is unchanged. + ## 0.1.0 — 2026-09-01 The first release of ScribeKit, published as source. There is no signed or diff --git a/CONTEXT.md b/CONTEXT.md index 69e963e..9495173 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -5,34 +5,34 @@ Current working state of the repository. Keep this short and current; see ## Current milestone -Interval 28 — the v0.1.0 release. No product code changed since Interval 25. -The identity frozen in Interval 26 stands: ScribeKit builds as 0.1.0, build 1, -`quang.ScribeKit`, macOS 26.5 or later, with an application icon in every -asset-catalog slot — **now human-approved as final for v0.1.0** in Finder and -the Dock. The distribution decision is settled and public: **v0.1.0 is a source -release.** No signed or notarized application is published, no disk image is -advertised, and the README, the docs site, the requirements page, the -limitations page and the releases page all say so in current-facing terms -rather than as a pending blocker. The repository now carries three screenshots -and a real 45-second demo of synthetic content, both build-from-source paths -were verified against a clean clone, and the exact v0.1.0 tag, title and -release notes are recorded below. - -Interval 28 verified the release commit end to end from a clean worktree — -full test suite, Release build, the documented ad-hoc source build, strict -documentation build, bundle metadata read off the built artifact, README -assets and links, and a secret and personal-content scan — and turned the -current-facing wording from *release candidate* to *source release*, dated the -changelog entry and stated the tag on the releases page. **Feature freeze for -v0.1.0 remains in force.** Publication itself — merge, tag, push and the -GitHub release — waits on explicit human authorization and is not done here. - -One correction to Interval 26's record: the archived executable is a -**universal `x86_64 arm64` binary**, not `arm64` alone — the project sets no -`ARCHS` and takes the standard architectures. `LC_BUILD_VERSION minos` 26.5 -still holds. Public wording therefore says *tested on Apple Silicon* and -states that Intel is untested, rather than claiming either support or an -arm64-only build. +Interval 29 — transcription-gap incidents. The first work after v0.1.0, and +unreleased: the tag, the release notes and the changelog entry for 0.1.0 are +untouched. + +A real meeting produced a transcript flooded with near-identical warnings — +one `> **Transcription gap:** approximately 0.5 seconds …` blockquote roughly +every half second, for minutes. That was truthful and unreadable. The pipeline +was reporting *observations* of a continuing condition as if each were a +separate gap. + +Gap observations are now accumulated into a **`TranscriptGapIncident`** and +written as one marker when the incident ends. A marker states two quantities +and keeps them apart: the range the incident spanned, and the audio it actually +cost. The total is a sum over distinct discarded buffers — the bounded backlog +hands each evicted buffer back exactly once — and is never derived from the +range, because recognition goes on transcribing between the losses. + +What closes an incident is evidence, not a delay. `TranscriptionAudioInput` +reports `closesIncident` once the backlog has accepted a full capacity of audio +without having to evict any of it, which it could not do while still full; +`MeetingRuntime` additionally closes one at every boundary the pipeline cannot +see — pause, recogniser restart, capture ending, stop, persistence failure. + +Because a marker cannot be written until the incident ends, an open incident is +noted in `.scribekit/session.json` as `openGapStartedAt` the moment it opens, +and cleared once the marker is in the file. Recovery reports it, stating only +the start: the end and the total were still being measured. The v0.1.0 release +state and history are unchanged. ## Current implementation @@ -40,19 +40,23 @@ arm64-only build. `MeetingSession`, `AudioCaptureState`, `TranscriptSegment` (with a `RecognitionState` of partial or final), `TranscriptionEvent` / `TranscriptionInterruption`, `TranscriptionState`, - `SpeechRecognitionAvailability`, and new for this interval `TranscriptGap`, - `TranscriptPersistenceState` and `MeetingStartRequest`; new for this - interval, `AudioRetentionState`. - `TranscriptionInterruption.audioDropped` now carries an optional - run-relative `startTime`. + `SpeechRecognitionAvailability`, `TranscriptGap`, + `TranscriptPersistenceState`, `MeetingStartRequest` and + `AudioRetentionState`; new for this interval, `TranscriptGapIncident`. + `TranscriptionInterruption.audioDropped` now carries a `DroppedAudio` + payload: the seconds lost, the run-relative start and end of the audio the + report covers, and whether the backlog has caught up. `TranscriptGap` gained + an `endTime`, so one marker can state the range an incident spanned beside + the audio it cost. - `ScribeKit/Capture/`: discovery, `CapturedPCMBuffer`, `AudioSampleConsuming` with `BroadcastingAudioSampleConsumer`, and `ScreenCaptureKitAudioCapturer`. `AudioCaptureConfiguration` now exposes `requestedFormat`, the one place that says what format capture is asked for. - `ScribeKit/Transcription/`: `SpeechTranscribing` (now with `eventTally`), `TranscriptionConfiguration` / `TranscriptionLocale`, `BoundedAudioQueue`, - `SpeechAudioConverter`, `TranscriptionAudioInput` (which now reports where - dropped audio fell), `AppleSpeechTranscriber`, + `SpeechAudioConverter`, `TranscriptionAudioInput` (which reports where + dropped audio fell and, now, when the backlog has caught up), + `AppleSpeechTranscriber`, `TranscriptionEventPublisher` / `TranscriptionEventTally`, and `SpeechAvailabilityProviding` / `SystemSpeechAvailability`. - `ScribeKit/Review/`, new for this interval: `TranscriptReviewReason` / @@ -389,6 +393,58 @@ a second, and the remainder is flushed at stop, so a badly behind recogniser produces a readable statement rather than one event per buffer. The interface states the total as untranscribed seconds. +Each report carries the start and end of the audio it covers, and each evicted +buffer is handed back exactly once — so the seconds across reports are distinct +audio and may be summed, while the range they fell in is a separate fact. The +input also reports `closesIncident` when the queue has accepted `capacity` +buffers without evicting one: appending to a full queue always evicts, so that +run of appends proves the backlog was never full across it, which is the +evidence that recognition caught up. A report that only carries that news has +zero seconds and is not shown as an interruption. + +## Gap incidents + +`TranscriptGapIncident` accumulates observations into the one condition they +describe, and `MeetingRuntime` owns the open one. The rules: + +- An observation extends the open incident unless the previous one closed it. +- A closed incident takes nothing further, so the next observation opens a new + one — separate trouble stays separate in the document. +- Pause, recogniser restart, capture ending by itself, stop and persistence + failure close the open incident explicitly. Those are boundaries the audio + pipeline cannot see, and merging across one would hide a failure. +- A `recognizerRestarted` gap is a single measured event and never coalesces. +- Closing writes one `TranscriptGap` through `recordGap`; an incident that cost + no audio is dropped rather than written. + +`TranscriptMarkdownFormatter.gap(_:)` picks the wording from what the incident +knows: a range of a second or more is written as `between … and …` with the +lost seconds stated separately, anything narrower keeps the concise `around …` +form, and an incident with no position states a length alone. + +Media offsets throughout: a pause adds nothing to an incident's range, and a +resumed incident's offsets are rebased onto the meeting's timeline before the +formatter maps them to the wall clock through its epochs. + +## Open gap incidents across a crash + +A marker cannot be written until the incident ends, so holding one only in +memory would let a killed process take it with it. `TranscriptPersisting` +therefore has a non-throwing `noteOpenGapIncident(startingAt:)`: +`MarkdownTranscriptStore` maps the media offset to the wall clock through its +own epochs and stores it as `SessionRecoveryMetadata.openGapStartedAt` — an +additive optional field, schema version deliberately unchanged, like +`pausedAt`. It is written when an incident opens and cleared **after** the +marker has reached the file, so the failure window repeats a notice rather than +losing one. `closed(_:at:capturedDuration:)` clears it too, so no finished +meeting leaves anything outstanding. + +`SessionRecoveryService.recordInterruption` prepends +`TranscriptMarkdownFormatter.unfinishedGapNotice(startedAt:timeZone:)` to the +interruption note, in the same single append. It states the start and refuses +the rest: the end and the total were still being measured when the process +died. + ## Retained audio formats Both are native, both are written incrementally by `AVAudioFile`, and the @@ -3451,6 +3507,48 @@ and the release notes have to say that plainly rather than implying a download that does not exist. Either way ScribeKit is not released, and nothing here is a tag. +## Interval 29's closing note + +**Closed by Interval 29.** A sustained recognition backlog no longer floods a +transcript. The observations the pipeline makes about it are summarised into +one incident and one marker, and the marker keeps the two quantities apart that +the old wording invited a reader to confuse — the range the trouble spanned and +the audio it actually cost. The total is a sum over distinct evicted buffers +rather than a length read off the range, so it is reported because it is known, +not because it was available. + +**What decides a boundary is evidence.** The backlog accepting a full capacity +without evicting anything is the proof that recognition caught up, and it is +the only thing that merges or separates two observations. There is no interface +delay in the rule. Everything the audio pipeline cannot see — a pause, a +recogniser restart, capture ending by itself, a stop, a persistence failure — +closes the open incident in `MeetingRuntime` instead. + +**Deliberate persistence.** A marker's range is not known until the incident +ends, so an open incident would otherwise live only in memory. It is now noted +in `.scribekit/session.json` when it opens and cleared after the marker reaches +the file — that order, so the failure window repeats a notice rather than +losing one — and recovery states the start and refuses to invent the end or the +total. The canonical Markdown is still append-only and nothing in it is +rewritten. + +**Open.** No real meeting has been driven through the new path by hand: the +evidence here is 753 unit tests over doubles and, for the regression, the real +Markdown writer against a real file. The 0.5-second reporting threshold in +`TranscriptionAudioInput` is unchanged, so a long incident still costs one +main-actor event every half second — cheap, but it is a rate nothing has +measured under a real recogniser. And an incident that ends without anything +following it — no further loss, no speech, no boundary — is closed only by the +backlog-recovery report, which needs capture to keep delivering buffers; a +meeting whose capture dies at that exact moment closes it at the boundary +instead, which is correct but later. + +Still open from earlier intervals: source disappearance during a running +capture, `ScreenCaptureKitAudioCapturer.stop()` not calling +`removeStreamOutput(_:type:)` since Interval 15, the visible-presentation cost +Interval 18 profiled, the VoiceOver-with-no-mouse human pass, and the packaged +first install that waits on a Developer ID certificate. + ## Interval 22's closing note **Closed by Interval 22.** ScribeKit can now be supported without being diff --git a/docs/internals/on-device-speech.md b/docs/internals/on-device-speech.md index c29ce44..77c7e85 100644 --- a/docs/internals/on-device-speech.md +++ b/docs/internals/on-device-speech.md @@ -35,6 +35,11 @@ A recogniser that stops by itself is restarted at most twice. Audio arriving during a restart is not transcribed and is counted as a gap, and a restarted run's spans keep the meeting's own offsets rather than starting again at zero. +A restart also closes whatever backpressure incident was open. The run that was +losing audio is being torn down and the next one counts from a new origin, so +folding losses from either side of it into one marker would hide one failure +inside another. + A recogniser that cannot be brought back ends the meeting rather than becoming a state to sit in: capture stops, the durable artifacts are closed and kept, the session is recorded as the failure it was, and every hold on the process is @@ -45,3 +50,11 @@ released. A missing model, an unsupported language, a recogniser that stops by itself, and audio that recognition fell too far behind to transcribe are all reported rather than absorbed. Transcription uncertainty is surfaced, not hidden. + +Reporting is summarised, never reduced. A backlog that lasts loses audio every +half second or so, and those observations become one incident with one marker; +the incident's total is a sum over distinct discarded audio, stated beside the +range it fell in rather than derived from it. The evidence that separates one +incident from the next is the backlog's own behaviour — it has accepted a full +capacity of audio without having to evict any — rather than a delay chosen for +readability. diff --git a/docs/reference/limitations.md b/docs/reference/limitations.md index 2e15f2f..499e436 100644 --- a/docs/reference/limitations.md +++ b/docs/reference/limitations.md @@ -89,6 +89,10 @@ are consequences of decisions, and several of them are deliberate. capture queue. - **Falling more than about three seconds behind capture drops the oldest audio** to keep memory bounded, and the lost time is reported as a gap. +- **A sustained backlog is reported as one incident, not one marker per + loss.** The marker states the range the incident spanned and, separately, how + much audio it cost; it is written when the incident ends, so it appears in + the document a moment after the trouble does. - **A recogniser that stops by itself is restarted at most twice.** Audio arriving during a restart is counted as a gap. One that cannot be brought back ends the meeting. diff --git a/docs/reference/transcript-format.md b/docs/reference/transcript-format.md index 71cfc27..cec04fc 100644 --- a/docs/reference/transcript-format.md +++ b/docs/reference/transcript-format.md @@ -29,11 +29,36 @@ Today, we are learning about closures in Swift. | `****` | The wall-clock time of the finalised span that follows. | | Plain paragraphs | Recognised speech, exactly as it was finalised. | | `> **...:**` blockquotes | ScribeKit's own structural remarks — gaps, pauses, resumes, capture interruptions. | +| `> **Transcription gap:**` | One incident of untranscribed audio, written once when the incident ends. | | `---` then footer | `**Ended:**`, `**Duration:**`, and `**Captured:**` for a meeting that was paused. | `**Duration:**` is the meeting's wall-clock length. `**Captured:**` is the length of the recording. Neither is derived from the other. +## Gap markers + +A gap marker takes one of three forms, and which one is written follows from +what the pipeline actually established: + +```markdown +> **Transcription gap:** approximately 0.8 seconds of audio around 10:01:41 AM was not transcribed; recognition fell behind capture. +> **Transcription gap:** approximately 215.0 seconds of audio was not transcribed between 11:38:00 AM and 11:41:23 AM; recognition fell behind capture. +> **Transcription gap:** approximately 2.4 seconds of audio was not transcribed; the recogniser was restarted. +``` + +The first names the moment a short loss fell at. The second is used once the +affected stretch is a second or more wide, and states two different quantities: +the **range** the incident spanned and the **seconds of audio** it cost. The +range is not a claim that everything inside it is missing — recognition carries +on between the losses, so spans from inside the range appear in the document +around the marker. The third states a length and no position, because no audio +clock was running to place it against. + +One marker means one incident, however many separate losses the pipeline +observed while it lasted. A marker never opens a minute heading, and one is +never rewritten: the range is known when the incident ends, which is when the +marker is appended. + ## Text handling Recognised text is written exactly as it was finalised, apart from trimming the diff --git a/docs/reliability/crash-recovery.md b/docs/reliability/crash-recovery.md index 57c7352..970d9e6 100644 --- a/docs/reliability/crash-recovery.md +++ b/docs/reliability/crash-recovery.md @@ -20,6 +20,22 @@ rewrite recognised speech. Reading a record back may not invent what was never written: not the moment the process stopped, not the length of a gap, and not a word of speech. +## A gap that was still open + +A transcription-gap incident is written to the transcript when it ends, so a +meeting killed in the middle of one has no marker for it. The session record +therefore notes when an open incident began, and recovery says so: + +```markdown +> **Transcription gap:** audio was not being fully transcribed from approximately 11:38:00 AM onwards, because recognition fell behind capture. ScribeKit stopped before that ended, so how long it lasted and how much audio it cost are not known. +``` + +Only the start is recorded, because only the start had been established: the +end and the total were still being measured when the process died, and stating +either would be inventing it. A meeting that closed normally has written its +marker already and leaves nothing outstanding, so the note appears only for a +meeting that genuinely stopped mid-incident, and only once. + ## What crossed the durability boundary | Survives | Does not survive | diff --git a/docs/reliability/failure-semantics.md b/docs/reliability/failure-semantics.md index 485ac35..a02d058 100644 --- a/docs/reliability/failure-semantics.md +++ b/docs/reliability/failure-semantics.md @@ -78,5 +78,21 @@ there too. Audio that was never transcribed is written into the transcript as an explicit gap marker, positioned where the audio fell when the pipeline knows and honest about the length alone when it does not. The timeline of what *was* transcribed -keeps its real offsets. See -[Live Transcription](../using/live-transcription.md). +keeps its real offsets. + +One marker describes one incident. A recogniser that stays behind capture loses +audio repeatedly, and those repeated losses are summarised into a single marker +stating the range they fell in and how much audio was actually lost — two +quantities the wording keeps apart, because audio between the losses was still +being transcribed. Nothing is merged across a boundary that would hide a +separate failure: recognition catching up, a pause, a recogniser restart, +capture ending and the meeting stopping each close the incident that was open. +Summarising changes what the document says once rather than what it reports: +no loss is dropped, and the total is a sum over distinct audio rather than a +length derived from the range. + +An incident still open when ScribeKit stops is recorded in the session record, +so recovery can state that one had started rather than losing it with the +process — without inventing the end or the total it never measured. See +[Live Transcription](../using/live-transcription.md) and +[Crash Recovery](crash-recovery.md). diff --git a/docs/using/live-transcription.md b/docs/using/live-transcription.md index 38d3929..27c8529 100644 --- a/docs/using/live-transcription.md +++ b/docs/using/live-transcription.md @@ -39,6 +39,35 @@ Two things produce a gap: A gap is positioned where the audio fell when the pipeline knows where that was, and is honest about the length alone when it does not. +## One incident, one marker + +A recogniser that has fallen behind does not lose audio once. It keeps losing +it for as long as it stays behind, and the pipeline sees that as a loss every +half second or so. Those observations are summarised into one **incident** +rather than written out one by one, so a meeting that spent four minutes behind +capture leaves one marker instead of several hundred: + +``` +> **Transcription gap:** approximately 215.0 seconds of audio was not transcribed between 11:38:00 AM and 11:41:23 AM; recognition fell behind capture. +``` + +Read that sentence as two separate facts. The **range** is how long the meeting +was in trouble; the **seconds** are how much audio that cost. They are not the +same number, and the range does not mean that everything inside it is missing: +recognition goes on transcribing between the losses, so speech from inside the +range is usually in the transcript above and below the marker. An incident +narrower than a second keeps the concise wording that names the moment instead. + +Nothing is merged that ScribeKit cannot show belongs together. A new incident +is started whenever the backlog demonstrably caught up in between, and a pause, +a recogniser restart, capture ending or the meeting stopping all close the +incident that was open — so two separate spells of trouble stay two markers. + +The marker is written when the incident ends, which is the first moment its +range is known. While one is going on, the session record beside the transcript +notes that it started, so a ScribeKit that never gets to finish still leaves +evidence of it: see [Recovery](recovery.md). + ## When recognition cannot be brought back A recogniser that has used up its restarts ends the meeting: capture stops, the diff --git a/docs/using/recovery.md b/docs/using/recovery.md index 966bb0a..a45d53e 100644 --- a/docs/using/recovery.md +++ b/docs/using/recovery.md @@ -29,6 +29,15 @@ Recovery also does not pretend a crashed meeting completed. Reading a record back may not invent what was never written: not the moment the process stopped, not the length of a gap, and not a word of speech. +## A gap that had not finished + +A transcription gap is written to the transcript as one marker when the +incident ends, so a meeting that was killed in the middle of one never got to +write it. Recording the interruption also states that an incident had started, +with the time it started and nothing else: how long it lasted and how much +audio it cost were still being measured when ScribeKit stopped. A meeting that +closed normally has already written its marker and gains no such note. + ## What survives ScribeKit preserves finalised transcript content that reached durable storage