From e18eef339f85375347236049f60d3f2c7c806bef Mon Sep 17 00:00:00 2001 From: htuzel Date: Sun, 13 Sep 2026 20:00:48 +0200 Subject: [PATCH] Keep Talk controls visible while captions scroll --- App/ConversationCoordinator.swift | 11 +++++- App/RootView.swift | 53 +++++++++++++++++-------- UITests/MuralUITests.swift | 64 +++++++++++++++++++++++++++++++ docs/build-and-test.md | 2 + verification/validation.md | 15 ++++++++ 5 files changed, 127 insertions(+), 18 deletions(-) diff --git a/App/ConversationCoordinator.swift b/App/ConversationCoordinator.swift index c48e32f5..613566d0 100644 --- a/App/ConversationCoordinator.swift +++ b/App/ConversationCoordinator.swift @@ -353,9 +353,16 @@ import MuralCore selectedTheme = language.themes.first { $0.id == "coffee" } var record = SessionRecord(languageID: "es", themeID: selectedTheme?.id, title: selectedTheme?.title) record.append(Fragment(speaker: .user, text: "Un café con leche, por favor.", startMS: 0, endMS: 2200)) - record.append(Fragment(speaker: .assistant, text: "¡Un café con leche! ¿Y algo para comer?", startMS: 2800, endMS: 6000)) + let longCaption = ProcessInfo.processInfo.arguments.contains("--preview-long-caption") + let caption = longCaption + ? String(repeating: "Podemos practicar cómo pedir un café y preguntar qué hay para comer. ", count: 12) + : "¡Un café con leche! ¿Y algo para comer?" + let meaning = longCaption + ? String(repeating: "We can practise ordering a coffee and asking what there is to eat. ", count: 12) + : "A coffee with milk! And something to eat?" + record.append(Fragment(speaker: .assistant, text: caption, startMS: 2800, endMS: 6000)) let passage = record.passages.last! - record.translations[MeaningRequest.cacheKey(revisionKey: passage.revisionKey, language: "English")] = "A coffee with milk! And something to eat?" + record.translations[MeaningRequest.cacheKey(revisionKey: passage.revisionKey, language: "English")] = meaning session = record; state = .active; outputLevel = 0.18 scheduleTranslation() } diff --git a/App/RootView.swift b/App/RootView.swift index 09fb2629..777a0d87 100644 --- a/App/RootView.swift +++ b/App/RootView.swift @@ -53,6 +53,11 @@ struct RootView: View { .task { if AudioVerification.requested { await AudioVerification.run(coordinator) } else if ProcessInfo.processInfo.arguments.contains("--ended-conversation") { coordinator.prepareEndedPreview() } + else { + #if targetEnvironment(simulator) + if ProcessInfo.processInfo.arguments.contains("--preview"), ProcessInfo.processInfo.arguments.contains("--preview-long-caption") { coordinator.prepareScreenshot(.conversation) } + #endif + } } #endif } @@ -90,25 +95,21 @@ struct TalkView: View { .accessibilityIdentifier("conversation-status") captionArea Spacer(minLength: 12) - controls - Text(coordinator.microphoneLabel).font(.caption2).foregroundStyle(MuralColor.secondary).padding(.top, 10) - .accessibilityIdentifier("microphone-status") - HStack(spacing: 24) { - if coordinator.state == .active { - Button("Type instead", systemImage: "keyboard") { typing = true } - Button("A little help", systemImage: "sparkles") { coordinator.help() } - } else if coordinator.session == nil { - Text("Reply in whichever language comes to you.").foregroundStyle(MuralColor.secondary) - } else if !coordinator.isRunning { - Button("New conversation", systemImage: "arrow.counterclockwise") { coordinator.resetConversation() } - .accessibilityIdentifier("new-conversation") - } - }.font(.caption).padding(.top, 6).padding(.bottom, 12) + if coordinator.session == nil { + Text("Reply in whichever language comes to you.").font(.caption).foregroundStyle(MuralColor.secondary) + .multilineTextAlignment(.center).padding(.bottom, 12) + } if let notice = coordinator.notice { Text(notice).font(.footnote).foregroundStyle(MuralColor.secondary).multilineTextAlignment(.center).padding(.bottom, 12) } }.padding(.horizontal, 30).frame(maxWidth: .infinity).frame(minHeight: geometry.size.height) }.scrollIndicators(.hidden) + .accessibilityIdentifier("conversation-content") + } + .safeAreaInset(edge: .bottom, spacing: 0) { + controlArea + .padding(.horizontal, 30).padding(.top, 12) + .frame(maxWidth: .infinity).background(MuralColor.cream) } .sheet(isPresented: $typing) { TypedReplyView(coordinator: coordinator) } .animation(.smooth(duration: 0.35), value: coordinator.state) @@ -117,6 +118,26 @@ struct TalkView: View { } .sheet(item: $lookup) { item in LookupView(item: item, coordinator: coordinator) } } + private var controlArea: some View { + VStack(spacing: 0) { + controls + Text(coordinator.microphoneLabel).font(.caption2).foregroundStyle(MuralColor.secondary).padding(.top, 10) + .accessibilityIdentifier("microphone-status") + ViewThatFits(in: .horizontal) { + HStack(spacing: 24) { secondaryControls } + VStack(spacing: 12) { secondaryControls } + }.font(.caption).padding(.top, 6).padding(.bottom, 12) + } + } + @ViewBuilder private var secondaryControls: some View { + if coordinator.state == .active { + Button("Type instead", systemImage: "keyboard") { typing = true } + Button("A little help", systemImage: "sparkles") { coordinator.help() } + } else if coordinator.session != nil && !coordinator.isRunning { + Button("New conversation", systemImage: "arrow.counterclockwise") { coordinator.resetConversation() } + .accessibilityIdentifier("new-conversation") + } + } private var captionArea: some View { VStack(spacing: 12) { Text(linkedCaption).font(.system(coordinator.assistantPassage == nil ? .largeTitle : .title2, design: .rounded, weight: .medium)) @@ -168,7 +189,7 @@ struct TalkView: View { VStack(spacing: 6) { Image(systemName: coordinator.store.preferences.meaningVisible ? "captions.bubble.fill" : "captions.bubble") .frame(width: 48, height: 48).modifier(SoftGlass(tint: coordinator.store.preferences.meaningVisible ? MuralColor.butter.opacity(0.7) : .white.opacity(0.4))) - Text("Meaning").font(.caption2) + Text("Meaning").font(.caption2).fixedSize(horizontal: false, vertical: true).multilineTextAlignment(.center) }.contentShape(Rectangle()) }.buttonStyle(.plain) .accessibilityLabel(coordinator.store.preferences.meaningVisible ? "Hide meaning subtitles" : "Show meaning subtitles") @@ -189,7 +210,7 @@ struct TalkView: View { Button { if coordinator.isRunning { coordinator.end() } else { transcript = coordinator.session } } label: { VStack(spacing: 6) { Image(systemName: coordinator.isRunning ? "phone.down" : "text.bubble").frame(width: 48, height: 48).modifier(SoftGlass()) - Text(coordinator.isRunning ? "End" : "Transcript").font(.caption2) + Text(coordinator.isRunning ? "End" : "Transcript").font(.caption2).fixedSize(horizontal: false, vertical: true).multilineTextAlignment(.center) }.contentShape(Rectangle()) }.buttonStyle(.plain).accessibilityLabel(coordinator.isRunning ? "End conversation" : "Conversation transcript") .disabled(coordinator.session == nil) diff --git a/UITests/MuralUITests.swift b/UITests/MuralUITests.swift index 01a1434c..e882dd36 100644 --- a/UITests/MuralUITests.swift +++ b/UITests/MuralUITests.swift @@ -132,6 +132,70 @@ final class MuralUITests: XCTestCase { let app = XCUIApplication(); app.launchArguments = ["--preview"] + (ended ? ["--ended-conversation"] : []) app.launch(); return app } + private func checkPinnedConversationControls(accessibilityText: Bool = false) { + let app = XCUIApplication() + app.launchArguments = ["--preview", "--preview-long-caption"] + if accessibilityText { + app.launchArguments += ["-UIPreferredContentSizeCategoryName", "UICTContentSizeCategoryAccessibilityXXXL"] + } + app.launch() + let microphone = app.buttons["start-conversation"] + XCTAssertTrue(microphone.waitForExistence(timeout: 10)) + let end = app.buttons["End conversation"] + XCTAssertTrue(end.waitForExistence(timeout: 5)) + let type = app.buttons["Type instead"] + let help = app.buttons["A little help"] + let pinned = [microphone, end, type, help] + for button in pinned { XCTAssertTrue(button.isHittable, button.label) } + let frames = pinned.map(\.frame) + func assertPinned() { + for (button, frame) in zip(pinned, frames) { + XCTAssertTrue(button.isHittable, button.label) + XCTAssertEqual(button.frame.midY, frame.midY, accuracy: 1, button.label) + } + } + let hideMeaning = app.buttons["Hide meaning subtitles"] + let showMeaning = app.buttons["Show meaning subtitles"] + XCTAssertEqual(hideMeaning.value as? String, "On") + hideMeaning.tap() + XCTAssertTrue(showMeaning.waitForExistence(timeout: 5), "Tapping the meaning control must turn subtitles off") + XCTAssertEqual(showMeaning.value as? String, "Off") + assertPinned() + // Exercise the label as well as the icon; both belong to the same button. + showMeaning.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.93)).tap() + XCTAssertTrue(hideMeaning.waitForExistence(timeout: 5), "Tapping the meaning label must turn subtitles on") + XCTAssertEqual(hideMeaning.value as? String, "On") + assertPinned() + + let content = app.scrollViews["conversation-content"] + XCTAssertTrue(content.exists) + let caption = app.staticTexts["target-caption"] + let captionY = caption.frame.minY + // The scroll view's accessibility frame can extend beneath the safe-area + // footer. Drag inside the visible content, above the pinned controls. + let top = max(content.frame.minY, app.navigationBars.firstMatch.frame.maxY) + 16 + let bottom = min(content.frame.maxY, hideMeaning.frame.minY - 12) - 16 + XCTAssertGreaterThan(bottom - top, 30) + let origin = app.coordinate(withNormalizedOffset: .zero) + let start = origin.withOffset(CGVector(dx: content.frame.midX, dy: bottom)) + let finish = origin.withOffset(CGVector(dx: content.frame.midX, dy: top)) + for _ in 0..<3 { start.press(forDuration: 0.05, thenDragTo: finish) } + XCTAssertLessThan(caption.frame.minY, captionY) + assertPinned() + let screen = XCTAttachment(screenshot: app.screenshot()) + screen.name = accessibilityText ? "Pinned controls - largest accessibility text" : "Pinned controls - long caption" + screen.lifetime = .keepAlways; add(screen) + + type.tap() + XCTAssertTrue(app.textFields.firstMatch.waitForExistence(timeout: 5) || app.textViews.firstMatch.exists) + app.buttons["Close"].tap() + XCTAssertTrue(type.waitForExistence(timeout: 5)) + assertPinned() + } + func testLongCaptionKeepsConversationControlsPinned() { checkPinnedConversationControls() } + func testLongCaptionKeepsConversationControlsPinnedAtLargestAccessibilityTextSize() { + checkPinnedConversationControls(accessibilityText: true) + } func testGreetingAndMeaningToggle() { let app = launch() XCTAssertTrue(app.staticTexts["target-caption"].waitForExistence(timeout: 10)) diff --git a/docs/build-and-test.md b/docs/build-and-test.md index f5e7855b..3b211130 100644 --- a/docs/build-and-test.md +++ b/docs/build-and-test.md @@ -30,6 +30,8 @@ In **Product → Scheme → Edit Scheme → Run → Arguments**, add `--preview` Remove preview arguments before testing normal persistence. For actual speech, [install on an iPhone](run-on-iphone.md) and use the key saved through Settings. +For the pinned Talk controls regression, launch a Debug simulator with `--preview --preview-long-caption`. This seeds an active conversation with long Spanish captions and cached English meanings, without an API key. Toggle Meaning, scroll the text, and open and dismiss Type instead; the controls should stay in place. Repeat at the largest accessibility text size. Do not combine this interactive fixture with `--screenshot=conversation`, which prepares static screenshot state in the root view initializer. + ## Update the generated project After adding or removing files under `App/`, run: diff --git a/verification/validation.md b/verification/validation.md index a49e42aa..e65c6fef 100644 --- a/verification/validation.md +++ b/verification/validation.md @@ -177,3 +177,18 @@ Visual review after the full UI run found that the fixed consent footer crowded After the owner unlocked the phone, the final build launched normally at 16:37:51 CEST. Its running process was confirmed. This reopened the persistent learning store without verification arguments. The device results verify the application/provider paths with synthetic typed input and real voice output. They do not verify recognition of a human speaker, pronunciation, tones, correction quality, unscripted interruptions, headphones or cellular operation. The proficient-speaker checks requested in issues #10–#13 remain open. Pinyin uses dictionary tones and may need correction for names, ambiguous words and connected-speech tone changes. The Android contribution is not integrated in this checkout, so there is no generated Android language catalog to update here. + + +## Pinned Talk controls with long captions + +13 September 2026, 19:59 CEST + +Long captions previously pushed the microphone and conversation actions offscreen. Talk now keeps the controls in a bottom safe-area inset while captions, notices and the idle instruction scroll. Secondary actions stack when needed; Meaning and Transcript labels wrap at large text sizes. + +- The long-caption regression failed against the original Talk layout because the microphone was not hittable, then passed with the pinned footer. +- **70 offline Core tests passed**. Native Debug simulator builds passed using Xcode 26.5, iOS 26.5 and arm64. +- **Both new UI tests passed on iPhone 17 and iPhone SE (3rd generation)**, at standard and largest accessibility text sizes. They verify actual Meaning state changes from icon and label taps, caption scrolling, stable action positions and typed-reply sheet dismissal. +- Greeting/Meaning and ended-conversation/manual-reset checks passed on SE again after the final idle-instruction adjustment. Existing automatic-reset/open-transcript and Mandarin transcript/pinyin checks also passed on iPhone 17 during this change. +- Simulator screenshot review confirmed readable scrolled captions and visible actions at both text sizes, and full Meaning/Transcript labels in the small-screen ended state. At the largest text size, the controls occupy more space and reading captions requires scrolling. + +The interactive long-caption fixture uses temporary preview storage and cached translations. It is seeded from the debug simulator task rather than the static screenshot initializer, so changing Meaning does not reset its state. These checks made no API calls and do not verify live microphone input, streaming audio or provider behavior. Audio, teaching logic, persistence, account settings and the proposed Android client are unchanged.