From e52a594338024d128d9465069d4869969b3e1550 Mon Sep 17 00:00:00 2001 From: Johnny IV Young Ospino Date: Tue, 6 Oct 2026 19:47:25 -0500 Subject: [PATCH 1/3] test(ui): profile documentation export phases without reducing coverage --- Makefile | 6 ++- UITests/DocumentationExportUITests.swift | 17 ++++++ docs/DOCUMENTATION-UI-PROFILING.md | 57 ++++++++++++++++++++ scripts/summarize-documentation-ui.py | 67 ++++++++++++++++++++++++ 4 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 docs/DOCUMENTATION-UI-PROFILING.md create mode 100644 scripts/summarize-documentation-ui.py diff --git a/Makefile b/Makefile index 9d1c1b60..bc6f2a31 100644 --- a/Makefile +++ b/Makefile @@ -407,7 +407,7 @@ format: $(SWIFTFORMAT) format --in-place --recursive Vitrine VitrineDomain VitrineRendering VitrineCLI VitrineMenuBarHelper DomainTests RenderingTests RepositoryTests Tests UITests ## lint: lint Swift sources and tracked repository metadata (fails on issues) -lint: sanitizer-check test-lanes-check golden-check hygiene project-version-check bump-check perf-check coverage-check swift-features-check build-boundaries-check informational-update-check release-promotion-check commit-ci-check qa-handoff-check memory-smoke-check ui-test-preflight-check screenshot-tour-check +lint: documentation-ui-profile-check sanitizer-check test-lanes-check golden-check hygiene project-version-check bump-check perf-check coverage-check swift-features-check build-boundaries-check informational-update-check release-promotion-check commit-ci-check qa-handoff-check memory-smoke-check ui-test-preflight-check screenshot-tour-check $(SWIFTFORMAT) lint --strict --recursive Vitrine VitrineDomain VitrineRendering VitrineCLI VitrineMenuBarHelper DomainTests RenderingTests RepositoryTests Tests UITests ## hygiene: reject private planning identifiers and tracked planning artifacts @@ -449,3 +449,7 @@ test-web-capture: project web-capture-check python3 scripts/web-capture-fixture-test.py env DEVELOPER_DIR="$(XCODE_DEVELOPER)" python3 scripts/test-web-capture.py \ --output "$(WEB_CAPTURE_OUTPUT)" $(WEB_CAPTURE_HOST_FLAGS) + +.PHONY: documentation-ui-profile-check +documentation-ui-profile-check: + python3 scripts/summarize-documentation-ui.py --self-test diff --git a/UITests/DocumentationExportUITests.swift b/UITests/DocumentationExportUITests.swift index 842c92a7..8fafceff 100644 --- a/UITests/DocumentationExportUITests.swift +++ b/UITests/DocumentationExportUITests.swift @@ -21,6 +21,15 @@ final class DocumentationExportUITests: XCTestCase { for language in ["en", "es"] { for dark in [false, true] { for request in requests { + let state = "\(language)-\(dark ? "dark" : "light")-\(request.label)" + var segmentStarted = ProcessInfo.processInfo.systemUptime + func recordSegment(_ phase: String) { + let now = ProcessInfo.processInfo.systemUptime + print( + "DOCUMENTATION_UI_SEGMENT state=\(state) phase=\(phase) " + + "seconds=\(now - segmentStarted)") + segmentStarted = now + } let robot = VitrineAppRobot(testCase: self, suitePrefix: "documentation-export") let app = robot.launch( arguments: VitrineLaunchArguments.editor + [ @@ -30,6 +39,7 @@ final class DocumentationExportUITests: XCTestCase { environment: [ "VITRINE_UI_TEST_EDITOR_VIEWPORT": request.viewport ]) + recordSegment("launch") defer { app.terminate() } let window = element("editor-window", in: app) XCTAssertTrue(window.waitForExistence(timeout: 8), app.debugDescription) @@ -52,6 +62,7 @@ final class DocumentationExportUITests: XCTestCase { XCTAssertTrue(output.waitForExistence(timeout: 8), app.debugDescription) reveal("inspector-disclosure-output", in: app) output.click() + recordSegment("navigation") let description = element("alternative-text-field", in: app) XCTAssertTrue(description.waitForExistence(timeout: 3), app.debugDescription) reveal("alternative-text-field", in: app) @@ -67,6 +78,7 @@ final class DocumentationExportUITests: XCTestCase { predicate: descriptionAccepted, object: nil) ], timeout: 3), .completed, "The description must be entered in its own field") + recordSegment("description") let options = element("copy-options-menu", in: app) XCTAssertTrue(options.wait(for: \.isHittable, toEqual: true, timeout: 3)) options.click() @@ -85,6 +97,7 @@ final class DocumentationExportUITests: XCTestCase { screenshot.name = name screenshot.lifetime = .keepAlways add(screenshot) + recordSegment("package-setup") element("documentation-export", in: app).click() app.typeKey("g", modifierFlags: [.command, .shift]) app.typeText(parent.path) @@ -96,10 +109,12 @@ final class DocumentationExportUITests: XCTestCase { let choose = panel.buttons["OKButton"] XCTAssertTrue(choose.waitForExistence(timeout: 5), app.debugDescription) XCTAssertTrue(choose.wait(for: \.isHittable, toEqual: true, timeout: 5)) + recordSegment("panel") choose.click() XCTAssertTrue( element("documentation-cancel", in: app).waitForNonExistence(timeout: 10), app.debugDescription) + recordSegment("export") let directory = parent.appendingPathComponent(name) XCTAssertEqual( Set(try FileManager.default.contentsOfDirectory(atPath: directory.path)), @@ -114,7 +129,9 @@ final class DocumentationExportUITests: XCTestCase { let source = try String( contentsOf: directory.appendingPathComponent("source.txt"), encoding: .utf8) XCTAssertFalse(source.contains("Documentation example"), source) + recordSegment("validation") app.terminate() + recordSegment("termination") } } } diff --git a/docs/DOCUMENTATION-UI-PROFILING.md b/docs/DOCUMENTATION-UI-PROFILING.md new file mode 100644 index 00000000..48f20b0c --- /dev/null +++ b/docs/DOCUMENTATION-UI-PROFILING.md @@ -0,0 +1,57 @@ +# Documentation export UI profiling + +The documentation-package test deliberately covers eight states: English/Spanish, +light/dark appearance, and minimum/1280×800 windows. Each state launches an isolated +app, enters alternative text through accessibility, configures a package, visits the +real folder panel, exports, and verifies image.png, README.md, index.html and source.txt. +The separate cancellation test remains part of the ordinary UI suite. + +## Diagnostic evidence + +The test emits `DOCUMENTATION_UI_SEGMENT` lines using monotonic system uptime: + +- launch: isolated app creation, launch and activation +- navigation: editor geometry and output-inspector navigation +- description: alternative-text entry and acceptance +- package-setup: menu, sheet, folder-name entry, output selections and screenshot +- panel: export click, Go to Folder typing, and localized panel readiness +- export: panel confirmation until the export sheet disappears +- validation: all four files and their text/alternative-text content +- termination: explicit app termination + +Each segment includes automation overhead. In particular, `export` is not a pure +renderer benchmark. Assertions, timeouts and test selection are unchanged. These +markers add measurement overhead and do not themselves optimize the test. + +Save the complete ordinary-UI job log for one OS and one attempt, then run: + +```sh +python3 scripts/summarize-documentation-ui.py job.log > timing.json +python3 scripts/summarize-documentation-ui.py --self-test +``` + +The summary fails closed on missing states/phases, duplicates, unknown labels and +nonfinite/negative durations. Do not combine two OS jobs or reruns in one input. +Passing timing parsing is not proof that the entire UI job passed; retain the exact +source head, run/job URL, Xcode/image identity, xcresult and terminal job conclusion. +A failed/incomplete test must not become a successful performance observation. + +## Controlled qualification before optimization claims + +First collect instrumentation-only baseline evidence. Then change only the measured +bottleneck while keeping eight states, real panel interactions, typing acceptance, +all file checks, cancellation and screenshot coverage intact. Compare matched base +and candidate runs on the same OS, Xcode version and runner image. Record queue time +separately from executed job, step and test durations. Five matched pairs per OS are +the qualification target; fewer pairs are diagnostic and remain unqualified. Do not +claim a 20–30% improvement from historic runs across different source commits. + +Keep Tahoe strict pixel-golden comparison separate from Sequoia render-only smoke. +Do not drop OS/locale states, relax timeout or performance/coverage floors, suppress +unexpected skips, reuse instrumented coverage products for performance, or share +build products across SDK/configuration/signing/sandbox boundaries. + +The performance lane's compile/link overhead is a separate hypothesis. Inspect the +actual build graph and configuration differences before proposing reuse. Existing +lane-partition checks already prevent duplicated performance/golden coverage work; +that existing behavior is not a new optimization. diff --git a/scripts/summarize-documentation-ui.py b/scripts/summarize-documentation-ui.py new file mode 100644 index 00000000..a10c950b --- /dev/null +++ b/scripts/summarize-documentation-ui.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Summarize complete eight-state documentation UI timings; never infer a speedup.""" +import argparse +import itertools +import json +import math +import re +from pathlib import Path + +STATES = {f"{locale}-{appearance}-{size}" for locale, appearance, size in + itertools.product(("en", "es"), ("light", "dark"), ("minimum", "1280"))} +PHASES = ("launch", "navigation", "description", "package-setup", "panel", "export", "validation", "termination") +PATTERN = re.compile(r"DOCUMENTATION_UI_SEGMENT state=(\S+) phase=(\S+) seconds=(\S+)") + + +def summarize(text): + states = {state: {} for state in STATES} + for state, phase, raw in PATTERN.findall(text): + if state not in STATES or phase not in PHASES: + raise ValueError(f"Unknown state or phase: {state}/{phase}") + if phase in states[state]: + raise ValueError(f"Duplicate timing: {state}/{phase}; provide one job attempt only") + seconds = float(raw) + if not math.isfinite(seconds) or seconds < 0: + raise ValueError(f"Invalid duration: {raw}") + states[state][phase] = seconds + for state, phases in states.items(): + if set(phases) != set(PHASES): + raise ValueError(f"Incomplete state {state}: missing {sorted(set(PHASES) - phases.keys())}") + totals = {phase: sum(states[state][phase] for state in STATES) for phase in PHASES} + return {"states": dict(sorted(states.items())), "phase_seconds": totals, + "total_seconds": sum(totals.values()), "state_count": len(states), + "qualification": "diagnostic timings only; no controlled performance improvement claim"} + + +def self_test(): + valid = "\n".join(f"DOCUMENTATION_UI_SEGMENT state={s} phase={p} seconds=1.25" + for s in sorted(STATES) for p in PHASES) + assert summarize(valid)["total_seconds"] == 80 + for invalid in ("", valid.split("\n", 1)[1], valid + "\n" + valid.splitlines()[0], + valid.replace("seconds=1.25", "seconds=nan", 1), + valid.replace("seconds=1.25", "seconds=-1", 1), + valid.replace("state=en-dark-1280", "state=unknown", 1), + valid.replace("phase=launch", "phase=unknown", 1)): + try: + summarize(invalid) + except ValueError: + pass + else: + raise AssertionError("Malformed/incomplete timing evidence was accepted") + print("documentation UI timing self-tests passed") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("log", nargs="?", type=Path) + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args() + if args.self_test: + self_test() + elif args.log: + try: + print(json.dumps(summarize(args.log.read_text()), indent=2)) + except (ValueError, OSError) as error: + parser.exit(1, f"Invalid documentation UI timing evidence: {error}\n") + else: + parser.error("provide one complete job log or --self-test") From 6c1de5695bab54d21265265c1f48c95c098a95a5 Mon Sep 17 00:00:00 2001 From: Johnny IV Young Ospino Date: Wed, 7 Oct 2026 11:27:27 -0500 Subject: [PATCH 2/3] perf(ui): avoid polling already-ready folder-panel controls --- UITests/DocumentationExportUITests.swift | 41 ++++++++++++++++++++++-- docs/DOCUMENTATION-UI-PROFILING.md | 20 ++++++++++++ 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/UITests/DocumentationExportUITests.swift b/UITests/DocumentationExportUITests.swift index 8fafceff..6670d6d3 100644 --- a/UITests/DocumentationExportUITests.swift +++ b/UITests/DocumentationExportUITests.swift @@ -103,12 +103,21 @@ final class DocumentationExportUITests: XCTestCase { app.typeText(parent.path) app.typeKey(.return, modifierFlags: []) let panel = app.dialogs["open-panel"] - XCTAssertTrue(panel.waitForExistence(timeout: 5), app.debugDescription) + // Avoid starting a polling wait when the real panel is already ready. + // Keep the same assertions and bounded waits for delayed readiness. + XCTAssertTrue( + readyOrWait(panel.exists) { panel.waitForExistence(timeout: 5) }, + app.debugDescription) // AppKit exposes this button's localized AXTitle, not an AXLabel. // Its in-dialog identifier excludes the Touch Bar duplicate. let choose = panel.buttons["OKButton"] - XCTAssertTrue(choose.waitForExistence(timeout: 5), app.debugDescription) - XCTAssertTrue(choose.wait(for: \.isHittable, toEqual: true, timeout: 5)) + XCTAssertTrue( + readyOrWait(choose.exists) { choose.waitForExistence(timeout: 5) }, + app.debugDescription) + XCTAssertTrue( + readyOrWait(choose.isHittable) { + choose.wait(for: \.isHittable, toEqual: true, timeout: 5) + }) recordSegment("panel") choose.click() XCTAssertTrue( @@ -137,6 +146,32 @@ final class DocumentationExportUITests: XCTestCase { } } + private func readyOrWait(_ ready: @autoclosure () -> Bool, wait: () -> Bool) -> Bool { + ready() || wait() + } + + func testReadinessFastPathPreservesDelayedAndMissingResults() { + var waits = 0 + XCTAssertTrue( + readyOrWait(true) { + waits += 1 + return false + }) + XCTAssertEqual(waits, 0, "Already-ready controls must not start a polling wait") + XCTAssertTrue( + readyOrWait(false) { + waits += 1 + return true + }) + XCTAssertEqual(waits, 1, "Delayed readiness must use the bounded wait") + XCTAssertFalse( + readyOrWait(false) { + waits += 1 + return false + }) + XCTAssertEqual(waits, 2, "A missing control must retain the wait's failure") + } + @MainActor private func reveal(_ identifier: String, in app: XCUIApplication) { if waitForHittableElement(identifier, in: app, timeout: 1) { return } diff --git a/docs/DOCUMENTATION-UI-PROFILING.md b/docs/DOCUMENTATION-UI-PROFILING.md index 48f20b0c..1a0eed53 100644 --- a/docs/DOCUMENTATION-UI-PROFILING.md +++ b/docs/DOCUMENTATION-UI-PROFILING.md @@ -55,3 +55,23 @@ The performance lane's compile/link overhead is a separate hypothesis. Inspect t actual build graph and configuration differences before proposing reuse. Existing lane-partition checks already prevent duplicated performance/golden coverage work; that existing behavior is not a new optimization. + +## Already-ready folder-panel checks + +The folder-panel journey checks the current `exists`/`isHittable` state before +starting XCTest's polling waits. The same three assertions and five-second +fallback waits remain: the real open panel must exist, its scoped OK button must +exist, and that button must be hittable. Export still uses the native button after +Go to Folder keyboard navigation and path entry; no pasteboard shortcut, seeded +panel location, or programmatic export replaces that interaction. + +This avoids invoking the polling wait when readiness is already established. +It is a candidate optimization, not a guaranteed duration reduction: extra +accessibility queries can have a cost, especially when readiness is delayed. +A focused control test verifies that already-ready state avoids the fallback, +delayed readiness returns its successful fallback result, and a missing control +retains the fallback failure. These are synthetic readiness controls, not a +simulated delayed or missing native panel. +Use the identical eight-phase markers and matched-run protocol above to determine +its effect. A single candidate CI run versus a historical baseline is diagnostic; +it cannot establish a reproducible improvement or quantify workflow savings. From 5c0b10257fd044be5093097bf46ecffcc37535e3 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 15:39:11 +0000 Subject: [PATCH 3/3] refactor(ui): inline folder-panel readiness checks - Replace the readyOrWait helper with direct short-circuiting checks - Drop the synthetic control test that only exercised the || operator - Attach the accessibility tree to the OK-button hittability assertion - Describe the change and its measurement limits accurately in the profiling doc --- UITests/DocumentationExportUITests.swift | 42 ++++-------------------- docs/DOCUMENTATION-UI-PROFILING.md | 19 +++++------ 2 files changed, 16 insertions(+), 45 deletions(-) diff --git a/UITests/DocumentationExportUITests.swift b/UITests/DocumentationExportUITests.swift index 6670d6d3..aaacc461 100644 --- a/UITests/DocumentationExportUITests.swift +++ b/UITests/DocumentationExportUITests.swift @@ -103,21 +103,19 @@ final class DocumentationExportUITests: XCTestCase { app.typeText(parent.path) app.typeKey(.return, modifierFlags: []) let panel = app.dialogs["open-panel"] - // Avoid starting a polling wait when the real panel is already ready. - // Keep the same assertions and bounded waits for delayed readiness. + // Check current readiness before starting a polling wait; `||` + // short-circuits, so delayed readiness keeps the same bounded waits. XCTAssertTrue( - readyOrWait(panel.exists) { panel.waitForExistence(timeout: 5) }, - app.debugDescription) + panel.exists || panel.waitForExistence(timeout: 5), app.debugDescription) // AppKit exposes this button's localized AXTitle, not an AXLabel. // Its in-dialog identifier excludes the Touch Bar duplicate. let choose = panel.buttons["OKButton"] XCTAssertTrue( - readyOrWait(choose.exists) { choose.waitForExistence(timeout: 5) }, - app.debugDescription) + choose.exists || choose.waitForExistence(timeout: 5), app.debugDescription) XCTAssertTrue( - readyOrWait(choose.isHittable) { - choose.wait(for: \.isHittable, toEqual: true, timeout: 5) - }) + choose.isHittable + || choose.wait(for: \.isHittable, toEqual: true, timeout: 5), + app.debugDescription) recordSegment("panel") choose.click() XCTAssertTrue( @@ -146,32 +144,6 @@ final class DocumentationExportUITests: XCTestCase { } } - private func readyOrWait(_ ready: @autoclosure () -> Bool, wait: () -> Bool) -> Bool { - ready() || wait() - } - - func testReadinessFastPathPreservesDelayedAndMissingResults() { - var waits = 0 - XCTAssertTrue( - readyOrWait(true) { - waits += 1 - return false - }) - XCTAssertEqual(waits, 0, "Already-ready controls must not start a polling wait") - XCTAssertTrue( - readyOrWait(false) { - waits += 1 - return true - }) - XCTAssertEqual(waits, 1, "Delayed readiness must use the bounded wait") - XCTAssertFalse( - readyOrWait(false) { - waits += 1 - return false - }) - XCTAssertEqual(waits, 2, "A missing control must retain the wait's failure") - } - @MainActor private func reveal(_ identifier: String, in app: XCUIApplication) { if waitForHittableElement(identifier, in: app, timeout: 1) { return } diff --git a/docs/DOCUMENTATION-UI-PROFILING.md b/docs/DOCUMENTATION-UI-PROFILING.md index 1a0eed53..fba159fb 100644 --- a/docs/DOCUMENTATION-UI-PROFILING.md +++ b/docs/DOCUMENTATION-UI-PROFILING.md @@ -65,13 +65,12 @@ exist, and that button must be hittable. Export still uses the native button aft Go to Folder keyboard navigation and path entry; no pasteboard shortcut, seeded panel location, or programmatic export replaces that interaction. -This avoids invoking the polling wait when readiness is already established. -It is a candidate optimization, not a guaranteed duration reduction: extra -accessibility queries can have a cost, especially when readiness is delayed. -A focused control test verifies that already-ready state avoids the fallback, -delayed readiness returns its successful fallback result, and a missing control -retains the fallback failure. These are synthetic readiness controls, not a -simulated delayed or missing native panel. -Use the identical eight-phase markers and matched-run protocol above to determine -its effect. A single candidate CI run versus a historical baseline is diagnostic; -it cannot establish a reproducible improvement or quantify workflow savings. +Each check is a short-circuiting `exists || waitForExistence(...)` (or the +`isHittable` equivalent), so an already-ready control skips the wait call and a +delayed or missing control reaches the same bounded wait and failure. It is a +candidate optimization, not a guaranteed duration reduction: XCTest's waits also +evaluate readiness when they start, and the extra accessibility query adds cost +when readiness is delayed. Use the identical eight-phase markers and matched-run +protocol above to determine its effect. A single candidate CI run versus a +historical baseline is diagnostic; it cannot establish a reproducible improvement +or quantify workflow savings.