From e52a594338024d128d9465069d4869969b3e1550 Mon Sep 17 00:00:00 2001 From: Johnny IV Young Ospino Date: Tue, 6 Oct 2026 19:47:25 -0500 Subject: [PATCH 1/2] test(ui): profile documentation export phases without reducing coverage --- Makefile | 6 ++- UITests/DocumentationExportUITests.swift | 17 ++++++ docs/DOCUMENTATION-UI-PROFILING.md | 57 ++++++++++++++++++++ scripts/summarize-documentation-ui.py | 67 ++++++++++++++++++++++++ 4 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 docs/DOCUMENTATION-UI-PROFILING.md create mode 100644 scripts/summarize-documentation-ui.py diff --git a/Makefile b/Makefile index 9d1c1b60..bc6f2a31 100644 --- a/Makefile +++ b/Makefile @@ -407,7 +407,7 @@ format: $(SWIFTFORMAT) format --in-place --recursive Vitrine VitrineDomain VitrineRendering VitrineCLI VitrineMenuBarHelper DomainTests RenderingTests RepositoryTests Tests UITests ## lint: lint Swift sources and tracked repository metadata (fails on issues) -lint: sanitizer-check test-lanes-check golden-check hygiene project-version-check bump-check perf-check coverage-check swift-features-check build-boundaries-check informational-update-check release-promotion-check commit-ci-check qa-handoff-check memory-smoke-check ui-test-preflight-check screenshot-tour-check +lint: documentation-ui-profile-check sanitizer-check test-lanes-check golden-check hygiene project-version-check bump-check perf-check coverage-check swift-features-check build-boundaries-check informational-update-check release-promotion-check commit-ci-check qa-handoff-check memory-smoke-check ui-test-preflight-check screenshot-tour-check $(SWIFTFORMAT) lint --strict --recursive Vitrine VitrineDomain VitrineRendering VitrineCLI VitrineMenuBarHelper DomainTests RenderingTests RepositoryTests Tests UITests ## hygiene: reject private planning identifiers and tracked planning artifacts @@ -449,3 +449,7 @@ test-web-capture: project web-capture-check python3 scripts/web-capture-fixture-test.py env DEVELOPER_DIR="$(XCODE_DEVELOPER)" python3 scripts/test-web-capture.py \ --output "$(WEB_CAPTURE_OUTPUT)" $(WEB_CAPTURE_HOST_FLAGS) + +.PHONY: documentation-ui-profile-check +documentation-ui-profile-check: + python3 scripts/summarize-documentation-ui.py --self-test diff --git a/UITests/DocumentationExportUITests.swift b/UITests/DocumentationExportUITests.swift index 842c92a7..8fafceff 100644 --- a/UITests/DocumentationExportUITests.swift +++ b/UITests/DocumentationExportUITests.swift @@ -21,6 +21,15 @@ final class DocumentationExportUITests: XCTestCase { for language in ["en", "es"] { for dark in [false, true] { for request in requests { + let state = "\(language)-\(dark ? "dark" : "light")-\(request.label)" + var segmentStarted = ProcessInfo.processInfo.systemUptime + func recordSegment(_ phase: String) { + let now = ProcessInfo.processInfo.systemUptime + print( + "DOCUMENTATION_UI_SEGMENT state=\(state) phase=\(phase) " + + "seconds=\(now - segmentStarted)") + segmentStarted = now + } let robot = VitrineAppRobot(testCase: self, suitePrefix: "documentation-export") let app = robot.launch( arguments: VitrineLaunchArguments.editor + [ @@ -30,6 +39,7 @@ final class DocumentationExportUITests: XCTestCase { environment: [ "VITRINE_UI_TEST_EDITOR_VIEWPORT": request.viewport ]) + recordSegment("launch") defer { app.terminate() } let window = element("editor-window", in: app) XCTAssertTrue(window.waitForExistence(timeout: 8), app.debugDescription) @@ -52,6 +62,7 @@ final class DocumentationExportUITests: XCTestCase { XCTAssertTrue(output.waitForExistence(timeout: 8), app.debugDescription) reveal("inspector-disclosure-output", in: app) output.click() + recordSegment("navigation") let description = element("alternative-text-field", in: app) XCTAssertTrue(description.waitForExistence(timeout: 3), app.debugDescription) reveal("alternative-text-field", in: app) @@ -67,6 +78,7 @@ final class DocumentationExportUITests: XCTestCase { predicate: descriptionAccepted, object: nil) ], timeout: 3), .completed, "The description must be entered in its own field") + recordSegment("description") let options = element("copy-options-menu", in: app) XCTAssertTrue(options.wait(for: \.isHittable, toEqual: true, timeout: 3)) options.click() @@ -85,6 +97,7 @@ final class DocumentationExportUITests: XCTestCase { screenshot.name = name screenshot.lifetime = .keepAlways add(screenshot) + recordSegment("package-setup") element("documentation-export", in: app).click() app.typeKey("g", modifierFlags: [.command, .shift]) app.typeText(parent.path) @@ -96,10 +109,12 @@ final class DocumentationExportUITests: XCTestCase { let choose = panel.buttons["OKButton"] XCTAssertTrue(choose.waitForExistence(timeout: 5), app.debugDescription) XCTAssertTrue(choose.wait(for: \.isHittable, toEqual: true, timeout: 5)) + recordSegment("panel") choose.click() XCTAssertTrue( element("documentation-cancel", in: app).waitForNonExistence(timeout: 10), app.debugDescription) + recordSegment("export") let directory = parent.appendingPathComponent(name) XCTAssertEqual( Set(try FileManager.default.contentsOfDirectory(atPath: directory.path)), @@ -114,7 +129,9 @@ final class DocumentationExportUITests: XCTestCase { let source = try String( contentsOf: directory.appendingPathComponent("source.txt"), encoding: .utf8) XCTAssertFalse(source.contains("Documentation example"), source) + recordSegment("validation") app.terminate() + recordSegment("termination") } } } diff --git a/docs/DOCUMENTATION-UI-PROFILING.md b/docs/DOCUMENTATION-UI-PROFILING.md new file mode 100644 index 00000000..48f20b0c --- /dev/null +++ b/docs/DOCUMENTATION-UI-PROFILING.md @@ -0,0 +1,57 @@ +# Documentation export UI profiling + +The documentation-package test deliberately covers eight states: English/Spanish, +light/dark appearance, and minimum/1280×800 windows. Each state launches an isolated +app, enters alternative text through accessibility, configures a package, visits the +real folder panel, exports, and verifies image.png, README.md, index.html and source.txt. +The separate cancellation test remains part of the ordinary UI suite. + +## Diagnostic evidence + +The test emits `DOCUMENTATION_UI_SEGMENT` lines using monotonic system uptime: + +- launch: isolated app creation, launch and activation +- navigation: editor geometry and output-inspector navigation +- description: alternative-text entry and acceptance +- package-setup: menu, sheet, folder-name entry, output selections and screenshot +- panel: export click, Go to Folder typing, and localized panel readiness +- export: panel confirmation until the export sheet disappears +- validation: all four files and their text/alternative-text content +- termination: explicit app termination + +Each segment includes automation overhead. In particular, `export` is not a pure +renderer benchmark. Assertions, timeouts and test selection are unchanged. These +markers add measurement overhead and do not themselves optimize the test. + +Save the complete ordinary-UI job log for one OS and one attempt, then run: + +```sh +python3 scripts/summarize-documentation-ui.py job.log > timing.json +python3 scripts/summarize-documentation-ui.py --self-test +``` + +The summary fails closed on missing states/phases, duplicates, unknown labels and +nonfinite/negative durations. Do not combine two OS jobs or reruns in one input. +Passing timing parsing is not proof that the entire UI job passed; retain the exact +source head, run/job URL, Xcode/image identity, xcresult and terminal job conclusion. +A failed/incomplete test must not become a successful performance observation. + +## Controlled qualification before optimization claims + +First collect instrumentation-only baseline evidence. Then change only the measured +bottleneck while keeping eight states, real panel interactions, typing acceptance, +all file checks, cancellation and screenshot coverage intact. Compare matched base +and candidate runs on the same OS, Xcode version and runner image. Record queue time +separately from executed job, step and test durations. Five matched pairs per OS are +the qualification target; fewer pairs are diagnostic and remain unqualified. Do not +claim a 20–30% improvement from historic runs across different source commits. + +Keep Tahoe strict pixel-golden comparison separate from Sequoia render-only smoke. +Do not drop OS/locale states, relax timeout or performance/coverage floors, suppress +unexpected skips, reuse instrumented coverage products for performance, or share +build products across SDK/configuration/signing/sandbox boundaries. + +The performance lane's compile/link overhead is a separate hypothesis. Inspect the +actual build graph and configuration differences before proposing reuse. Existing +lane-partition checks already prevent duplicated performance/golden coverage work; +that existing behavior is not a new optimization. diff --git a/scripts/summarize-documentation-ui.py b/scripts/summarize-documentation-ui.py new file mode 100644 index 00000000..a10c950b --- /dev/null +++ b/scripts/summarize-documentation-ui.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Summarize complete eight-state documentation UI timings; never infer a speedup.""" +import argparse +import itertools +import json +import math +import re +from pathlib import Path + +STATES = {f"{locale}-{appearance}-{size}" for locale, appearance, size in + itertools.product(("en", "es"), ("light", "dark"), ("minimum", "1280"))} +PHASES = ("launch", "navigation", "description", "package-setup", "panel", "export", "validation", "termination") +PATTERN = re.compile(r"DOCUMENTATION_UI_SEGMENT state=(\S+) phase=(\S+) seconds=(\S+)") + + +def summarize(text): + states = {state: {} for state in STATES} + for state, phase, raw in PATTERN.findall(text): + if state not in STATES or phase not in PHASES: + raise ValueError(f"Unknown state or phase: {state}/{phase}") + if phase in states[state]: + raise ValueError(f"Duplicate timing: {state}/{phase}; provide one job attempt only") + seconds = float(raw) + if not math.isfinite(seconds) or seconds < 0: + raise ValueError(f"Invalid duration: {raw}") + states[state][phase] = seconds + for state, phases in states.items(): + if set(phases) != set(PHASES): + raise ValueError(f"Incomplete state {state}: missing {sorted(set(PHASES) - phases.keys())}") + totals = {phase: sum(states[state][phase] for state in STATES) for phase in PHASES} + return {"states": dict(sorted(states.items())), "phase_seconds": totals, + "total_seconds": sum(totals.values()), "state_count": len(states), + "qualification": "diagnostic timings only; no controlled performance improvement claim"} + + +def self_test(): + valid = "\n".join(f"DOCUMENTATION_UI_SEGMENT state={s} phase={p} seconds=1.25" + for s in sorted(STATES) for p in PHASES) + assert summarize(valid)["total_seconds"] == 80 + for invalid in ("", valid.split("\n", 1)[1], valid + "\n" + valid.splitlines()[0], + valid.replace("seconds=1.25", "seconds=nan", 1), + valid.replace("seconds=1.25", "seconds=-1", 1), + valid.replace("state=en-dark-1280", "state=unknown", 1), + valid.replace("phase=launch", "phase=unknown", 1)): + try: + summarize(invalid) + except ValueError: + pass + else: + raise AssertionError("Malformed/incomplete timing evidence was accepted") + print("documentation UI timing self-tests passed") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("log", nargs="?", type=Path) + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args() + if args.self_test: + self_test() + elif args.log: + try: + print(json.dumps(summarize(args.log.read_text()), indent=2)) + except (ValueError, OSError) as error: + parser.exit(1, f"Invalid documentation UI timing evidence: {error}\n") + else: + parser.error("provide one complete job log or --self-test") From b4204ab557f9fe6160c2f8faef0847f7fbbdb1d4 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 18:49:33 +0000 Subject: [PATCH 2/2] test(ui): lock the documentation timing summarizer to the test's marker labels - Fail the self-test when the UI test's phase, locale, appearance, or size labels drift from the summarizer's constants. - Read job logs with replacement decoding so a stray byte elsewhere in a large log cannot reject complete marker evidence. - Isolate the unknown-label guard in the self-test, cover non-numeric durations, and replace the bare assert so python -O cannot skip the positive check. - Describe the lint sub-check in the Makefile and link the profiling guide from the release documentation. --- Makefile | 3 ++ docs/RELEASING.md | 4 ++- scripts/summarize-documentation-ui.py | 42 +++++++++++++++++++++++---- 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index bc6f2a31..60de377d 100644 --- a/Makefile +++ b/Makefile @@ -450,6 +450,9 @@ test-web-capture: project web-capture-check env DEVELOPER_DIR="$(XCODE_DEVELOPER)" python3 scripts/test-web-capture.py \ --output "$(WEB_CAPTURE_OUTPUT)" $(WEB_CAPTURE_HOST_FLAGS) +## documentation-ui-profile-check: validate the documentation UI timing summarizer without Xcode +## It also proves the summarizer's state and phase labels still match the markers the +## UI test emits, so a renamed phase cannot silently make CI logs unsummarizable. .PHONY: documentation-ui-profile-check documentation-ui-profile-check: python3 scripts/summarize-documentation-ui.py --self-test diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 1f233f7c..b7357555 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -435,7 +435,9 @@ frame in a fresh process. Mode switching is restricted to disposable GitHub-host runners; local and self-hosted invocations are rejected. An unsupported display fails the lane rather than skipping the journey. The `documentation-ui` artifact contains its display receipt and all eight EN/ES, light/dark, minimum/1280×800 -screenshots; missing or duplicate states fail evidence extraction. +screenshots; missing or duplicate states fail evidence extraction. The same journey +prints per-phase timing markers; [docs/DOCUMENTATION-UI-PROFILING.md](DOCUMENTATION-UI-PROFILING.md) +explains how to summarize them without turning diagnostics into performance claims. For local runs, keep Vitrine/XCTest frontmost while the suite is active. XCUITest can report unrelated windows as "interrupting elements" (Slack, browsers, or an diff --git a/scripts/summarize-documentation-ui.py b/scripts/summarize-documentation-ui.py index a10c950b..f6ddd11b 100644 --- a/scripts/summarize-documentation-ui.py +++ b/scripts/summarize-documentation-ui.py @@ -7,10 +7,15 @@ import re from pathlib import Path +LOCALES = ("en", "es") +APPEARANCES = ("light", "dark") +SIZES = ("minimum", "1280") STATES = {f"{locale}-{appearance}-{size}" for locale, appearance, size in - itertools.product(("en", "es"), ("light", "dark"), ("minimum", "1280"))} + itertools.product(LOCALES, APPEARANCES, SIZES)} PHASES = ("launch", "navigation", "description", "package-setup", "panel", "export", "validation", "termination") PATTERN = re.compile(r"DOCUMENTATION_UI_SEGMENT state=(\S+) phase=(\S+) seconds=(\S+)") +# The test that emits the markers; its labels must stay in lockstep with the constants above. +SOURCE = Path(__file__).resolve().parent.parent / "UITests" / "DocumentationExportUITests.swift" def summarize(text): @@ -33,21 +38,46 @@ def summarize(text): "qualification": "diagnostic timings only; no controlled performance improvement claim"} +def check_source(source): + """Fail when the UI test's marker labels drift from the constants the summarizer expects.""" + phases = tuple(re.findall(r'recordSegment\("([^"]+)"\)', source)) + languages = re.search(r"for language in \[([^\]]+)\]", source) + locales = tuple(re.findall(r'"([^"]+)"', languages.group(1) if languages else "")) + ternary = re.search(r'dark \? "([^"]+)" : "([^"]+)"', source) + appearances = (ternary.group(2), ternary.group(1)) if ternary else () + sizes = tuple(re.findall(r'label: "([^"]+)"', source)) + found = (phases, locales, appearances, sizes) + if found != (PHASES, LOCALES, APPEARANCES, SIZES): + raise ValueError(f"Test marker labels {found} differ from summarizer labels " + f"{(PHASES, LOCALES, APPEARANCES, SIZES)}") + + def self_test(): + check_source(SOURCE.read_text(encoding="utf-8")) valid = "\n".join(f"DOCUMENTATION_UI_SEGMENT state={s} phase={p} seconds=1.25" for s in sorted(STATES) for p in PHASES) - assert summarize(valid)["total_seconds"] == 80 + if summarize(valid)["total_seconds"] != 80: + raise AssertionError("Complete timing evidence was not summed correctly") for invalid in ("", valid.split("\n", 1)[1], valid + "\n" + valid.splitlines()[0], valid.replace("seconds=1.25", "seconds=nan", 1), valid.replace("seconds=1.25", "seconds=-1", 1), - valid.replace("state=en-dark-1280", "state=unknown", 1), - valid.replace("phase=launch", "phase=unknown", 1)): + valid.replace("seconds=1.25", "seconds=soon", 1), + valid + "\nDOCUMENTATION_UI_SEGMENT state=unknown phase=launch seconds=1.25", + valid + "\nDOCUMENTATION_UI_SEGMENT state=en-dark-1280 phase=unknown seconds=1.25"): try: summarize(invalid) except ValueError: pass else: raise AssertionError("Malformed/incomplete timing evidence was accepted") + for drifted in (SOURCE.read_text(encoding="utf-8").replace('recordSegment("panel")', 'recordSegment("dialog")'), + SOURCE.read_text(encoding="utf-8").replace('label: "1280"', 'label: "large"')): + try: + check_source(drifted) + except ValueError: + pass + else: + raise AssertionError("Drifted test marker labels were accepted") print("documentation UI timing self-tests passed") @@ -60,7 +90,9 @@ def self_test(): self_test() elif args.log: try: - print(json.dumps(summarize(args.log.read_text()), indent=2)) + # Marker lines are ASCII; a stray byte elsewhere in a large job log is not evidence. + text = args.log.read_text(encoding="utf-8", errors="replace") + print(json.dumps(summarize(text), indent=2)) except (ValueError, OSError) as error: parser.exit(1, f"Invalid documentation UI timing evidence: {error}\n") else: