diff --git a/Cargo.lock b/Cargo.lock index b427b73..64d4bb1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1227,7 +1227,7 @@ dependencies = [ [[package]] name = "mobench" -version = "0.1.37" +version = "0.1.39" dependencies = [ "anyhow", "clap", @@ -1254,7 +1254,7 @@ dependencies = [ [[package]] name = "mobench-macros" -version = "0.1.37" +version = "0.1.39" dependencies = [ "proc-macro2", "quote", @@ -1264,7 +1264,7 @@ dependencies = [ [[package]] name = "mobench-sdk" -version = "0.1.37" +version = "0.1.39" dependencies = [ "include_dir", "inventory", @@ -1884,7 +1884,7 @@ dependencies = [ [[package]] name = "sample-fns" -version = "0.1.37" +version = "0.1.39" dependencies = [ "camino", "mobench-sdk", diff --git a/Cargo.toml b/Cargo.toml index a1db2ed..75fee9b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ resolver = "2" edition = "2024" license = "MIT" rust-version = "1.85" -version = "0.1.37" +version = "0.1.39" [workspace.dependencies] anyhow = "1" @@ -34,5 +34,5 @@ proc-macro2 = "1" # Phase 1: Template embedding include_dir = "0.7" -mobench-macros = { version = "0.1.37", path = "crates/mobench-macros" } -mobench-sdk = { version = "0.1.37", path = "crates/mobench-sdk", default-features = false } +mobench-macros = { version = "0.1.39", path = "crates/mobench-macros" } +mobench-sdk = { version = "0.1.39", path = "crates/mobench-sdk", default-features = false } diff --git a/README.md b/README.md index a0a121a..faa851d 100644 --- a/README.md +++ b/README.md @@ -29,99 +29,6 @@ that CI and humans can compare. - Mobile apps call `run_benchmark` via the generated bindings and return timing samples - The CLI collects results locally or from BrowserStack and writes summaries -## Workflow diagrams - -The Mermaid sources live under `docs/diagrams/` so the same diagrams can be -reused in launch posts and landing-page assets. - -### Crate architecture - -```mermaid -flowchart LR - user["Benchmark crate"] --> macro["mobench-macros\n#[benchmark]"] - macro --> registry["mobench-sdk registry\ninventory"] - registry --> runner["mobench-sdk runner\nBenchSpec -> BenchReport"] - runner --> templates["Generated Android/iOS runners"] - cli["mobench CLI"] --> builders["SDK builders"] - builders --> templates - cli --> reports["JSON / Markdown / CSV / plots"] - templates --> reports -``` - -### Benchmark lifecycle - -```mermaid -sequenceDiagram - participant Dev as Developer - participant CLI as mobench CLI - participant SDK as mobench-sdk - participant App as Generated mobile app - participant Device as Device or BrowserStack - participant Reports as Reports - - Dev->>CLI: cargo mobench run - CLI->>SDK: resolve crate and benchmark spec - SDK->>SDK: build native libraries and generate bindings - SDK->>App: embed bench_spec.json and templates - CLI->>Device: install/upload and start run - Device->>App: execute benchmark function - App->>CLI: emit BenchReport JSON - CLI->>Reports: write summary.json, summary.md, results.csv -``` - -### BrowserStack CI lifecycle - -```mermaid -flowchart TD - workflow["GitHub Actions"] --> resolve["Resolve device matrix"] - resolve --> build["Build APK or IPA/XCUITest"] - build --> upload["Upload artifacts to BrowserStack"] - upload --> run["Run benchmark on selected devices"] - run --> fetch["Fetch logs, reports, and metrics"] - fetch --> normalize["Normalize timing, CPU, and memory"] - normalize --> outputs["summary.json\nsummary.md\nresults.csv\nplots"] - outputs --> pr["Optional PR comment/check run"] -``` - -### Profiling artifact lifecycle - -```mermaid -flowchart LR - run["profile run"] --> manifest["profile.json\nnative_capture\nsemantic_profile\ncapture_metadata"] - run --> raw["raw capture\nsimpleperf or sample"] - raw --> processed["processed stacks\nstacks.folded\nnative-report.txt"] - processed --> viewer["flamegraph.html\nfull and focused SVGs"] - manifest --> summary["summary.md"] - manifest --> semantic["artifacts/semantic/phases.json"] - viewer --> diff["profile diff\nbaseline vs candidate"] - summary --> diff -``` - -### SDK versus CLI responsibilities - -```mermaid -flowchart TB - subgraph SDK["mobench-sdk"] - timing["timing harness"] - registry["benchmark registry"] - builders["Android/iOS builders"] - codegen["template/codegen"] - ffi["FFI-safe types"] - end - - subgraph CLI["mobench CLI"] - config["config and project resolution"] - orchestration["build/run/profile orchestration"] - providers["BrowserStack and local providers"] - reporting["summary, plots, PR reports"] - end - - SDK --> CLI - CLI --> SDK - user["Downstream benchmark crate"] --> SDK - ci["CI workflow"] --> CLI -``` - ## Workspace crates - `crates/mobench` ([mobench](https://crates.io/crates/mobench)): CLI tool that builds, runs, and fetches benchmarks @@ -299,6 +206,7 @@ min_sdk = 24 [ios] bundle_id = "com.example.bench" deployment_target = "15.0" +# runner = "swiftui" # or "uikit-legacy" for legacy iOS deployment targets [benchmarks] default_function = "my_crate::my_benchmark" @@ -310,6 +218,7 @@ Resolution precedence is: explicit CLI flags (`--project-root`, `--crate-path`) CLI flags override config file values when provided. - In `cargo mobench run --config ` mode, `--device-matrix ` overrides `device_matrix` from the config file. +- iOS deployment targets below 15.0 use the UIKit legacy runner by default; forcing `runner = "swiftui"` below 15.0 fails early. Legacy targets such as iPhone 7 on iOS 10 also require an older Xcode lane capable of building for that OS. - For regression comparisons, `--baseline` should point to a previous run summary; if it resolves to the same output path, mobench snapshots the prior file before writing the candidate summary. - In the reusable GitHub workflow, the default baseline source is the latest successful run on the repository default branch when matching artifacts are available. - `cargo mobench verify --smoke-test` is only supported for benchmark crates linked into the `mobench` CLI binary. External crates discovered through `mobench.toml`, `--project-root`, or `--crate-path` should use `cargo mobench list` and `cargo mobench verify --check-artifacts`. @@ -405,6 +314,99 @@ fn db_query(db: &Database) { | `#[benchmark(setup = fn, per_iteration)]` | Benchmarks that mutate input, need fresh data each time | | `#[benchmark(setup = fn, teardown = fn)]` | Resources requiring cleanup (connections, files, etc.) | +## Workflow diagrams + +The Mermaid sources live under `docs/diagrams/` so the same diagrams can be +reused in launch posts and landing-page assets. + +### Crate architecture + +```mermaid +flowchart LR + user["Benchmark crate"] --> macro["mobench-macros\n#[benchmark]"] + macro --> registry["mobench-sdk registry\ninventory"] + registry --> runner["mobench-sdk runner\nBenchSpec -> BenchReport"] + runner --> templates["Generated Android/iOS runners"] + cli["mobench CLI"] --> builders["SDK builders"] + builders --> templates + cli --> reports["JSON / Markdown / CSV / plots"] + templates --> reports +``` + +### Benchmark lifecycle + +```mermaid +sequenceDiagram + participant Dev as Developer + participant CLI as mobench CLI + participant SDK as mobench-sdk + participant App as Generated mobile app + participant Device as Device or BrowserStack + participant Reports as Reports + + Dev->>CLI: cargo mobench run + CLI->>SDK: resolve crate and benchmark spec + SDK->>SDK: build native libraries and generate bindings + SDK->>App: embed bench_spec.json and templates + CLI->>Device: install/upload and start run + Device->>App: execute benchmark function + App->>CLI: emit BenchReport JSON + CLI->>Reports: write summary.json, summary.md, results.csv +``` + +### BrowserStack CI lifecycle + +```mermaid +flowchart TD + workflow["GitHub Actions"] --> resolve["Resolve device matrix"] + resolve --> build["Build APK or IPA/XCUITest"] + build --> upload["Upload artifacts to BrowserStack"] + upload --> run["Run benchmark on selected devices"] + run --> fetch["Fetch logs, reports, and metrics"] + fetch --> normalize["Normalize timing, CPU, and memory"] + normalize --> outputs["summary.json\nsummary.md\nresults.csv\nplots"] + outputs --> pr["Optional PR comment/check run"] +``` + +### Profiling artifact lifecycle + +```mermaid +flowchart LR + run["profile run"] --> manifest["profile.json\nnative_capture\nsemantic_profile\ncapture_metadata"] + run --> raw["raw capture\nsimpleperf or sample"] + raw --> processed["processed stacks\nstacks.folded\nnative-report.txt"] + processed --> viewer["flamegraph.html\nfull and focused SVGs"] + manifest --> summary["summary.md"] + manifest --> semantic["artifacts/semantic/phases.json"] + viewer --> diff["profile diff\nbaseline vs candidate"] + summary --> diff +``` + +### SDK versus CLI responsibilities + +```mermaid +flowchart TB + subgraph SDK["mobench-sdk"] + timing["timing harness"] + registry["benchmark registry"] + builders["Android/iOS builders"] + codegen["template/codegen"] + ffi["FFI-safe types"] + end + + subgraph CLI["mobench CLI"] + config["config and project resolution"] + orchestration["build/run/profile orchestration"] + providers["BrowserStack and local providers"] + reporting["summary, plots, PR reports"] + end + + SDK --> CLI + CLI --> SDK + user["Downstream benchmark crate"] --> SDK + ci["CI workflow"] --> CLI +``` + ## Release Notes Published release history and support status live in diff --git a/crates/mobench-sdk/README.md b/crates/mobench-sdk/README.md index cf9179a..abcc762 100644 --- a/crates/mobench-sdk/README.md +++ b/crates/mobench-sdk/README.md @@ -408,6 +408,7 @@ target_sdk = 34 [ios] bundle_id = "com.example.bench" deployment_target = "15.0" +# runner = "swiftui" # or "uikit-legacy" for legacy iOS targets [benchmarks] default_function = "my_crate::my_benchmark" diff --git a/crates/mobench-sdk/src/builders/common.rs b/crates/mobench-sdk/src/builders/common.rs index f8c7b49..68a18ae 100644 --- a/crates/mobench-sdk/src/builders/common.rs +++ b/crates/mobench-sdk/src/builders/common.rs @@ -791,10 +791,21 @@ members = ["crates/*"] let _ = std::fs::remove_dir_all(&temp_dir); std::fs::create_dir_all(&temp_dir).unwrap(); - let spec = EmbeddedBenchSpec { + #[derive(serde::Serialize)] + struct AndroidSpec { + function: String, + iterations: u32, + warmup: u32, + android_benchmark_timeout_secs: Option, + android_heartbeat_interval_secs: Option, + } + + let spec = AndroidSpec { function: "test_crate::first_run".to_string(), iterations: 7, warmup: 1, + android_benchmark_timeout_secs: Some(30), + android_heartbeat_interval_secs: Some(5), }; embed_bench_spec(&temp_dir, &spec).expect("embed spec"); @@ -812,6 +823,11 @@ members = ["crates/*"] let contents = std::fs::read_to_string(android_spec).unwrap(); assert!(contents.contains("test_crate::first_run")); + assert!(contents.contains("android_benchmark_timeout_secs")); + assert!(contents.contains("android_heartbeat_interval_secs")); + let json: serde_json::Value = serde_json::from_str(&contents).unwrap(); + assert_eq!(json["android_benchmark_timeout_secs"], 30); + assert_eq!(json["android_heartbeat_interval_secs"], 5); std::fs::remove_dir_all(&temp_dir).unwrap(); } diff --git a/crates/mobench-sdk/src/builders/ios.rs b/crates/mobench-sdk/src/builders/ios.rs index 9b92fdb..c8ae744 100644 --- a/crates/mobench-sdk/src/builders/ios.rs +++ b/crates/mobench-sdk/src/builders/ios.rs @@ -71,6 +71,7 @@ //! ``` use super::common::{get_cargo_target_dir, host_lib_path, run_command, validate_project_root}; +use crate::codegen::{IosDeploymentTarget, IosProjectOptions, IosRunner, resolve_ios_runner}; use crate::types::{BenchError, BuildConfig, BuildProfile, BuildResult, Target}; use std::env; use std::fs; @@ -78,6 +79,91 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::time::{SystemTime, UNIX_EPOCH}; +fn resolve_ios_benchmark_timeout_secs_from_env() -> u64 { + env::var("MOBENCH_IOS_BENCHMARK_TIMEOUT_SECS") + .ok() + .and_then(|raw| raw.parse::().ok()) + .filter(|secs| *secs > 0) + .unwrap_or(crate::codegen::DEFAULT_IOS_BENCHMARK_TIMEOUT_SECS) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct XcodeVersion { + pub major: u16, + pub minor: u16, + pub raw: String, +} + +fn parse_xcode_version(output: &str) -> Option { + let line = output.lines().find(|line| line.starts_with("Xcode "))?; + let raw_version = line.trim_start_matches("Xcode ").trim(); + let mut parts = raw_version.split('.'); + let major = parts.next()?.parse::().ok()?; + let minor = parts + .next() + .and_then(|part| part.parse::().ok()) + .unwrap_or(0); + Some(XcodeVersion { + major, + minor, + raw: raw_version.to_string(), + }) +} + +fn selected_xcode_version() -> Result { + let output = Command::new("xcodebuild") + .arg("-version") + .output() + .map_err(|err| { + BenchError::Build(format!( + "Failed to run `xcodebuild -version`: {err}. Install/select Xcode before building iOS artifacts." + )) + })?; + + if !output.status.success() { + return Err(BenchError::Build(format!( + "`xcodebuild -version` failed: {}", + String::from_utf8_lossy(&output.stderr) + ))); + } + + let stdout = String::from_utf8_lossy(&output.stdout); + parse_xcode_version(&stdout).ok_or_else(|| { + BenchError::Build(format!( + "Failed to parse selected Xcode version from `xcodebuild -version` output:\n{stdout}" + )) + }) +} + +fn minimum_supported_ios_deployment_target_for_xcode( + xcode: &XcodeVersion, +) -> Result { + let floor = if xcode.major >= 15 { + "15.0" + } else if xcode.major == 14 { + "11.0" + } else { + "10.0" + }; + IosDeploymentTarget::parse(floor) +} + +pub fn validate_xcode_supports_ios_deployment_target( + deployment_target: &IosDeploymentTarget, +) -> Result<(), BenchError> { + let xcode = selected_xcode_version()?; + let supported_floor = minimum_supported_ios_deployment_target_for_xcode(&xcode)?; + if deployment_target < &supported_floor { + return Err(BenchError::Build(format!( + "iOS deployment target {deployment_target} requires an older Xcode toolchain; \ + selected Xcode {} supports iOS {}+ in mobench's supported lanes. \ + Use a legacy CI lane with an older Xcode capable of iOS 10/11/12, or raise `[ios].deployment_target`.", + xcode.raw, supported_floor + ))); + } + Ok(()) +} + /// iOS builder that handles the complete build pipeline. /// /// This builder automates the process of compiling Rust code to iOS static @@ -119,6 +205,10 @@ pub struct IosBuilder { crate_dir: Option, /// Whether to run in dry-run mode (print what would be done without making changes) dry_run: bool, + /// iOS deployment target to emit into the generated Xcode project. + deployment_target: IosDeploymentTarget, + /// Optional requested runner. If omitted, the runner is selected from the deployment target. + runner: Option, } impl IosBuilder { @@ -153,6 +243,8 @@ impl IosBuilder { verbose: false, crate_dir: None, dry_run: false, + deployment_target: IosDeploymentTarget::default_target(), + runner: None, } } @@ -194,6 +286,18 @@ impl IosBuilder { self } + /// Sets the iOS deployment target for generated app and XCUITest targets. + pub fn deployment_target(mut self, deployment_target: IosDeploymentTarget) -> Self { + self.deployment_target = deployment_target; + self + } + + /// Sets the iOS runner template explicitly. + pub fn runner(mut self, runner: Option) -> Self { + self.runner = runner; + self + } + /// Builds the iOS app with the given configuration /// /// This performs the following steps: @@ -213,6 +317,10 @@ impl IosBuilder { if self.crate_dir.is_none() { validate_project_root(&self.project_root, &self.crate_name)?; } + let runner = resolve_ios_runner(&self.deployment_target, self.runner)?; + if !self.dry_run { + validate_xcode_supports_ios_deployment_target(&self.deployment_target)?; + } let framework_name = self.crate_name.replace("-", "_"); let ios_dir = self.output_dir.join("ios"); @@ -279,11 +387,16 @@ impl IosBuilder { // Step 0: Ensure iOS project scaffolding exists // Pass project_root and crate_dir for better benchmark function detection - crate::codegen::ensure_ios_project_with_options( + crate::codegen::ensure_ios_project_with_project_options( &self.output_dir, &self.crate_name, Some(&self.project_root), self.crate_dir.as_deref(), + IosProjectOptions { + deployment_target: self.deployment_target.clone(), + runner, + ios_benchmark_timeout_secs: resolve_ios_benchmark_timeout_secs_from_env(), + }, )?; // Step 1: Build Rust libraries @@ -1494,6 +1607,7 @@ impl IosBuilder { /// # Ok::<(), mobench_sdk::BenchError>(()) /// ``` pub fn package_ipa(&self, scheme: &str, method: SigningMethod) -> Result { + validate_xcode_supports_ios_deployment_target(&self.deployment_target)?; // For repository structure: ios/BenchRunner/BenchRunner.xcodeproj // The directory and scheme happen to have the same name let ios_dir = self.output_dir.join("ios").join(scheme); @@ -1782,6 +1896,7 @@ impl IosBuilder { /// This requires the app project to be generated first with `build()`. /// The resulting zip can be supplied to BrowserStack as the test suite. pub fn package_xcuitest(&self, scheme: &str) -> Result { + validate_xcode_supports_ios_deployment_target(&self.deployment_target)?; let ios_dir = self.output_dir.join("ios").join(scheme); let project_path = ios_dir.join(format!("{}.xcodeproj", scheme)); @@ -2090,6 +2205,26 @@ mod tests { assert_eq!(builder.output_dir, PathBuf::from("/custom/output")); } + #[test] + fn parse_xcode_version_reads_major_minor() { + let parsed = parse_xcode_version("Xcode 16.2\nBuild version 16C5032a\n").unwrap(); + assert_eq!(parsed.major, 16); + assert_eq!(parsed.minor, 2); + assert_eq!(parsed.raw, "16.2"); + } + + #[test] + fn xcode_floor_rejects_legacy_target_on_current_lane() { + let xcode = XcodeVersion { + major: 16, + minor: 2, + raw: "16.2".to_string(), + }; + let floor = minimum_supported_ios_deployment_target_for_xcode(&xcode).unwrap(); + assert_eq!(floor.to_string(), "15.0"); + assert!(IosDeploymentTarget::parse("10.0").unwrap() < floor); + } + #[cfg(target_os = "macos")] #[test] fn test_validate_ipa_archive_rejects_missing_info_plist() { diff --git a/crates/mobench-sdk/src/codegen.rs b/crates/mobench-sdk/src/codegen.rs index 43d9566..45ededf 100644 --- a/crates/mobench-sdk/src/codegen.rs +++ b/crates/mobench-sdk/src/codegen.rs @@ -12,7 +12,161 @@ use include_dir::{Dir, DirEntry, include_dir}; const ANDROID_TEMPLATES: Dir = include_dir!("$CARGO_MANIFEST_DIR/templates/android"); const IOS_TEMPLATES: Dir = include_dir!("$CARGO_MANIFEST_DIR/templates/ios"); -const DEFAULT_IOS_BENCHMARK_TIMEOUT_SECS: u64 = 300; +pub const DEFAULT_IOS_BENCHMARK_TIMEOUT_SECS: u64 = 300; +pub const DEFAULT_IOS_DEPLOYMENT_TARGET: &str = "15.0"; +pub const SWIFTUI_RUNNER_MIN_IOS: &str = "15.0"; + +/// Supported generated iOS application runner templates. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IosRunner { + /// Current SwiftUI runner. This is the default for iOS 15+. + Swiftui, + /// UIKit-based runner for legacy deployment targets. + UikitLegacy, +} + +impl IosRunner { + pub fn parse(value: &str) -> Result { + match value.trim().to_ascii_lowercase().as_str() { + "swiftui" => Ok(Self::Swiftui), + "uikit-legacy" | "uikit_legacy" => Ok(Self::UikitLegacy), + other => Err(BenchError::Build(format!( + "Unsupported iOS runner `{other}`. Supported values: swiftui, uikit-legacy" + ))), + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::Swiftui => "swiftui", + Self::UikitLegacy => "uikit-legacy", + } + } +} + +/// Parsed iOS deployment target used for explicit compatibility decisions. +#[derive(Debug, Clone, Eq)] +pub struct IosDeploymentTarget { + major: u16, + minor: u16, + patch: u16, + raw: String, +} + +impl PartialEq for IosDeploymentTarget { + fn eq(&self, other: &Self) -> bool { + (self.major, self.minor, self.patch) == (other.major, other.minor, other.patch) + } +} + +impl Ord for IosDeploymentTarget { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + (self.major, self.minor, self.patch).cmp(&(other.major, other.minor, other.patch)) + } +} + +impl PartialOrd for IosDeploymentTarget { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl IosDeploymentTarget { + pub fn parse(value: &str) -> Result { + let raw = value.trim(); + if raw.is_empty() { + return Err(BenchError::Build( + "iOS deployment target must not be empty".to_string(), + )); + } + + let parts = raw.split('.').collect::>(); + if parts.len() > 3 { + return Err(BenchError::Build(format!( + "Invalid iOS deployment target `{raw}`. Expected VERSION like 15.0" + ))); + } + + let major = parse_ios_version_part(raw, parts[0], "major")?; + let minor = parts + .get(1) + .map(|part| parse_ios_version_part(raw, part, "minor")) + .transpose()? + .unwrap_or(0); + let patch = parts + .get(2) + .map(|part| parse_ios_version_part(raw, part, "patch")) + .transpose()? + .unwrap_or(0); + + Ok(Self { + major, + minor, + patch, + raw: raw.to_string(), + }) + } + + pub fn default_target() -> Self { + Self::parse(DEFAULT_IOS_DEPLOYMENT_TARGET) + .expect("default iOS deployment target should be valid") + } +} + +fn parse_ios_version_part(raw: &str, part: &str, label: &str) -> Result { + if part.is_empty() || !part.chars().all(|ch| ch.is_ascii_digit()) { + return Err(BenchError::Build(format!( + "Invalid iOS deployment target `{raw}`: {label} version component must be numeric" + ))); + } + part.parse::().map_err(|err| { + BenchError::Build(format!( + "Invalid iOS deployment target `{raw}`: failed to parse {label} component: {err}" + )) + }) +} + +impl std::fmt::Display for IosDeploymentTarget { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.raw) + } +} + +/// Fully resolved iOS project generation options. +#[derive(Debug, Clone)] +pub struct IosProjectOptions { + pub deployment_target: IosDeploymentTarget, + pub runner: IosRunner, + pub ios_benchmark_timeout_secs: u64, +} + +impl Default for IosProjectOptions { + fn default() -> Self { + Self { + deployment_target: IosDeploymentTarget::default_target(), + runner: IosRunner::Swiftui, + ios_benchmark_timeout_secs: DEFAULT_IOS_BENCHMARK_TIMEOUT_SECS, + } + } +} + +pub fn resolve_ios_runner( + deployment_target: &IosDeploymentTarget, + requested_runner: Option, +) -> Result { + let swiftui_floor = IosDeploymentTarget::parse(SWIFTUI_RUNNER_MIN_IOS)?; + match requested_runner { + Some(IosRunner::Swiftui) if deployment_target < &swiftui_floor => { + Err(BenchError::Build(format!( + "iOS runner `swiftui` requires deployment target {SWIFTUI_RUNNER_MIN_IOS}+; \ + requested deployment target is {deployment_target}. Use `uikit-legacy` or raise the deployment target." + ))) + } + Some(runner) => Ok(runner), + None if deployment_target < &swiftui_floor => Ok(IosRunner::UikitLegacy), + None => Ok(IosRunner::Swiftui), + } +} /// Template variable that can be replaced in template files #[derive(Debug, Clone)] @@ -555,16 +709,21 @@ pub fn generate_ios_project( .ok() .as_deref(), ); - generate_ios_project_with_timeout( + generate_ios_project_with_options( output_dir, project_slug, project_pascal, bundle_prefix, default_function, - ios_benchmark_timeout_secs, + IosProjectOptions { + ios_benchmark_timeout_secs, + ..IosProjectOptions::default() + }, ) } +#[cfg(test)] +#[allow(dead_code)] fn generate_ios_project_with_timeout( output_dir: &Path, project_slug: &str, @@ -573,6 +732,28 @@ fn generate_ios_project_with_timeout( default_function: &str, ios_benchmark_timeout_secs: u64, ) -> Result<(), BenchError> { + generate_ios_project_with_options( + output_dir, + project_slug, + project_pascal, + bundle_prefix, + default_function, + IosProjectOptions { + ios_benchmark_timeout_secs, + ..IosProjectOptions::default() + }, + ) +} + +pub fn generate_ios_project_with_options( + output_dir: &Path, + project_slug: &str, + project_pascal: &str, + bundle_prefix: &str, + default_function: &str, + options: IosProjectOptions, +) -> Result<(), BenchError> { + let runner = resolve_ios_runner(&options.deployment_target, Some(options.runner))?; let target_dir = output_dir.join("ios"); let preserved_resources = collect_preserved_ios_resources(&target_dir)?; reset_generated_project_dir(&target_dir)?; @@ -612,10 +793,18 @@ fn generate_ios_project_with_timeout( }, TemplateVar { name: "IOS_BENCHMARK_TIMEOUT_SECS", - value: ios_benchmark_timeout_secs.to_string(), + value: options.ios_benchmark_timeout_secs.to_string(), + }, + TemplateVar { + name: "IOS_DEPLOYMENT_TARGET", + value: options.deployment_target.to_string(), + }, + TemplateVar { + name: "IOS_RUNNER", + value: runner.as_str().to_string(), }, ]; - render_dir(&IOS_TEMPLATES, &target_dir, &vars)?; + render_ios_dir(&IOS_TEMPLATES, &target_dir, &vars, runner)?; restore_preserved_ios_resources(&target_dir, &preserved_resources)?; Ok(()) } @@ -737,6 +926,32 @@ const TEMPLATE_EXTENSIONS: &[&str] = &[ ]; fn render_dir(dir: &Dir, out_root: &Path, vars: &[TemplateVar]) -> Result<(), BenchError> { + render_dir_filtered(dir, out_root, vars, &|_| false) +} + +fn render_ios_dir( + dir: &Dir, + out_root: &Path, + vars: &[TemplateVar], + runner: IosRunner, +) -> Result<(), BenchError> { + render_dir_filtered(dir, out_root, vars, &|path| match runner { + IosRunner::Swiftui => { + path == Path::new("BenchRunner/BenchRunner/UIKitLegacyRunner.swift.template") + } + IosRunner::UikitLegacy => { + path == Path::new("BenchRunner/BenchRunner/BenchRunnerApp.swift.template") + || path == Path::new("BenchRunner/BenchRunner/ContentView.swift.template") + } + }) +} + +fn render_dir_filtered( + dir: &Dir, + out_root: &Path, + vars: &[TemplateVar], + skip_file: &dyn Fn(&Path) -> bool, +) -> Result<(), BenchError> { for entry in dir.entries() { match entry { DirEntry::Dir(sub) => { @@ -744,12 +959,15 @@ fn render_dir(dir: &Dir, out_root: &Path, vars: &[TemplateVar]) -> Result<(), Be if sub.path().components().any(|c| c.as_os_str() == ".gradle") { continue; } - render_dir(sub, out_root, vars)?; + render_dir_filtered(sub, out_root, vars, skip_file)?; } DirEntry::File(file) => { if file.path().components().any(|c| c.as_os_str() == ".gradle") { continue; } + if skip_file(file.path()) { + continue; + } // file.path() returns the full relative path from the embedded dir root let mut relative = file.path().to_path_buf(); let mut contents = file.contents().to_vec(); @@ -1219,6 +1437,22 @@ pub fn ensure_ios_project_with_options( crate_name: &str, project_root: Option<&Path>, crate_dir: Option<&Path>, +) -> Result<(), BenchError> { + ensure_ios_project_with_project_options( + output_dir, + crate_name, + project_root, + crate_dir, + IosProjectOptions::default(), + ) +} + +pub fn ensure_ios_project_with_project_options( + output_dir: &Path, + crate_name: &str, + project_root: Option<&Path>, + crate_dir: Option<&Path>, + options: IosProjectOptions, ) -> Result<(), BenchError> { let library_name = crate_name.replace('-', "_"); let project_exists = ios_project_exists(output_dir); @@ -1244,12 +1478,13 @@ pub fn ensure_ios_project_with_options( let effective_root = project_root.unwrap_or_else(|| output_dir.parent().unwrap_or(output_dir)); let default_function = resolve_default_function(effective_root, crate_name, crate_dir); - generate_ios_project( + generate_ios_project_with_options( output_dir, &library_name, project_pascal, &bundle_prefix, &default_function, + options, )?; println!(" Generated iOS project at {:?}", output_dir.join("ios")); println!(" Default benchmark function: {}", default_function); @@ -1497,6 +1732,13 @@ mod tests { assert!(android.contains("startForegroundService(intent)")); assert!(android.contains("startForeground(FOREGROUND_NOTIFICATION_ID")); assert!(android.contains("fun isBenchmarkComplete()")); + assert!(android.contains("BENCH_JSON ${json}")); + assert!(android.contains("BENCH_HEARTBEAT_JSON $json")); + assert!(android.contains("BENCH_FAILURE_JSON $encoded")); + assert!(android.contains("getHistoricalProcessExitReasons")); + assert!(android.contains("ApplicationExitInfo.REASON_LOW_MEMORY")); + assert!(android.contains("android_benchmark_timeout_secs")); + assert!(android.contains("android_heartbeat_interval_secs")); assert!(!android.contains("resultLatch.await")); assert!(android.contains("memory_process\", \"isolated_worker\"")); @@ -1504,9 +1746,13 @@ mod tests { "../templates/android/app/src/androidTest/java/MainActivityTest.kt.template" ); assert!(android_test.contains("Log.i(\"BenchRunnerTest\"")); - assert!(android_test.contains("Thread.sleep(heartbeatMs)")); + assert!(android_test.contains("Thread.sleep(pollMs)")); assert!(android_test.contains("TimeUnit.SECONDS.toMillis(10)")); assert!(android_test.contains("activity.isBenchmarkComplete()")); + assert!(android_test.contains("activity.isBenchmarkFailed()")); + assert!(android_test.contains("activity.emitTimeoutFailureFromTest()")); + assert!(android_test.contains("activity.checkWorkerExit()")); + assert!(android_test.contains("Benchmark failed before BENCH_JSON")); let ios_test = include_str!( "../templates/ios/BenchRunner/BenchRunnerUITests/BenchRunnerUITests.swift.template" @@ -1559,6 +1805,45 @@ mod tests { assert!(ios.contains("\"memory_process\": \"benchmark_app\"")); assert!(ios.contains("generateJSONReport(report, runProcessPeakMemoryKb:")); assert!(ios.contains("processPeakSamplesKb.max() ?? runProcessPeakMemoryKb")); + + let legacy = include_str!( + "../templates/ios/BenchRunner/BenchRunner/UIKitLegacyRunner.swift.template" + ); + assert!(legacy.contains("import UIKit")); + assert!(!legacy.contains("import SwiftUI")); + assert!(!legacy.contains("Task.detached")); + assert!(!legacy.contains("Task.sleep")); + assert!(!legacy.contains("MainActor")); + assert!(legacy.contains("DispatchQueue.global(qos: .userInitiated)")); + assert!(legacy.contains("DispatchQueue.main.async")); + assert!(legacy.contains("textColor = .clear")); + assert!(!legacy.contains(".alpha = 0")); + assert!(legacy.contains("benchmarkReport")); + assert!(legacy.contains("benchmarkCompleted")); + assert!(legacy.contains("benchmarkReportJSON")); + assert!(legacy.contains("BENCH_REPORT_JSON_START")); + assert!(legacy.contains("BENCH_REPORT_JSON_END")); + } + + #[test] + fn test_ios_deployment_target_and_runner_selection() { + let ios15 = IosDeploymentTarget::parse("15.0").unwrap(); + let ios10 = IosDeploymentTarget::parse("10.0").unwrap(); + + assert_eq!(IosDeploymentTarget::parse("10").unwrap(), ios10); + assert_eq!( + resolve_ios_runner(&ios15, None).unwrap(), + IosRunner::Swiftui + ); + assert_eq!( + resolve_ios_runner(&ios10, None).unwrap(), + IosRunner::UikitLegacy + ); + assert!(resolve_ios_runner(&ios10, Some(IosRunner::Swiftui)).is_err()); + assert_eq!( + resolve_ios_runner(&ios15, Some(IosRunner::UikitLegacy)).unwrap(), + IosRunner::UikitLegacy + ); } #[test] @@ -1910,6 +2195,49 @@ pub fn public_bench() { fs::remove_dir_all(&temp_dir).ok(); } + #[test] + fn test_generate_ios_project_uses_configured_deployment_target() { + let temp_dir = env::temp_dir().join("mobench-sdk-ios-deployment-target-test"); + let _ = fs::remove_dir_all(&temp_dir); + fs::create_dir_all(&temp_dir).unwrap(); + + generate_ios_project_with_options( + &temp_dir, + "sample_fns", + "BenchRunner", + "dev.world.samplefns", + "sample_fns::example_benchmark", + IosProjectOptions { + deployment_target: IosDeploymentTarget::parse("10.0").unwrap(), + runner: IosRunner::UikitLegacy, + ios_benchmark_timeout_secs: 300, + }, + ) + .expect("generate legacy iOS project"); + + let project_yml = fs::read_to_string(temp_dir.join("ios/BenchRunner/project.yml")).unwrap(); + assert!(project_yml.contains("deploymentTarget: \"10.0\"")); + assert!(!project_yml.contains("deploymentTarget: \"15.0\"")); + + let runner = fs::read_to_string( + temp_dir.join("ios/BenchRunner/BenchRunner/UIKitLegacyRunner.swift"), + ) + .unwrap(); + assert!(runner.contains("import UIKit")); + assert!( + !temp_dir + .join("ios/BenchRunner/BenchRunner/ContentView.swift") + .exists() + ); + assert!( + !temp_dir + .join("ios/BenchRunner/BenchRunner/BenchRunnerApp.swift") + .exists() + ); + + fs::remove_dir_all(&temp_dir).ok(); + } + #[test] fn test_resolve_ios_benchmark_timeout_secs_defaults_invalid_values() { assert_eq!(resolve_ios_benchmark_timeout_secs(None), 300); diff --git a/crates/mobench-sdk/templates/android/app/src/androidTest/java/MainActivityTest.kt.template b/crates/mobench-sdk/templates/android/app/src/androidTest/java/MainActivityTest.kt.template index 24205e0..490f58d 100644 --- a/crates/mobench-sdk/templates/android/app/src/androidTest/java/MainActivityTest.kt.template +++ b/crates/mobench-sdk/templates/android/app/src/androidTest/java/MainActivityTest.kt.template @@ -25,28 +25,47 @@ class MainActivityTest { @Test fun showsBenchOutput() { - val deadline = System.currentTimeMillis() + benchmarkTimeoutMs + var timeoutMs = benchmarkTimeoutMs + var pollMs = heartbeatMs + activityRule.scenario.onActivity { activity -> + timeoutMs = TimeUnit.SECONDS.toMillis(activity.benchmarkTimeoutSecs()) + pollMs = TimeUnit.SECONDS.toMillis(activity.heartbeatIntervalSecs()).coerceAtLeast(1000L) + } + + val deadline = System.currentTimeMillis() + timeoutMs var completed = false + var failed = false + var failureJson: String? = null while (System.currentTimeMillis() < deadline) { activityRule.scenario.onActivity { activity -> + activity.checkWorkerExit() completed = activity.isBenchmarkComplete() + failed = activity.isBenchmarkFailed() + failureJson = activity.getBenchmarkFailureJson() } - if (completed) { + if (completed || failed) { break } Log.i("BenchRunnerTest", "Waiting for benchmark output...") try { - Thread.sleep(heartbeatMs) + Thread.sleep(pollMs) } catch (e: InterruptedException) { Thread.currentThread().interrupt() fail("Interrupted while waiting for benchmark output") } } - if (!completed) { - fail("Timed out waiting for benchmark output") + if (!completed && !failed) { + activityRule.scenario.onActivity { activity -> + failureJson = activity.emitTimeoutFailureFromTest() + failed = true + } + } + + if (failed) { + fail("Benchmark failed before BENCH_JSON: ${failureJson ?: "missing failure payload"}") } onView(withId(R.id.result_text)) diff --git a/crates/mobench-sdk/templates/android/app/src/main/java/MainActivity.kt.template b/crates/mobench-sdk/templates/android/app/src/main/java/MainActivity.kt.template index e60d924..1cd444a 100644 --- a/crates/mobench-sdk/templates/android/app/src/main/java/MainActivity.kt.template +++ b/crates/mobench-sdk/templates/android/app/src/main/java/MainActivity.kt.template @@ -3,7 +3,11 @@ package {{PACKAGE_NAME}} import android.app.Notification import android.app.NotificationChannel import android.app.NotificationManager +import android.app.ActivityManager +import android.app.Application +import android.app.ApplicationExitInfo import android.app.Service +import android.content.Context import android.content.Intent import android.os.Build import android.os.Bundle @@ -11,6 +15,7 @@ import android.os.Debug import android.os.Handler import android.os.IBinder import android.os.Looper +import android.os.Process import android.os.ResultReceiver import android.widget.TextView import androidx.appcompat.app.AppCompatActivity @@ -24,28 +29,48 @@ import uniffi.{{UNIFFI_NAMESPACE}}.runBenchmark private const val DEFAULT_FUNCTION = "{{DEFAULT_FUNCTION}}" private const val DEFAULT_ITERATIONS = 20u private const val DEFAULT_WARMUP = 3u +private const val DEFAULT_ANDROID_BENCHMARK_TIMEOUT_SECS = 1800L +private const val DEFAULT_ANDROID_HEARTBEAT_INTERVAL_SECS = 10L private const val FUNCTION_EXTRA = "bench_function" private const val ITERATIONS_EXTRA = "bench_iterations" private const val WARMUP_EXTRA = "bench_warmup" +private const val TIMEOUT_EXTRA = "bench_timeout_secs" +private const val HEARTBEAT_EXTRA = "bench_heartbeat_secs" private const val SPEC_ASSET = "bench_spec.json" private const val RUN_BENCHMARK_ACTION = "{{PACKAGE_NAME}}.RUN_BENCHMARK" private const val RESULT_RECEIVER_EXTRA = "bench_result_receiver" private const val RESULT_DISPLAY_EXTRA = "bench_display" private const val RESULT_ERROR_EXTRA = "bench_error" +private const val RESULT_FAILURE_JSON_EXTRA = "bench_failure_json" +private const val RESULT_HEARTBEAT_JSON_EXTRA = "bench_heartbeat_json" +private const val RESULT_PID_EXTRA = "bench_pid" +private const val RESULT_PROCESS_NAME_EXTRA = "bench_process_name" private const val RESULT_OK = 1 private const val RESULT_ERROR = 2 +private const val RESULT_HEARTBEAT = 3 private const val FOREGROUND_NOTIFICATION_ID = 1001 private const val FOREGROUND_CHANNEL_ID = "mobench_benchmark" private const val FOREGROUND_CHANNEL_NAME = "Mobench benchmark" +private const val WORKER_PROCESS_SUFFIX = ":mobench_worker" +private const val FAILURE_SCHEMA_VERSION = 1 class MainActivity : AppCompatActivity() { @Volatile private var benchmarkComplete = false - - private data class BenchParams( + @Volatile private var benchmarkFailed = false + @Volatile private var failureJson: String? = null + @Volatile private var workerPid: Int? = null + @Volatile private var workerProcessName: String? = null + @Volatile private var lastProgressAtMs: Long? = null + @Volatile private var startedAtMs: Long = 0L + private lateinit var params: BenchParams + + data class BenchParams( val function: String, val iterations: UInt, val warmup: UInt, + val timeoutSecs: Long, + val heartbeatIntervalSecs: Long, ) override fun onCreate(savedInstanceState: Bundle?) { @@ -55,19 +80,38 @@ class MainActivity : AppCompatActivity() { val resultText = findViewById(R.id.result_text) resultText?.text = "Running benchmark..." - val params = resolveBenchParams() + params = resolveBenchParams() + startedAtMs = android.os.SystemClock.elapsedRealtime() + lastProgressAtMs = startedAtMs val resultReceiver = object : ResultReceiver(Handler(Looper.getMainLooper())) { override fun onReceiveResult(resultCode: Int, resultData: Bundle?) { - val display = resultData?.getString(RESULT_DISPLAY_EXTRA) - ?: resultData?.getString(RESULT_ERROR_EXTRA) - ?: "Benchmark worker returned no result" - resultText?.text = display - benchmarkComplete = true - - if (resultCode == RESULT_OK) { - android.util.Log.i("BenchRunner", "Benchmark worker completed") - } else { - android.util.Log.e("BenchRunner", display) + when (resultCode) { + RESULT_HEARTBEAT -> { + workerPid = resultData?.getInt(RESULT_PID_EXTRA)?.takeIf { it > 0 } + workerProcessName = resultData?.getString(RESULT_PROCESS_NAME_EXTRA) + lastProgressAtMs = android.os.SystemClock.elapsedRealtime() + } + RESULT_OK -> { + val display = resultData?.getString(RESULT_DISPLAY_EXTRA) + ?: "Benchmark worker completed" + resultText?.text = display + benchmarkComplete = true + android.util.Log.i("BenchRunner", "Benchmark worker completed") + } + else -> { + val payload = resultData?.getString(RESULT_FAILURE_JSON_EXTRA) + val display = resultData?.getString(RESULT_DISPLAY_EXTRA) + ?: resultData?.getString(RESULT_ERROR_EXTRA) + ?: "Benchmark worker returned no result" + resultText?.text = display + benchmarkFailed = true + benchmarkComplete = true + failureJson = payload + if (payload == null) { + emitFailure("unknown", display) + } + android.util.Log.e("BenchRunner", display) + } } } } @@ -78,6 +122,8 @@ class MainActivity : AppCompatActivity() { putExtra(FUNCTION_EXTRA, params.function) putExtra(ITERATIONS_EXTRA, params.iterations.toInt()) putExtra(WARMUP_EXTRA, params.warmup.toInt()) + putExtra(TIMEOUT_EXTRA, params.timeoutSecs) + putExtra(HEARTBEAT_EXTRA, params.heartbeatIntervalSecs) putExtra(RESULT_RECEIVER_EXTRA, resultReceiver) } if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { @@ -87,8 +133,9 @@ class MainActivity : AppCompatActivity() { } } catch (e: Exception) { android.util.Log.e("BenchRunner", "Failed to run benchmark worker", e) - resultText?.text = "Failed to run benchmark worker: ${e.message}" - benchmarkComplete = true + val message = "Failed to run benchmark worker: ${e.message}" + resultText?.text = message + emitFailure("exception", message) } } @@ -96,12 +143,79 @@ class MainActivity : AppCompatActivity() { return benchmarkComplete } + fun isBenchmarkFailed(): Boolean { + return benchmarkFailed + } + + fun getBenchmarkFailureJson(): String? { + return failureJson + } + + fun benchmarkTimeoutSecs(): Long { + return params.timeoutSecs + } + + fun heartbeatIntervalSecs(): Long { + return params.heartbeatIntervalSecs + } + + fun checkWorkerExit(): String? { + if (benchmarkComplete || workerPid == null) { + return failureJson + } + + val pid = workerPid ?: return failureJson + val processName = workerProcessName ?: workerProcessName() + val alive = runningAppProcesses().any { + it.pid == pid || it.processName == processName + } + val graceMs = params.heartbeatIntervalSecs.coerceAtLeast(1L) * 3_000L + val stale = android.os.SystemClock.elapsedRealtime() - (lastProgressAtMs ?: startedAtMs) > graceMs + if (!alive && stale) { + emitFailure("worker_exit", "Benchmark worker process exited before BENCH_JSON was emitted") + } + return failureJson + } + + fun emitTimeoutFailureFromTest(): String { + checkWorkerExit() + if (failureJson == null) { + emitFailure("timeout", "Timed out waiting ${params.timeoutSecs}s for benchmark completion") + } + return failureJson ?: "{}" + } + + private fun emitFailure(kind: String, message: String) { + if (failureJson != null) { + benchmarkFailed = true + benchmarkComplete = true + return + } + val payload = buildFailureJson( + this, + params.function, + kind, + message, + startedAtMs, + lastProgressAtMs, + workerPid, + workerProcessName ?: workerProcessName(), + ) + val encoded = payload.toString() + failureJson = encoded + benchmarkFailed = true + benchmarkComplete = true + android.util.Log.e("BenchRunner", "BENCH_FAILURE_JSON $encoded") + } + private fun resolveBenchParams(): BenchParams { val assetParams = loadBenchParamsFromAssets() val defaults = assetParams ?: BenchParams( DEFAULT_FUNCTION, DEFAULT_ITERATIONS, - DEFAULT_WARMUP + DEFAULT_WARMUP, + DEFAULT_ANDROID_BENCHMARK_TIMEOUT_SECS, + DEFAULT_ANDROID_HEARTBEAT_INTERVAL_SECS, ) // Check for intent extras used by local automation, smoke tests, and provider-driven runs. @@ -119,6 +233,9 @@ class MainActivity : AppCompatActivity() { val fn = intentFunction ?: defaults.function val iterations = intentIterations ?: defaults.iterations val warmup = intentWarmup ?: defaults.warmup + val timeoutSecs = intent?.getLongExtra(TIMEOUT_EXTRA, defaults.timeoutSecs) ?: defaults.timeoutSecs + val heartbeatSecs = intent?.getLongExtra(HEARTBEAT_EXTRA, defaults.heartbeatIntervalSecs) + ?: defaults.heartbeatIntervalSecs // Log the resolution source for debugging if (assetParams == null && intentFunction == null && intentIterations == null && intentWarmup == null) { @@ -136,7 +253,7 @@ class MainActivity : AppCompatActivity() { android.util.Log.i("BenchRunner", "Resolved params: function=$fn, iterations=$iterations, warmup=$warmup (sources: ${sources.joinToString(", ")})") } - return BenchParams(fn, iterations, warmup) + return BenchParams(fn, iterations, warmup, timeoutSecs, heartbeatSecs) } private fun loadBenchParamsFromAssets(): BenchParams? { @@ -179,9 +296,13 @@ class MainActivity : AppCompatActivity() { android.util.Log.w("BenchRunner", "Config missing 'warmup' key, using default: $DEFAULT_WARMUP") DEFAULT_WARMUP } + val timeoutSecs = json.optLong("android_benchmark_timeout_secs", DEFAULT_ANDROID_BENCHMARK_TIMEOUT_SECS) + .takeIf { it > 0L } ?: DEFAULT_ANDROID_BENCHMARK_TIMEOUT_SECS + val heartbeatSecs = json.optLong("android_heartbeat_interval_secs", DEFAULT_ANDROID_HEARTBEAT_INTERVAL_SECS) + .takeIf { it > 0L } ?: DEFAULT_ANDROID_HEARTBEAT_INTERVAL_SECS - android.util.Log.i("BenchRunner", "Loaded config from bench_spec.json: function=$function, iterations=$iterations, warmup=$warmup") - BenchParams(function, iterations, warmup) + android.util.Log.i("BenchRunner", "Loaded config from bench_spec.json: function=$function, iterations=$iterations, warmup=$warmup, timeout=${timeoutSecs}s, heartbeat=${heartbeatSecs}s") + BenchParams(function, iterations, warmup, timeoutSecs, heartbeatSecs) } } catch (e: java.io.FileNotFoundException) { android.util.Log.d("BenchRunner", "No bench_spec.json in assets, will use intent extras or defaults") @@ -195,10 +316,12 @@ class MainActivity : AppCompatActivity() { class BenchmarkWorkerService : Service() { - private data class BenchParams( + data class BenchParams( val function: String, val iterations: UInt, val warmup: UInt, + val timeoutSecs: Long, + val heartbeatIntervalSecs: Long, ) override fun onBind(intent: Intent?): IBinder? = null @@ -214,19 +337,26 @@ class BenchmarkWorkerService : Service() { function = intent.getStringExtra(FUNCTION_EXTRA)?.takeUnless { it.isBlank() } ?: DEFAULT_FUNCTION, iterations = intent.getIntExtra(ITERATIONS_EXTRA, DEFAULT_ITERATIONS.toInt()).toUInt(), warmup = intent.getIntExtra(WARMUP_EXTRA, DEFAULT_WARMUP.toInt()).toUInt(), + timeoutSecs = intent.getLongExtra(TIMEOUT_EXTRA, DEFAULT_ANDROID_BENCHMARK_TIMEOUT_SECS), + heartbeatIntervalSecs = intent.getLongExtra(HEARTBEAT_EXTRA, DEFAULT_ANDROID_HEARTBEAT_INTERVAL_SECS), ) startBenchmarkForeground() Thread { + val startMs = android.os.SystemClock.elapsedRealtime() + val heartbeat = WorkerHeartbeat(resultReceiver, params, startMs) + heartbeat.start() try { val result = runBenchmarkInWorker(params) val bundle = Bundle().apply { putString(RESULT_DISPLAY_EXTRA, result.displayText) result.errorMessage?.let { putString(RESULT_ERROR_EXTRA, it) } + result.failureJson?.let { putString(RESULT_FAILURE_JSON_EXTRA, it) } } resultReceiver?.send(if (result.errorMessage == null) RESULT_OK else RESULT_ERROR, bundle) } finally { + heartbeat.stop() stopBenchmarkForeground() stopSelf(startId) } @@ -284,6 +414,7 @@ class BenchmarkWorkerService : Service() { private fun runBenchmarkInWorker(params: BenchParams): WorkerBenchmarkResult { BenchNativeLibrary.ensureLoaded() + val startMs = android.os.SystemClock.elapsedRealtime() val display = try { val spec = BenchSpec( name = params.function, @@ -307,27 +438,102 @@ class BenchmarkWorkerService : Service() { formatBenchReport(report) } catch (e: BenchException) { android.util.Log.e("BenchRunner", "Benchmark error: ${e.message}", e) + val payload = buildFailureJson( + this, + params.function, + "exception", + "Benchmark error: ${e.message}", + startMs, + android.os.SystemClock.elapsedRealtime(), + Process.myPid(), + currentProcessName() + ).toString() + android.util.Log.e("BenchRunner", "BENCH_FAILURE_JSON $payload") return WorkerBenchmarkResult( displayText = "Benchmark error: ${e.message}", - errorMessage = "Benchmark error: ${e.message}" + errorMessage = "Benchmark error: ${e.message}", + failureJson = payload ) } catch (e: Exception) { android.util.Log.e("BenchRunner", "Unexpected error during benchmark execution", e) + val payload = buildFailureJson( + this, + params.function, + "exception", + "Unexpected error: ${e.message}", + startMs, + android.os.SystemClock.elapsedRealtime(), + Process.myPid(), + currentProcessName() + ).toString() + android.util.Log.e("BenchRunner", "BENCH_FAILURE_JSON $payload") return WorkerBenchmarkResult( displayText = "Unexpected error: ${e.message}", - errorMessage = "Unexpected error: ${e.message}" + errorMessage = "Unexpected error: ${e.message}", + failureJson = payload ) } - return WorkerBenchmarkResult(displayText = display, errorMessage = null) + return WorkerBenchmarkResult(displayText = display, errorMessage = null, failureJson = null) } } private data class WorkerBenchmarkResult( val displayText: String, val errorMessage: String?, + val failureJson: String?, ) +private class WorkerHeartbeat( + private val receiver: ResultReceiver?, + private val params: BenchmarkWorkerService.BenchParams, + private val startMs: Long, +) { + @Volatile private var running = false + private var thread: Thread? = null + + fun start() { + if (running) { + return + } + running = true + thread = Thread { + while (running) { + val now = android.os.SystemClock.elapsedRealtime() + val json = JSONObject() + .put("schema_version", FAILURE_SCHEMA_VERSION) + .put("platform", "android") + .put("function_name", params.function) + .put("elapsed_ms", now - startMs) + .put("pid", Process.myPid()) + .put("process_name", currentProcessName()) + .put("memory", currentMemoryJson()) + android.util.Log.i("BenchRunner", "BENCH_HEARTBEAT_JSON $json") + receiver?.send(RESULT_HEARTBEAT, Bundle().apply { + putString(RESULT_HEARTBEAT_JSON_EXTRA, json.toString()) + putInt(RESULT_PID_EXTRA, Process.myPid()) + putString(RESULT_PROCESS_NAME_EXTRA, currentProcessName()) + }) + try { + Thread.sleep(params.heartbeatIntervalSecs.coerceAtLeast(1L) * 1000L) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + break + } + } + }.apply { + name = "mobench-worker-heartbeat" + isDaemon = true + start() + } + } + + fun stop() { + running = false + thread?.interrupt() + } +} + private object BenchNativeLibrary { init { System.loadLibrary("{{LIBRARY_NAME}}") @@ -336,6 +542,121 @@ private object BenchNativeLibrary { fun ensureLoaded() = Unit } +private fun buildFailureJson( + context: Context, + functionName: String, + kind: String, + message: String, + startedAtMs: Long, + lastProgressAtMs: Long?, + pid: Int?, + processName: String?, +): JSONObject { + val elapsedNow = android.os.SystemClock.elapsedRealtime() + val json = JSONObject() + json.put("schema_version", FAILURE_SCHEMA_VERSION) + json.put("platform", "android") + json.put("device", "${Build.MANUFACTURER} ${Build.MODEL}".trim()) + json.put("function_name", functionName) + json.put("kind", kind) + json.put("message", message) + json.put("elapsed_ms", elapsedNow - startedAtMs) + if (pid != null) { + json.put("pid", pid) + } else { + json.put("pid", JSONObject.NULL) + } + json.put("process_name", processName ?: JSONObject.NULL) + json.put("last_progress_at_ms", lastProgressAtMs ?: JSONObject.NULL) + json.put("memory", currentMemoryJson()) + json.put("android_exit_info", androidExitInfoJson(context, pid, processName) ?: JSONObject.NULL) + return json +} + +private fun androidExitInfoJson(context: Context, pid: Int?, processName: String?): JSONObject? { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) { + return null + } + return try { + val manager = context.getSystemService(ActivityManager::class.java) ?: return null + val reasons = manager.getHistoricalProcessExitReasons(context.packageName, pid ?: 0, 10) + val match = reasons.firstOrNull { + (pid != null && it.pid == pid) || (processName != null && it.processName == processName) + } ?: reasons.firstOrNull() + match?.let { info -> + JSONObject() + .put("reason", exitReasonName(info.reason)) + .put("raw_reason", info.reason) + .put("description", info.description ?: JSONObject.NULL) + .put("importance", info.importance) + .put("timestamp", info.timestamp) + .put("pid", info.pid) + .put("process_name", info.processName ?: JSONObject.NULL) + } + } catch (e: Exception) { + android.util.Log.d("BenchRunner", "Unable to read historical process exit reasons", e) + null + } +} + +private fun exitReasonName(reason: Int): String { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) { + return "unknown" + } + return when (reason) { + ApplicationExitInfo.REASON_ANR -> "anr" + ApplicationExitInfo.REASON_CRASH -> "crash" + ApplicationExitInfo.REASON_CRASH_NATIVE -> "crash_native" + ApplicationExitInfo.REASON_DEPENDENCY_DIED -> "dependency_died" + ApplicationExitInfo.REASON_EXCESSIVE_RESOURCE_USAGE -> "excessive_resource_usage" + ApplicationExitInfo.REASON_EXIT_SELF -> "exit_self" + ApplicationExitInfo.REASON_INITIALIZATION_FAILURE -> "initialization_failure" + ApplicationExitInfo.REASON_LOW_MEMORY -> "low_memory" + ApplicationExitInfo.REASON_OTHER -> "other" + ApplicationExitInfo.REASON_PERMISSION_CHANGE -> "permission_change" + ApplicationExitInfo.REASON_SIGNALED -> "signaled" + ApplicationExitInfo.REASON_USER_REQUESTED -> "user_requested" + else -> "unknown" + } +} + +private fun currentMemoryJson(): JSONObject { + val memInfo = Debug.MemoryInfo() + return try { + Debug.getMemoryInfo(memInfo) + JSONObject() + .put("total_pss_kb", memInfo.totalPss) + .put("private_dirty_kb", memInfo.totalPrivateDirty) + .put("native_heap_kb", Debug.getNativeHeapAllocatedSize() / 1024) + .put("java_heap_kb", (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024) + .put("process_pss_kb", currentProcessPssKb() ?: JSONObject.NULL) + } catch (e: Exception) { + JSONObject() + } +} + +private fun Context.runningAppProcesses(): List { + return try { + val manager = getSystemService(ActivityManager::class.java) + manager?.runningAppProcesses ?: emptyList() + } catch (e: Exception) { + emptyList() + } +} + +private fun Context.workerProcessName(): String = "$packageName$WORKER_PROCESS_SUFFIX" + +private fun currentProcessName(): String { + if (Build.VERSION.SDK_INT >= 28) { + return Application.getProcessName() + } + return try { + java.io.File("/proc/self/cmdline").readText().trim('\u0000', ' ', '\n') + } catch (e: Exception) { + "unknown" + } +} + @Suppress("DEPRECATION") private fun Intent.resultReceiverExtra(): ResultReceiver? { return if (Build.VERSION.SDK_INT >= 33) { diff --git a/crates/mobench-sdk/templates/ios/BenchRunner/BenchRunner/UIKitLegacyRunner.swift.template b/crates/mobench-sdk/templates/ios/BenchRunner/BenchRunner/UIKitLegacyRunner.swift.template new file mode 100644 index 0000000..7797c53 --- /dev/null +++ b/crates/mobench-sdk/templates/ios/BenchRunner/BenchRunner/UIKitLegacyRunner.swift.template @@ -0,0 +1,163 @@ +import UIKit + +private struct ProfileLaunchOptions { + let benchDelayMs: UInt64 + let resultHoldMs: UInt64 + let repeatUntilMs: UInt64 + let warmupOnly: Bool + + static func resolved() -> ProfileLaunchOptions { + let info = ProcessInfo.processInfo + + var benchDelayMs = UInt64(info.environment["MOBENCH_BENCH_DELAY_MS"] ?? "0") ?? 0 + var resultHoldMs = UInt64( + info.environment["MOBENCH_PROFILE_RESULT_HOLD_MS"] ?? "5000" + ) ?? 5000 + var repeatUntilMs = UInt64( + info.environment["MOBENCH_PROFILE_REPEAT_UNTIL_MS"] ?? "0" + ) ?? 0 + var warmupOnly = info.environment["MOBENCH_PROFILE_WARMUP_ONLY"] == "1" + + for arg in info.arguments { + if arg.hasPrefix("--mobench-profile-bench-delay-ms="), + let value = arg.split(separator: "=", maxSplits: 1).last, + let parsed = UInt64(value) { + benchDelayMs = parsed + } else if arg.hasPrefix("--mobench-profile-result-hold-ms="), + let value = arg.split(separator: "=", maxSplits: 1).last, + let parsed = UInt64(value) { + resultHoldMs = parsed + } else if arg.hasPrefix("--mobench-profile-repeat-until-ms="), + let value = arg.split(separator: "=", maxSplits: 1).last, + let parsed = UInt64(value) { + repeatUntilMs = parsed + } else if arg == "--mobench-profile-warmup-only" + || arg == "--mobench-profile-warmup-only=1" { + warmupOnly = true + } + } + + NSLog( + "[BenchRunner] Profile launch options: delayMs=%llu, repeatUntilMs=%llu, resultHoldMs=%llu, warmupOnly=%@", + benchDelayMs, + repeatUntilMs, + resultHoldMs, + warmupOnly ? "true" : "false" + ) + + return ProfileLaunchOptions( + benchDelayMs: benchDelayMs, + resultHoldMs: resultHoldMs, + repeatUntilMs: repeatUntilMs, + warmupOnly: warmupOnly + ) + } +} + +@main +final class AppDelegate: UIResponder, UIApplicationDelegate { + var window: UIWindow? + + func application( + _ application: UIApplication, + didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? + ) -> Bool { + let window = UIWindow(frame: UIScreen.main.bounds) + window.rootViewController = BenchmarkViewController() + window.makeKeyAndVisible() + self.window = window + return true + } +} + +final class BenchmarkViewController: UIViewController { + private let reportView = UITextView() + private let completionLabel = UILabel() + private let jsonLabel = UILabel() + + override func viewDidLoad() { + super.viewDidLoad() + view.backgroundColor = .white + + reportView.translatesAutoresizingMaskIntoConstraints = false + reportView.isEditable = false + reportView.font = UIFont(name: "Menlo", size: 14) ?? UIFont.systemFont(ofSize: 14) + reportView.text = "Running benchmarks..." + reportView.accessibilityIdentifier = "benchmarkReport" + view.addSubview(reportView) + + completionLabel.translatesAutoresizingMaskIntoConstraints = false + completionLabel.text = "" + completionLabel.accessibilityIdentifier = "benchmarkCompleted" + completionLabel.isAccessibilityElement = false + completionLabel.textColor = .clear + view.addSubview(completionLabel) + + jsonLabel.translatesAutoresizingMaskIntoConstraints = false + jsonLabel.text = "" + jsonLabel.accessibilityIdentifier = "benchmarkReportJSON" + jsonLabel.isAccessibilityElement = true + jsonLabel.textColor = .clear + view.addSubview(jsonLabel) + + NSLayoutConstraint.activate([ + reportView.leadingAnchor.constraint(equalTo: view.leadingAnchor, constant: 16), + reportView.trailingAnchor.constraint(equalTo: view.trailingAnchor, constant: -16), + reportView.topAnchor.constraint(equalTo: view.topAnchor, constant: 16), + reportView.bottomAnchor.constraint(equalTo: view.bottomAnchor, constant: -16), + completionLabel.leadingAnchor.constraint(equalTo: view.leadingAnchor), + completionLabel.topAnchor.constraint(equalTo: view.topAnchor), + completionLabel.widthAnchor.constraint(equalToConstant: 1), + completionLabel.heightAnchor.constraint(equalToConstant: 1), + jsonLabel.leadingAnchor.constraint(equalTo: view.leadingAnchor), + jsonLabel.topAnchor.constraint(equalTo: completionLabel.bottomAnchor), + jsonLabel.widthAnchor.constraint(equalToConstant: 1), + jsonLabel.heightAnchor.constraint(equalToConstant: 1), + ]) + + runBenchmark() + } + + private func runBenchmark() { + DispatchQueue.global(qos: .userInitiated).async { + let options = ProfileLaunchOptions.resolved() + if options.benchDelayMs > 0 { + Thread.sleep(forTimeInterval: Double(options.benchDelayMs) / 1_000.0) + } + + let repeatDeadline = Date().addingTimeInterval( + Double(options.repeatUntilMs) / 1_000.0 + ) + var repeatedRuns = 1 + var result = {{PROJECT_NAME_PASCAL}}FFI.runCurrentBenchmark() + while !options.warmupOnly && options.repeatUntilMs > 0 && Date() < repeatDeadline { + result = {{PROJECT_NAME_PASCAL}}FFI.runCurrentBenchmark() + repeatedRuns += 1 + } + + DispatchQueue.main.async { + self.reportView.text = result.displayText + self.jsonLabel.text = result.jsonReport + self.jsonLabel.accessibilityLabel = result.jsonReport + self.completionLabel.text = "completed" + self.completionLabel.isAccessibilityElement = true + } + + NSLog("BENCH_REPORT_JSON_START") + NSLog("%@", result.jsonReport) + NSLog("BENCH_REPORT_JSON_END") + if repeatedRuns > 1 { + NSLog("Repeated benchmark %d time(s) during profile capture", repeatedRuns) + } + + if options.warmupOnly { + NSLog("Warmup-only profile run complete") + return + } + + NSLog("Displaying results for \(options.resultHoldMs) ms for capture output...") + Thread.sleep(forTimeInterval: Double(options.resultHoldMs) / 1_000.0) + NSLog("Display hold complete") + } + } +} diff --git a/crates/mobench-sdk/templates/ios/BenchRunner/project.yml.template b/crates/mobench-sdk/templates/ios/BenchRunner/project.yml.template index 7ccad1a..cdf35c5 100644 --- a/crates/mobench-sdk/templates/ios/BenchRunner/project.yml.template +++ b/crates/mobench-sdk/templates/ios/BenchRunner/project.yml.template @@ -8,7 +8,7 @@ targets: {{PROJECT_NAME_PASCAL}}: type: application platform: iOS - deploymentTarget: "15.0" + deploymentTarget: "{{IOS_DEPLOYMENT_TARGET}}" sources: - path: {{PROJECT_NAME_PASCAL}} resources: @@ -33,7 +33,7 @@ targets: {{PROJECT_NAME_PASCAL}}UITests: type: bundle.ui-testing platform: iOS - deploymentTarget: "15.0" + deploymentTarget: "{{IOS_DEPLOYMENT_TARGET}}" sources: - path: {{PROJECT_NAME_PASCAL}}UITests info: diff --git a/crates/mobench/src/browserstack.rs b/crates/mobench/src/browserstack.rs index c0dacc9..949b723 100644 --- a/crates/mobench/src/browserstack.rs +++ b/crates/mobench/src/browserstack.rs @@ -422,12 +422,30 @@ impl BrowserStackClient { /// * `platform` - "espresso" or "xcuitest" /// * `timeout_secs` - Maximum time to wait in seconds (default: 600) /// * `poll_interval_secs` - How often to check status in seconds (default: 10) + #[allow(dead_code)] pub fn poll_build_completion( &self, build_id: &str, platform: &str, timeout_secs: u64, poll_interval_secs: u64, + ) -> Result { + self.poll_build_completion_with_terminal_failures( + build_id, + platform, + timeout_secs, + poll_interval_secs, + false, + ) + } + + fn poll_build_completion_with_terminal_failures( + &self, + build_id: &str, + platform: &str, + timeout_secs: u64, + poll_interval_secs: u64, + allow_terminal_failure_status: bool, ) -> Result { use std::time::{Duration, Instant}; @@ -445,6 +463,9 @@ impl BrowserStackClient { match status.status.to_lowercase().as_str() { "done" | "passed" | "completed" => return Ok(status), "failed" | "error" | "timeout" => { + if allow_terminal_failure_status { + return Ok(status); + } return Err(anyhow!( "Build {} failed with status: {}", build_id, @@ -596,6 +617,26 @@ impl BrowserStackClient { } } + /// Extract structured Android benchmark failures from logs. + pub fn extract_benchmark_failures(&self, logs: &str) -> Result> { + let mut failures = Vec::new(); + let failure_marker = "BENCH_FAILURE_JSON "; + for line in logs.lines() { + if let Some(idx) = line.find(failure_marker) { + let json_part = &line[idx + failure_marker.len()..]; + if let Ok(json) = serde_json::from_str::(json_part) { + Self::extend_unique_results(&mut failures, vec![json]); + } + } + } + + if failures.is_empty() { + Err(anyhow!("No benchmark failures found in device logs")) + } else { + Ok(failures) + } + } + pub(crate) fn extract_benchmark_results_from_artifact( &self, contents: &str, @@ -656,6 +697,36 @@ impl BrowserStackClient { } } + pub(crate) fn extract_failures_from_session_artifacts( + &self, + session_json: &Value, + mut fetch_text: F, + ) -> Result> + where + F: FnMut(&str) -> Result, + { + let artifact_urls = Self::collect_text_artifact_urls(session_json); + if artifact_urls.is_empty() { + return Err(anyhow!("No text artifact URLs found in session response")); + } + + let mut failures = Vec::new(); + for (_, url) in artifact_urls { + let Ok(contents) = fetch_text(&url) else { + continue; + }; + if let Ok(mut artifact_failures) = self.extract_benchmark_failures(&contents) { + failures.append(&mut artifact_failures); + } + } + + if failures.is_empty() { + Err(anyhow!("No benchmark failures found in session artifacts")) + } else { + Ok(failures) + } + } + /// Extract benchmark JSON from iOS logs using START/END markers. /// iOS uses NSLog which may split the JSON across multiple log lines. fn extract_ios_bench_json(logs: &str) -> Option { @@ -825,8 +896,13 @@ impl BrowserStackClient { "Waiting for build {} to complete (timeout: {}s, poll: {}s)...", build_id, timeout, poll_interval ); - let build_status = - self.poll_build_completion(build_id, platform, timeout, poll_interval)?; + let build_status = self.poll_build_completion_with_terminal_failures( + build_id, + platform, + timeout, + poll_interval, + true, + )?; println!("Build completed with status: {}", build_status.status); println!( @@ -836,6 +912,7 @@ impl BrowserStackClient { let mut benchmark_results = std::collections::HashMap::new(); let mut performance_metrics = std::collections::HashMap::new(); + let mut benchmark_failures = std::collections::HashMap::new(); for device in &build_status.devices { println!( @@ -845,6 +922,7 @@ impl BrowserStackClient { let mut device_benchmark_results: Option> = None; let mut device_performance_metrics = PerformanceMetrics::default(); + let mut device_failures: Option> = None; match self.get_device_logs(build_id, &device.session_id, platform) { Ok(logs) => { @@ -859,6 +937,16 @@ impl BrowserStackClient { } } + match self.extract_benchmark_failures(&logs) { + Ok(failures) => { + println!(" Found {} benchmark failure marker(s)", failures.len()); + device_failures = Some(failures); + } + Err(e) => { + println!(" No benchmark failures in live logs: {}", e); + } + } + // Extract performance metrics match self.extract_performance_metrics(&logs) { Ok(perf_metrics) if perf_metrics.sample_count > 0 => { @@ -882,33 +970,49 @@ impl BrowserStackClient { } if device_benchmark_results.is_none() { - match self - .get_session_json(build_id, &device.session_id, platform) - .and_then(|session_json| { - self.extract_results_from_session_artifacts(&session_json, |url| { + match self.get_session_json(build_id, &device.session_id, platform) { + Ok(session_json) => { + match self.extract_results_from_session_artifacts(&session_json, |url| { self.download_text_url(url) - }) - }) { - Ok((results, perf_metrics)) => { - println!( - " Found {} benchmark result(s) from session artifacts", - results.len() - ); - if device_performance_metrics.sample_count == 0 - && perf_metrics.sample_count > 0 + }) { + Ok((results, perf_metrics)) => { + println!( + " Found {} benchmark result(s) from session artifacts", + results.len() + ); + if device_performance_metrics.sample_count == 0 + && perf_metrics.sample_count > 0 + { + println!( + " Found {} performance metric snapshot(s) from session artifacts", + perf_metrics.sample_count + ); + device_performance_metrics = perf_metrics; + } + device_benchmark_results = Some(results); + } + Err(e) => { + println!( + " Warning: Failed to fetch results from session artifacts: {e}" + ); + } + } + + if device_failures.is_none() + && let Ok(failures) = self + .extract_failures_from_session_artifacts(&session_json, |url| { + self.download_text_url(url) + }) { println!( - " Found {} performance metric snapshot(s) from session artifacts", - perf_metrics.sample_count + " Found {} benchmark failure marker(s) from session artifacts", + failures.len() ); - device_performance_metrics = perf_metrics; + device_failures = Some(failures); } - device_benchmark_results = Some(results); } Err(e) => { - println!( - " Warning: Failed to fetch results from session artifacts: {e}" - ); + println!(" Warning: Failed to fetch session artifacts metadata: {e}"); } } } @@ -927,12 +1031,34 @@ impl BrowserStackClient { if let Some(results) = device_benchmark_results { benchmark_results.insert(device.device.clone(), results); } + if let Some(failures) = device_failures { + benchmark_failures.insert(device.device.clone(), failures); + } if device_performance_metrics.sample_count > 0 { performance_metrics.insert(device.device.clone(), device_performance_metrics); } } if benchmark_results.is_empty() { + if let Some((device, failures)) = benchmark_failures.iter().next() + && let Some(failure) = failures.first() + { + let function = failure + .get("function_name") + .and_then(|value| value.as_str()) + .unwrap_or("unknown"); + let kind = failure + .get("kind") + .and_then(|value| value.as_str()) + .unwrap_or("unknown"); + let message = failure + .get("message") + .and_then(|value| value.as_str()) + .unwrap_or("no message"); + return Err(anyhow!( + "Android benchmark failure on {device} for {function}: {kind}: {message}" + )); + } Err(anyhow!("No benchmark results found from any device")) } else { Ok((benchmark_results, performance_metrics)) @@ -1998,6 +2124,45 @@ mod tests { (format!("http://{addr}"), paths, handle) } + fn spawn_browserstack_path_json_server( + expected_path: &'static str, + payload: Value, + ) -> (String, thread::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind test server"); + let addr = listener.local_addr().expect("read test server address"); + let body = payload.to_string(); + + let handle = thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("accept request"); + let mut buf = [0_u8; 4096]; + let bytes_read = stream.read(&mut buf).expect("read request"); + let request = String::from_utf8_lossy(&buf[..bytes_read]); + let path = request + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .unwrap_or("/"); + let (status, response_body) = if path == expected_path { + ("200 OK", body) + } else { + ( + "404 Not Found", + format!("{{\"error\":\"unexpected path: {path}\"}}"), + ) + }; + let response = format!( + "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + response_body.len(), + response_body + ); + stream + .write_all(response.as_bytes()) + .expect("write response"); + }); + + (format!("http://{addr}"), handle) + } + #[test] fn rejects_missing_artifact() { let client = BrowserStackClient::new( @@ -2358,6 +2523,57 @@ Test completed assert_eq!(status.devices.len(), 0); } + #[test] + fn public_poll_build_completion_errors_on_terminal_failure_status() { + let payload = json!({ + "build_id": "build123", + "status": "failed", + "devices": [{ + "device": "Google Pixel 8-14.0", + "sessionId": "session123", + "status": "failed" + }] + }); + let (base_url, handle) = spawn_browserstack_path_json_server( + "/app-automate/espresso/v2/builds/build123", + payload, + ); + let client = test_client_with_base_url(base_url); + + let error = client + .poll_build_completion("build123", "espresso", 1, 1) + .expect_err("public poll should preserve failure-status errors"); + handle.join().expect("join test server"); + + assert!(error.to_string().contains("failed with status: failed")); + } + + #[test] + fn internal_poll_can_return_terminal_failure_status_for_log_fetching() { + let payload = json!({ + "build_id": "build123", + "status": "failed", + "devices": [{ + "device": "Google Pixel 8-14.0", + "sessionId": "session123", + "status": "failed" + }] + }); + let (base_url, handle) = spawn_browserstack_path_json_server( + "/app-automate/espresso/v2/builds/build123", + payload, + ); + let client = test_client_with_base_url(base_url); + + let status = client + .poll_build_completion_with_terminal_failures("build123", "espresso", 1, 1, true) + .expect("internal poll should allow log-fetching from failed builds"); + handle.join().expect("join test server"); + + assert_eq!(status.status, "failed"); + assert_eq!(status.devices[0].session_id, "session123"); + } + #[test] fn device_session_deserializes_from_json() { let json = r#"{ @@ -2710,6 +2926,30 @@ BENCH_REPORT_JSON_END ); } + #[test] + fn extract_benchmark_results_handles_legacy_uikit_ios_logs() { + let client = BrowserStackClient::new( + BrowserStackAuth { + username: "user".into(), + access_key: "key".into(), + }, + None, + ) + .unwrap(); + + let logs = r#" +2026-05-01 10:00:00.000 BenchRunner[42:7] BENCH_REPORT_JSON_START +2026-05-01 10:00:00.001 BenchRunner[42:7] {"function":"legacy::bench","samples":[{"duration_ns":100},{"duration_ns":200}],"samples_ns":[100,200],"mean_ns":150,"resources":{"platform":"ios","memory_process":"benchmark_app"}} +2026-05-01 10:00:00.002 BenchRunner[42:7] BENCH_REPORT_JSON_END + "#; + + let results = client.extract_benchmark_results(logs).unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0]["function"], "legacy::bench"); + assert_eq!(results[0]["mean_ns"], 150); + assert_eq!(results[0]["samples"].as_array().unwrap().len(), 2); + } + #[test] fn extract_benchmark_results_handles_android_bench_json_marker() { let client = BrowserStackClient::new( @@ -2735,6 +2975,97 @@ BENCH_REPORT_JSON_END .any(|r| r.get("function").and_then(|f| f.as_str()) == Some("sample_fns::checksum"))); } + #[test] + fn extract_benchmark_failures_handles_failure_only_logs() { + let client = BrowserStackClient::new( + BrowserStackAuth { + username: "user".into(), + access_key: "key".into(), + }, + None, + ) + .unwrap(); + + let logs = r#" +2026-01-20 12:34:57 E/BenchRunner: BENCH_FAILURE_JSON {"schema_version":1,"platform":"android","function_name":"sample_fns::sleep","kind":"timeout","message":"Timed out","elapsed_ms":30000,"android_exit_info":null} + "#; + + let failures = client.extract_benchmark_failures(logs).unwrap(); + assert_eq!(failures.len(), 1); + assert_eq!( + failures[0].get("kind").and_then(|kind| kind.as_str()), + Some("timeout") + ); + assert!(client.extract_benchmark_results(logs).is_err()); + } + + #[test] + fn extract_benchmark_failures_ignores_heartbeat_and_reads_failure() { + let client = BrowserStackClient::new( + BrowserStackAuth { + username: "user".into(), + access_key: "key".into(), + }, + None, + ) + .unwrap(); + + let logs = r#" +I/BenchRunner: BENCH_HEARTBEAT_JSON {"schema_version":1,"platform":"android","function_name":"sample_fns::sleep","elapsed_ms":10000} +E/BenchRunner: BENCH_FAILURE_JSON {"schema_version":1,"platform":"android","function_name":"sample_fns::sleep","kind":"worker_exit","message":"worker exited","elapsed_ms":12000,"android_exit_info":{"reason":"low_memory","raw_reason":3}} + "#; + + let failures = client.extract_benchmark_failures(logs).unwrap(); + assert_eq!(failures.len(), 1); + assert_eq!( + failures[0] + .get("android_exit_info") + .and_then(|info| info.get("reason")) + .and_then(|reason| reason.as_str()), + Some("low_memory") + ); + } + + #[test] + fn extract_failures_from_session_artifacts_reads_failure_marker() { + let client = BrowserStackClient::new( + BrowserStackAuth { + username: "user".into(), + access_key: "key".into(), + }, + None, + ) + .unwrap(); + let session_json = serde_json::json!({ + "automation_session": { + "device_log_url": "https://example.invalid/device.log" + } + }); + let logs = r#" +I/BenchRunner: BENCH_HEARTBEAT_JSON {"schema_version":1,"platform":"android","function_name":"sample_fns::sleep","elapsed_ms":10000} +E/BenchRunner: BENCH_FAILURE_JSON {"schema_version":1,"platform":"android","device":"Vivo Y21-11.0","function_name":"sample_fns::sleep","kind":"timeout","message":"Timed out","elapsed_ms":30000,"android_exit_info":null} + "#; + + let failures = client + .extract_failures_from_session_artifacts(&session_json, |url| { + assert_eq!(url, "https://example.invalid/device.log"); + Ok(logs.to_string()) + }) + .unwrap(); + + assert_eq!(failures.len(), 1); + assert_eq!( + failures[0] + .get("function_name") + .and_then(|value| value.as_str()), + Some("sample_fns::sleep") + ); + assert_eq!( + failures[0].get("kind").and_then(|value| value.as_str()), + Some("timeout") + ); + } + #[test] fn extract_ios_bench_json_finds_last_occurrence() { // Test that we find the last occurrence of markers (in case of multiple runs) diff --git a/crates/mobench/src/cli.rs b/crates/mobench/src/cli.rs index 129f4fb..8c419df 100644 --- a/crates/mobench/src/cli.rs +++ b/crates/mobench/src/cli.rs @@ -111,6 +111,27 @@ pub(crate) enum Command { help = "Deprecated compatibility flag for generated XCUITest harness timeout" )] ios_completion_timeout_secs: Option, + #[arg( + long, + help = "iOS deployment target for generated app and XCUITest targets" + )] + ios_deployment_target: Option, + #[arg( + long, + value_enum, + help = "iOS runner template (swiftui or uikit-legacy)" + )] + ios_runner: Option, + #[arg( + long, + help = "Android benchmark watchdog timeout in seconds for the generated harness" + )] + android_benchmark_timeout_secs: Option, + #[arg( + long, + help = "Android benchmark heartbeat interval in seconds for the generated harness" + )] + android_heartbeat_interval_secs: Option, #[arg(long, help = "Fetch BrowserStack artifacts after the run completes")] fetch: bool, #[arg(long, default_value = "target/browserstack")] @@ -217,6 +238,17 @@ pub(crate) enum Command { help = "Deprecated compatibility flag for generated XCUITest harness timeout" )] ios_completion_timeout_secs: Option, + #[arg( + long, + help = "iOS deployment target for generated app and XCUITest targets" + )] + ios_deployment_target: Option, + #[arg( + long, + value_enum, + help = "iOS runner template (swiftui or uikit-legacy)" + )] + ios_runner: Option, #[arg( long, help = "Project root containing mobench.toml or the Cargo workspace" @@ -612,6 +644,27 @@ pub(crate) struct CiRunArgs { help = "Deprecated compatibility flag for generated XCUITest harness timeout" )] pub(crate) ios_completion_timeout_secs: Option, + #[arg( + long, + help = "iOS deployment target for generated app and XCUITest targets" + )] + pub(crate) ios_deployment_target: Option, + #[arg( + long, + value_enum, + help = "iOS runner template (swiftui or uikit-legacy)" + )] + pub(crate) ios_runner: Option, + #[arg( + long, + help = "Android benchmark watchdog timeout in seconds for the generated harness" + )] + pub(crate) android_benchmark_timeout_secs: Option, + #[arg( + long, + help = "Android benchmark heartbeat interval in seconds for the generated harness" + )] + pub(crate) android_heartbeat_interval_secs: Option, #[arg(long, help = "Fetch BrowserStack artifacts after the run completes")] pub(crate) fetch: bool, #[arg(long, default_value = "target/browserstack")] @@ -821,3 +874,20 @@ impl From for mobench_sdk::builders::SigningMethod { } } } + +#[derive(Copy, Clone, Debug, Eq, PartialEq, ValueEnum, Serialize, Deserialize)] +#[clap(rename_all = "kebab-case")] +#[serde(rename_all = "kebab-case")] +pub enum IosRunnerArg { + Swiftui, + UikitLegacy, +} + +impl From for mobench_sdk::codegen::IosRunner { + fn from(arg: IosRunnerArg) -> Self { + match arg { + IosRunnerArg::Swiftui => mobench_sdk::codegen::IosRunner::Swiftui, + IosRunnerArg::UikitLegacy => mobench_sdk::codegen::IosRunner::UikitLegacy, + } + } +} diff --git a/crates/mobench/src/config.rs b/crates/mobench/src/config.rs index 910deda..7cb3bc4 100644 --- a/crates/mobench/src/config.rs +++ b/crates/mobench/src/config.rs @@ -24,6 +24,7 @@ //! [ios] //! bundle_id = "com.example.bench" //! deployment_target = "15.0" +//! runner = "swiftui" //! //! [benchmarks] //! default_function = "my_crate::my_benchmark" @@ -380,6 +381,10 @@ bundle_id = "{package}" # iOS deployment target version (default: 15.0) deployment_target = "15.0" +# iOS app runner: swiftui (iOS 15+) or uikit-legacy (legacy targets). +# If omitted, mobench chooses uikit-legacy for deployment targets below 15.0. +# runner = "swiftui" + # Development team ID for code signing (optional, uses ad-hoc signing if not set) # team_id = "YOUR_TEAM_ID" @@ -396,6 +401,12 @@ default_warmup = 10 [browserstack] # Timeout in seconds for the generated iOS XCUITest harness to wait for completion # ios_completion_timeout_secs = 1200 + +# Timeout in seconds for the generated Android benchmark watchdog +# android_benchmark_timeout_secs = 1800 + +# Heartbeat interval in seconds for Android benchmark progress logging +# android_heartbeat_interval_secs = 10 "#, crate_name = crate_name, library_name = library_name, @@ -647,9 +658,12 @@ crate = "discovered-bench" assert!(toml.contains("min_sdk = 24")); assert!(toml.contains("target_sdk = 34")); assert!(toml.contains("deployment_target = \"15.0\"")); + assert!(toml.contains("runner = \"swiftui\"")); assert!(toml.contains("default_iterations = 100")); assert!(toml.contains("default_warmup = 10")); assert!(toml.contains("[browserstack]")); assert!(toml.contains("ios_completion_timeout_secs")); + assert!(toml.contains("android_benchmark_timeout_secs")); + assert!(toml.contains("android_heartbeat_interval_secs")); } } diff --git a/crates/mobench/src/lib.rs b/crates/mobench/src/lib.rs index b13a973..e7a52a9 100644 --- a/crates/mobench/src/lib.rs +++ b/crates/mobench/src/lib.rs @@ -145,7 +145,8 @@ pub use cli::MobileTarget; pub(crate) use cli::{ CheckOutputFormat, CiCheckRunArgs, CiCommand, CiRunArgs, CiSummarizeArgs, Cli, Command, ConfigCommand, ContractErrorCategory, DevicePlatform, DevicesCommand, FixtureCommand, - IosSigningMethodArg, ProfileCommand, ReportCommand, SdkTarget, SummarizeFormat, SummaryFormat, + IosRunnerArg, IosSigningMethodArg, ProfileCommand, ReportCommand, SdkTarget, SummarizeFormat, + SummaryFormat, }; #[cfg(test)] pub(crate) use doctor::{ @@ -174,6 +175,10 @@ struct BrowserStackConfig { project: Option, #[serde(skip_serializing_if = "Option::is_none", default)] ios_completion_timeout_secs: Option, + #[serde(skip_serializing_if = "Option::is_none", default)] + android_benchmark_timeout_secs: Option, + #[serde(skip_serializing_if = "Option::is_none", default)] + android_heartbeat_interval_secs: Option, } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -200,6 +205,7 @@ struct BenchConfig { struct DeviceEntry { name: String, os: String, + #[serde(default)] os_version: String, tags: Option>, } @@ -218,6 +224,14 @@ pub(crate) struct RunSpec { pub(crate) devices: Vec, #[serde(skip_serializing_if = "Option::is_none", default)] pub(crate) ios_completion_timeout_secs: Option, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub(crate) ios_deployment_target: Option, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub(crate) ios_runner: Option, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub(crate) android_benchmark_timeout_secs: Option, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub(crate) android_heartbeat_interval_secs: Option, #[serde(skip_serializing, skip_deserializing, default)] pub(crate) browserstack: Option, #[serde(skip_serializing_if = "Option::is_none", default)] @@ -250,6 +264,8 @@ struct RunSummary { #[serde(skip_serializing_if = "Option::is_none")] benchmark_results: Option>>, #[serde(skip_serializing_if = "Option::is_none")] + benchmark_failures: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] performance_metrics: Option>, } @@ -261,6 +277,10 @@ pub(crate) struct ResolvedProjectLayout { pub(crate) library_name: String, pub(crate) android_abis: Option>, pub(crate) ios_completion_timeout_secs: Option, + pub(crate) ios_deployment_target: String, + pub(crate) ios_runner: Option, + pub(crate) android_benchmark_timeout_secs: Option, + pub(crate) android_heartbeat_interval_secs: Option, pub(crate) config_path: Option, pub(crate) output_dir: PathBuf, pub(crate) default_function: Option, @@ -315,6 +335,18 @@ struct BenchmarkStats { max_ns: Option, #[serde(skip_serializing_if = "Option::is_none")] resource_usage: Option, + #[serde(skip_serializing_if = "Option::is_none")] + failure: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +struct BenchmarkFailureStats { + kind: String, + message: String, + #[serde(skip_serializing_if = "Option::is_none")] + elapsed_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + exit_reason: Option, } #[derive(Debug, Serialize, Deserialize, Clone, Default)] @@ -405,6 +437,10 @@ pub fn run() -> Result<()> { ios_app, ios_test_suite, ios_completion_timeout_secs, + ios_deployment_target, + ios_runner, + android_benchmark_timeout_secs, + android_heartbeat_interval_secs, fetch, fetch_output_dir, fetch_poll_interval_secs, @@ -431,6 +467,10 @@ pub fn run() -> Result<()> { ios_app, ios_test_suite, ios_completion_timeout_secs, + ios_deployment_target, + ios_runner, + android_benchmark_timeout_secs, + android_heartbeat_interval_secs, local_only, release, cli.dry_run, @@ -498,6 +538,17 @@ pub fn run() -> Result<()> { spec.devices.len() ); } + if spec.target == MobileTarget::Ios + && let Some(deployment_target) = spec.ios_deployment_target.as_deref() + { + let parsed_deployment_target = + mobench_sdk::codegen::IosDeploymentTarget::parse(deployment_target) + .map_err(|err| anyhow!("config_error: {err}"))?; + validate_ios_device_specs_support_deployment_target( + &validation.valid, + &parsed_deployment_target, + )?; + } println!( " All {} device(s) validated successfully.", validation.valid.len() @@ -648,6 +699,8 @@ pub fn run() -> Result<()> { release, cli.dry_run, spec.ios_completion_timeout_secs, + spec.ios_deployment_target.as_deref(), + spec.ios_runner.as_deref(), )?; if !progress { println!("\u{2713} Built iOS xcframework at {:?}", xcframework); @@ -674,6 +727,8 @@ pub fn run() -> Result<()> { &layout, release, spec.ios_completion_timeout_secs, + spec.ios_deployment_target.as_deref(), + spec.ios_runner.as_deref(), )?; println!(" ✓ IPA: {}", packaged.app.display()); println!(" ✓ XCUITest: {}", packaged.test_suite.display()); @@ -707,6 +762,7 @@ pub fn run() -> Result<()> { remote_run, summary: summary_placeholder, benchmark_results: None, + benchmark_failures: None, performance_metrics: None, }; @@ -716,6 +772,7 @@ pub fn run() -> Result<()> { return Ok(()); } + let mut pending_browserstack_error: Option = None; if fetch && let Some(remote) = &run_summary.remote_run { let build_id = match remote { RemoteRun::Android { build_id, .. } => build_id, @@ -744,6 +801,7 @@ pub fn run() -> Result<()> { println!("Waiting for build {} to complete...", build_id); println!("Dashboard: {}", dashboard_url); + let mut browserstack_artifacts_fetched = false; match client.wait_and_fetch_all_results_with_poll( build_id, platform, @@ -802,31 +860,51 @@ pub fn run() -> Result<()> { run_summary.performance_metrics = Some(perf_metrics.into_iter().collect()); } Err(e) => { - bail!( + let output_root = fetch_output_dir.join(build_id); + if let Err(fetch_err) = fetch_browserstack_artifacts( + &client, + run_summary.spec.target, + build_id, + &output_root, + false, + fetch_poll_interval_secs, + fetch_timeout_secs, + ) { + eprintln!( + "Warning: failed to fetch detailed BrowserStack artifacts after benchmark failure: {fetch_err}" + ); + } else if let Ok(failures) = load_browserstack_failure_reports(&output_root) + && !failures.is_empty() + { + run_summary.benchmark_failures = Some(failures); + } + browserstack_artifacts_fetched = true; + pending_browserstack_error = Some(format!( "failed to fetch BrowserStack benchmark results: {}. Build may still be accessible at: {}", - e, - dashboard_url - ); + e, dashboard_url + )); } } // Also save detailed artifacts to separate directory let output_root = fetch_output_dir.join(build_id); - fetch_browserstack_artifacts( - &client, - run_summary.spec.target, - build_id, - &output_root, - false, // Don't wait again, we already did - fetch_poll_interval_secs, - fetch_timeout_secs, - ) - .with_context(|| { - format!( - "failed to fetch detailed BrowserStack artifacts for build {}", - build_id + if !browserstack_artifacts_fetched { + fetch_browserstack_artifacts( + &client, + run_summary.spec.target, + build_id, + &output_root, + false, // Don't wait again, we already did + fetch_poll_interval_secs, + fetch_timeout_secs, ) - })?; + .with_context(|| { + format!( + "failed to fetch detailed BrowserStack artifacts for build {}", + build_id + ) + })?; + } } else if fetch { println!("No BrowserStack run to fetch (devices not provided?)"); } @@ -908,6 +986,17 @@ pub fn run() -> Result<()> { write_junit_report(junit_path, &run_summary.summary, ®ression_findings)?; } + if let Some(error) = pending_browserstack_error { + println!(); + println!("Results saved to:"); + println!(" * {} (machine-readable)", summary_paths.json.display()); + println!(" * {} (human-readable)", summary_paths.markdown.display()); + if summary_csv { + println!(" * {} (spreadsheet)", summary_paths.csv.display()); + } + bail!("{error}"); + } + // Print clear completion summary println!(); println!("\u{2713} Benchmark complete!"); @@ -1034,6 +1123,8 @@ pub fn run() -> Result<()> { target, release, ios_completion_timeout_secs, + ios_deployment_target, + ios_runner, project_root, output_dir, crate_path, @@ -1043,6 +1134,8 @@ pub fn run() -> Result<()> { target, release, ios_completion_timeout_secs, + ios_deployment_target, + ios_runner, project_root, output_dir, crate_path, @@ -1270,6 +1363,29 @@ fn load_layout_config( config::MobenchConfig::discover_from(&discovery_base) } +fn load_toml_config_value(path: &Path) -> Result { + let contents = + fs::read_to_string(path).with_context(|| format!("reading config {}", path.display()))?; + toml::from_str(&contents).with_context(|| format!("parsing config {}", path.display())) +} + +fn toml_path<'a>(value: &'a toml::Value, path: &[&str]) -> Option<&'a toml::Value> { + path.iter() + .try_fold(value, |current, key| current.get(*key)) +} + +fn toml_string(value: &toml::Value, path: &[&str]) -> Option { + toml_path(value, path) + .and_then(|value| value.as_str()) + .map(ToOwned::to_owned) +} + +fn toml_u64(value: &toml::Value, path: &[&str]) -> Option { + toml_path(value, path) + .and_then(|value| value.as_integer()) + .and_then(|value| u64::try_from(value).ok()) +} + fn resolve_project_root_for_layout( start_dir: &Path, explicit_project_root: Option, @@ -1390,6 +1506,10 @@ pub(crate) fn resolve_project_layout( Some((config, path)) => (Some(config), Some(path)), None => (None, None), }; + let raw_config = config_path + .as_deref() + .map(load_toml_config_value) + .transpose()?; let project_root = resolve_project_root_for_layout( &start_dir, @@ -1429,6 +1549,19 @@ pub(crate) fn resolve_project_layout( let ios_completion_timeout_secs = config .as_ref() .and_then(|cfg| cfg.browserstack.ios_completion_timeout_secs); + let ios_deployment_target = config + .as_ref() + .map(|cfg| cfg.ios.deployment_target.clone()) + .unwrap_or_else(|| mobench_sdk::codegen::DEFAULT_IOS_DEPLOYMENT_TARGET.to_string()); + let ios_runner = raw_config + .as_ref() + .and_then(|cfg| toml_string(cfg, &["ios", "runner"])); + let android_benchmark_timeout_secs = raw_config + .as_ref() + .and_then(|cfg| toml_u64(cfg, &["browserstack", "android_benchmark_timeout_secs"])); + let android_heartbeat_interval_secs = raw_config + .as_ref() + .and_then(|cfg| toml_u64(cfg, &["browserstack", "android_heartbeat_interval_secs"])); let output_dir = config .as_ref() .and_then(|cfg| cfg.project.output_dir.clone()) @@ -1451,6 +1584,10 @@ pub(crate) fn resolve_project_layout( library_name, android_abis, ios_completion_timeout_secs, + ios_deployment_target, + ios_runner, + android_benchmark_timeout_secs, + android_heartbeat_interval_secs, config_path, output_dir, default_function, @@ -1493,6 +1630,58 @@ fn configured_ios_completion_timeout_secs( ios_completion_timeout_secs.or(layout.ios_completion_timeout_secs) } +fn configured_ios_deployment_target( + layout: &ResolvedProjectLayout, + ios_deployment_target: Option<&str>, +) -> Result { + let raw = ios_deployment_target.unwrap_or(&layout.ios_deployment_target); + mobench_sdk::codegen::IosDeploymentTarget::parse(raw) + .map_err(|err| anyhow!("config_error: {err}")) +} + +fn configured_ios_runner( + layout: &ResolvedProjectLayout, + deployment_target: &mobench_sdk::codegen::IosDeploymentTarget, + ios_runner: Option<&str>, +) -> Result { + let requested = if let Some(raw_runner) = ios_runner { + Some( + mobench_sdk::codegen::IosRunner::parse(raw_runner) + .map_err(|err| anyhow!("config_error: {err}"))?, + ) + } else { + layout + .ios_runner + .as_deref() + .map(mobench_sdk::codegen::IosRunner::parse) + .transpose() + .map_err(|err| anyhow!("config_error: {err}"))? + }; + mobench_sdk::codegen::resolve_ios_runner(deployment_target, requested) + .map_err(|err| anyhow!("config_error: {err}")) +} + +fn ios_runner_arg_name(runner: IosRunnerArg) -> &'static str { + match runner { + IosRunnerArg::Swiftui => "swiftui", + IosRunnerArg::UikitLegacy => "uikit-legacy", + } +} + +fn configured_android_benchmark_timeout_secs( + layout: &ResolvedProjectLayout, + android_benchmark_timeout_secs: Option, +) -> Option { + android_benchmark_timeout_secs.or(layout.android_benchmark_timeout_secs) +} + +fn configured_android_heartbeat_interval_secs( + layout: &ResolvedProjectLayout, + android_heartbeat_interval_secs: Option, +) -> Option { + android_heartbeat_interval_secs.or(layout.android_heartbeat_interval_secs) +} + fn write_config_template(path: &Path, target: MobileTarget, overwrite: bool) -> Result<()> { ensure_can_write(path, overwrite)?; @@ -1517,6 +1706,8 @@ fn write_config_template(path: &Path, target: MobileTarget, overwrite: bool) -> app_automate_access_key: "${BROWSERSTACK_ACCESS_KEY}".into(), project: Some("mobile-bench-rs".into()), ios_completion_timeout_secs: None, + android_benchmark_timeout_secs: None, + android_heartbeat_interval_secs: None, }, ios_xcuitest, }; @@ -1705,6 +1896,10 @@ pub mod bench_support { /// /// Returns [`RunResult`] containing file paths and process exit semantics. pub fn run_request(request: &RunRequest) -> Result { + run_request_with_extra_args(request, &[]) +} + +fn run_request_with_extra_args(request: &RunRequest, extra_args: &[String]) -> Result { fs::create_dir_all(&request.output_dir) .with_context(|| format!("creating output dir {}", request.output_dir.display()))?; @@ -1775,6 +1970,7 @@ pub fn run_request(request: &RunRequest) -> Result { if let Some(path) = &request.crate_path { cmd.arg("--crate-path").arg(path); } + cmd.args(extra_args); if request.fetch { cmd.arg("--fetch"); } @@ -2396,34 +2592,55 @@ fn cmd_ci_run_single( } }); - let result = run_request(&RunRequest { - target, - function: args.function.clone().unwrap_or_default(), - crate_path: args.crate_path.clone(), - iterations: args.iterations, - warmup: args.warmup, - device_selection: DeviceSelection { - devices: args.devices.clone(), - device_matrix: args.device_matrix.clone(), - device_tags: args.device_tags.clone(), + let mut extra_args = Vec::new(); + if let Some(deployment_target) = &args.ios_deployment_target { + extra_args.push("--ios-deployment-target".to_string()); + extra_args.push(deployment_target.clone()); + } + if let Some(runner) = args.ios_runner { + extra_args.push("--ios-runner".to_string()); + extra_args.push(ios_runner_arg_name(runner).to_string()); + } + if let Some(timeout_secs) = args.android_benchmark_timeout_secs { + extra_args.push("--android-benchmark-timeout-secs".to_string()); + extra_args.push(timeout_secs.to_string()); + } + if let Some(interval_secs) = args.android_heartbeat_interval_secs { + extra_args.push("--android-heartbeat-interval-secs".to_string()); + extra_args.push(interval_secs.to_string()); + } + + let result = run_request_with_extra_args( + &RunRequest { + target, + function: args.function.clone().unwrap_or_default(), + crate_path: args.crate_path.clone(), + iterations: args.iterations, + warmup: args.warmup, + device_selection: DeviceSelection { + devices: args.devices.clone(), + device_matrix: args.device_matrix.clone(), + device_tags: args.device_tags.clone(), + }, + config: args.config.clone(), + baseline: baseline_source, + regression_threshold_pct: args.regression_threshold_pct, + junit: args.junit.clone(), + local_only: args.local_only, + release: args.release, + ios_app: args.ios_app.clone(), + ios_test_suite: args.ios_test_suite.clone(), + ios_completion_timeout_secs: args.ios_completion_timeout_secs, + fetch: args.fetch, + fetch_output_dir: args.fetch_output_dir.clone(), + fetch_poll_interval_secs: args.fetch_poll_interval_secs, + fetch_timeout_secs: args.fetch_timeout_secs, + progress: args.progress, + output_dir: output_dir.to_path_buf(), + plots: args.plots, }, - config: args.config.clone(), - baseline: baseline_source, - regression_threshold_pct: args.regression_threshold_pct, - junit: args.junit.clone(), - local_only: args.local_only, - release: args.release, - ios_app: args.ios_app.clone(), - ios_test_suite: args.ios_test_suite.clone(), - ios_completion_timeout_secs: args.ios_completion_timeout_secs, - fetch: args.fetch, - fetch_output_dir: args.fetch_output_dir.clone(), - fetch_poll_interval_secs: args.fetch_poll_interval_secs, - fetch_timeout_secs: args.fetch_timeout_secs, - progress: args.progress, - output_dir: output_dir.to_path_buf(), - plots: args.plots, - })?; + &extra_args, + )?; let summary_json = result.report.summary_json; let summary_md = result.report.summary_md; @@ -2564,6 +2781,15 @@ fn fetch_browserstack_artifacts( write_json(session_dir.join("session.json"), &session_json)?; let mut downloaded_texts = BTreeMap::new(); + let platform = match target { + MobileTarget::Android => "espresso", + MobileTarget::Ios => "xcuitest", + }; + if let Ok(device_logs) = client.get_device_logs(build_id, &session_id, platform) { + let device_log_path = session_dir.join("device.log"); + write_file(&device_log_path, device_logs.as_bytes())?; + downloaded_texts.insert(format!("live-log:{session_id}"), device_logs); + } for (key, url) in extract_url_fields(&session_json) { let file_name = filename_for_url(&key, &url); let dest = session_dir.join(file_name); @@ -2591,12 +2817,119 @@ fn fetch_browserstack_artifacts( }; write_json(session_dir.join("bench-report.json"), &report)?; } + + let mut live_failures = Vec::new(); + for contents in downloaded_texts.values() { + if let Ok(mut failures) = client.extract_benchmark_failures(contents) { + live_failures.append(&mut failures); + } + } + if !live_failures.is_empty() { + let report = if live_failures.len() == 1 { + live_failures.into_iter().next().unwrap_or(Value::Null) + } else { + Value::Array(live_failures) + }; + write_json(session_dir.join("failure.json"), &report)?; + let markdown = render_failure_markdown(&report); + write_file(&session_dir.join("failure.md"), markdown.as_bytes())?; + } else if let Ok(failures) = + client.extract_failures_from_session_artifacts(&session_json, |url| { + downloaded_texts + .get(url) + .cloned() + .ok_or_else(|| anyhow!("artifact {url} was not downloaded as text")) + }) + { + let report = if failures.len() == 1 { + failures.into_iter().next().unwrap_or(Value::Null) + } else { + Value::Array(failures) + }; + write_json(session_dir.join("failure.json"), &report)?; + let markdown = render_failure_markdown(&report); + write_file(&session_dir.join("failure.md"), markdown.as_bytes())?; + } } println!("Fetched BrowserStack artifacts to {:?}", output_root); Ok(()) } +fn load_browserstack_failure_reports(output_root: &Path) -> Result>> { + let mut failures_by_device: BTreeMap> = BTreeMap::new(); + for entry in fs::read_dir(output_root).with_context(|| { + format!( + "reading BrowserStack artifact dir {}", + output_root.display() + ) + })? { + let entry = entry?; + if !entry.file_type()?.is_dir() { + continue; + } + let failure_path = entry.path().join("failure.json"); + if !failure_path.exists() { + continue; + } + let contents = fs::read_to_string(&failure_path) + .with_context(|| format!("reading {}", failure_path.display()))?; + let value: Value = serde_json::from_str(&contents) + .with_context(|| format!("parsing {}", failure_path.display()))?; + let reports = match value { + Value::Array(values) => values, + value => vec![value], + }; + for report in reports { + let device = report + .get("device") + .and_then(|value| value.as_str()) + .map(ToOwned::to_owned) + .unwrap_or_else(|| entry.file_name().to_string_lossy().to_string()); + failures_by_device.entry(device).or_default().push(report); + } + } + + Ok(failures_by_device) +} + +fn render_failure_markdown(value: &Value) -> String { + let first = value + .as_array() + .and_then(|values| values.first()) + .unwrap_or(value); + let function = first + .get("function_name") + .and_then(|value| value.as_str()) + .unwrap_or("unknown"); + let kind = first + .get("kind") + .and_then(|value| value.as_str()) + .unwrap_or("unknown"); + let message = first + .get("message") + .and_then(|value| value.as_str()) + .unwrap_or("no message"); + let device = first + .get("device") + .and_then(|value| value.as_str()) + .unwrap_or("unknown"); + let elapsed = first + .get("elapsed_ms") + .and_then(|value| value.as_u64()) + .map(|value| format!("{value} ms")) + .unwrap_or_else(|| "unknown".to_string()); + let exit_reason = first + .get("android_exit_info") + .and_then(|value| value.get("reason")) + .and_then(|value| value.as_str()) + .unwrap_or("unavailable"); + + format!( + "# Android Benchmark Failure\n\n- Device: {device}\n- Function: {function}\n- Kind: {kind}\n- Message: {message}\n- Elapsed: {elapsed}\n- Exit reason: {exit_reason}\n" + ) +} + fn browserstack_base_path(target: MobileTarget) -> &'static str { match target { MobileTarget::Android => "app-automate/espresso/v2", @@ -2750,15 +3083,36 @@ fn resolve_run_spec( ios_app: Option, ios_test_suite: Option, ios_completion_timeout_secs: Option, + ios_deployment_target: Option, + ios_runner: Option, + android_benchmark_timeout_secs: Option, + android_heartbeat_interval_secs: Option, local_only: bool, _release: bool, dry_run: bool, ) -> Result { if let Some(cfg_path) = config { let cfg = load_config(cfg_path)?; + let resolved_target = target.unwrap_or(cfg.target); let configured_ios_completion_timeout_secs = ios_completion_timeout_secs .or(cfg.browserstack.ios_completion_timeout_secs) .or(layout.ios_completion_timeout_secs); + let (configured_ios_deployment_target, configured_ios_runner) = + if resolved_target == MobileTarget::Ios { + let deployment_target = + configured_ios_deployment_target(layout, ios_deployment_target.as_deref())?; + let runner_name = ios_runner.map(ios_runner_arg_name); + let runner = configured_ios_runner(layout, &deployment_target, runner_name)?; + (Some(deployment_target), Some(runner)) + } else { + (None, None) + }; + let configured_android_benchmark_timeout_secs = android_benchmark_timeout_secs + .or(cfg.browserstack.android_benchmark_timeout_secs) + .or(layout.android_benchmark_timeout_secs); + let configured_android_heartbeat_interval_secs = android_heartbeat_interval_secs + .or(cfg.browserstack.android_heartbeat_interval_secs) + .or(layout.android_heartbeat_interval_secs); if device_matrix.is_some() && !devices.is_empty() { bail!( "--device-matrix cannot be combined with --devices; choose one source for devices" @@ -2776,12 +3130,41 @@ fn resolve_run_spec( cfg.device_tags.clone() }; let device_names = if !devices.is_empty() { + if resolved_target == MobileTarget::Ios { + validate_ios_device_specs_support_deployment_target( + &devices, + configured_ios_deployment_target + .as_ref() + .expect("iOS deployment target should be resolved"), + )?; + } devices } else { let matrix = load_device_matrix(&matrix_path)?; match resolved_tags.as_ref() { - Some(tags) if !tags.is_empty() => filter_devices_by_tags(matrix.devices, tags)?, - _ => matrix.devices.into_iter().map(|d| d.name).collect(), + Some(tags) if !tags.is_empty() => { + let entries = filter_device_entries_by_tags(matrix.devices, tags)?; + if resolved_target == MobileTarget::Ios { + validate_ios_device_entries_support_deployment_target( + &entries, + configured_ios_deployment_target + .as_ref() + .expect("iOS deployment target should be resolved"), + )?; + } + entries.into_iter().map(|d| d.name).collect() + } + _ => { + if resolved_target == MobileTarget::Ios { + validate_ios_device_entries_support_deployment_target( + &matrix.devices, + configured_ios_deployment_target + .as_ref() + .expect("iOS deployment target should be resolved"), + )?; + } + matrix.devices.into_iter().map(|d| d.name).collect() + } } }; let ios_xcuitest = match (ios_app, ios_test_suite) { @@ -2792,12 +3175,17 @@ fn resolve_run_spec( ), }; return Ok(RunSpec { - target: target.unwrap_or(cfg.target), + target: resolved_target, function: function.unwrap_or(cfg.function), iterations: iterations.unwrap_or(cfg.iterations), warmup: warmup.unwrap_or(cfg.warmup), devices: device_names, ios_completion_timeout_secs: configured_ios_completion_timeout_secs, + ios_deployment_target: configured_ios_deployment_target + .map(|target| target.to_string()), + ios_runner: configured_ios_runner.map(|runner| runner.as_str().to_string()), + android_benchmark_timeout_secs: configured_android_benchmark_timeout_secs, + android_heartbeat_interval_secs: configured_android_heartbeat_interval_secs, browserstack: Some(cfg.browserstack), ios_xcuitest, }); @@ -2805,6 +3193,15 @@ fn resolve_run_spec( let target = target.context("target must be provided with --target or set in the config file")?; + let (configured_ios_deployment_target, configured_ios_runner) = if target == MobileTarget::Ios { + let deployment_target = + configured_ios_deployment_target(layout, ios_deployment_target.as_deref())?; + let runner_name = ios_runner.map(ios_runner_arg_name); + let runner = configured_ios_runner(layout, &deployment_target, runner_name)?; + (Some(deployment_target), Some(runner)) + } else { + (None, None) + }; let function = function.unwrap_or_default(); let iterations = iterations.unwrap_or(100); let warmup = warmup.unwrap_or(10); @@ -2823,13 +3220,38 @@ fn resolve_run_spec( } let resolved_devices = if !devices.is_empty() { + if target == MobileTarget::Ios { + validate_ios_device_specs_support_deployment_target( + &devices, + configured_ios_deployment_target + .as_ref() + .expect("iOS deployment target should be resolved"), + )?; + } devices } else if let Some(matrix_path) = device_matrix { let matrix = load_device_matrix(matrix_path)?; if device_tags.is_empty() { + if target == MobileTarget::Ios { + validate_ios_device_entries_support_deployment_target( + &matrix.devices, + configured_ios_deployment_target + .as_ref() + .expect("iOS deployment target should be resolved"), + )?; + } matrix.devices.into_iter().map(|d| d.name).collect() } else { - filter_devices_by_tags(matrix.devices, &device_tags)? + let entries = filter_device_entries_by_tags(matrix.devices, &device_tags)?; + if target == MobileTarget::Ios { + validate_ios_device_entries_support_deployment_target( + &entries, + configured_ios_deployment_target + .as_ref() + .expect("iOS deployment target should be resolved"), + )?; + } + entries.into_iter().map(|d| d.name).collect() } } else { Vec::new() @@ -2866,6 +3288,16 @@ fn resolve_run_spec( layout, ios_completion_timeout_secs, ), + ios_deployment_target: configured_ios_deployment_target.map(|target| target.to_string()), + ios_runner: configured_ios_runner.map(|runner| runner.as_str().to_string()), + android_benchmark_timeout_secs: configured_android_benchmark_timeout_secs( + layout, + android_benchmark_timeout_secs, + ), + android_heartbeat_interval_secs: configured_android_heartbeat_interval_secs( + layout, + android_heartbeat_interval_secs, + ), browserstack: None, ios_xcuitest, }) @@ -2884,13 +3316,23 @@ fn load_device_matrix(path: &Path) -> Result { } fn filter_devices_by_tags(devices: Vec, tags: &[String]) -> Result> { + Ok(filter_device_entries_by_tags(devices, tags)? + .into_iter() + .map(|d| d.name) + .collect()) +} + +fn filter_device_entries_by_tags( + devices: Vec, + tags: &[String], +) -> Result> { let wanted: Vec = tags .iter() .map(|tag| tag.trim().to_lowercase()) .filter(|tag| !tag.is_empty()) .collect(); if wanted.is_empty() { - return Ok(devices.into_iter().map(|d| d.name).collect()); + return Ok(devices); } let mut matched = Vec::new(); @@ -2910,7 +3352,7 @@ fn filter_devices_by_tags(devices: Vec, tags: &[String]) -> Result< wanted.iter().any(|wanted_tag| wanted_tag == &candidate) }); if has_match { - matched.push(device.name); + matched.push(device); } } @@ -2931,6 +3373,76 @@ fn filter_devices_by_tags(devices: Vec, tags: &[String]) -> Result< Ok(matched) } +fn parse_ios_version_from_device_identifier(spec: &str) -> Option<&str> { + let dash_pos = spec.rfind('-')?; + let version = spec[dash_pos + 1..].trim(); + version + .chars() + .next() + .filter(|ch| ch.is_ascii_digit()) + .map(|_| version) +} + +fn ios_device_version_is_supported( + device_version: &str, + deployment_target: &mobench_sdk::codegen::IosDeploymentTarget, +) -> Result { + let device_target = + mobench_sdk::codegen::IosDeploymentTarget::parse(device_version).map_err(|err| { + anyhow!("config_error: invalid iOS device version `{device_version}`: {err}") + })?; + Ok(&device_target >= deployment_target) +} + +fn validate_ios_device_specs_support_deployment_target( + devices: &[String], + deployment_target: &mobench_sdk::codegen::IosDeploymentTarget, +) -> Result<()> { + for device in devices { + let Some(os_version) = parse_ios_version_from_device_identifier(device) else { + continue; + }; + if !ios_device_version_is_supported(os_version, deployment_target)? { + bail!( + "`{}` cannot run app with iOS deployment target `{}`.", + device, + deployment_target + ); + } + } + Ok(()) +} + +fn validate_ios_device_entries_support_deployment_target( + devices: &[DeviceEntry], + deployment_target: &mobench_sdk::codegen::IosDeploymentTarget, +) -> Result<()> { + for device in devices { + if !device.os.eq_ignore_ascii_case("ios") { + continue; + } + let parsed_from_name = parse_ios_version_from_device_identifier(&device.name); + let os_version = if device.os_version.trim().is_empty() { + parsed_from_name + } else { + Some(device.os_version.trim()) + }; + let Some(os_version) = os_version else { + continue; + }; + if !ios_device_version_is_supported(os_version, deployment_target)? { + let (identifier, _) = + browserstack_identifier_and_os_version(&device.name, &device.os_version); + bail!( + "`{}` cannot run app with iOS deployment target `{}`.", + identifier, + deployment_target + ); + } + } + Ok(()) +} + fn with_ios_benchmark_timeout_env( timeout_secs: Option, f: impl FnOnce() -> Result, @@ -2964,13 +3476,19 @@ pub(crate) fn run_ios_build( release: bool, dry_run: bool, ios_completion_timeout_secs: Option, + ios_deployment_target: Option<&str>, + ios_runner: Option<&str>, ) -> Result<(PathBuf, PathBuf)> { let ios_completion_timeout_secs = configured_ios_completion_timeout_secs(layout, ios_completion_timeout_secs); + let ios_deployment_target = configured_ios_deployment_target(layout, ios_deployment_target)?; + let ios_runner = configured_ios_runner(layout, &ios_deployment_target, ios_runner)?; let builder = mobench_sdk::builders::IosBuilder::new(&layout.project_root, layout.crate_name.clone()) .verbose(true) .dry_run(dry_run) + .deployment_target(ios_deployment_target) + .runner(Some(ios_runner)) .crate_dir(&layout.crate_dir) .output_dir(&layout.output_dir); let profile = if release { @@ -2997,12 +3515,18 @@ fn package_ios_xcuitest_artifacts( layout: &ResolvedProjectLayout, release: bool, ios_completion_timeout_secs: Option, + ios_deployment_target: Option<&str>, + ios_runner: Option<&str>, ) -> Result { let ios_completion_timeout_secs = configured_ios_completion_timeout_secs(layout, ios_completion_timeout_secs); + let ios_deployment_target = configured_ios_deployment_target(layout, ios_deployment_target)?; + let ios_runner = configured_ios_runner(layout, &ios_deployment_target, ios_runner)?; let builder = mobench_sdk::builders::IosBuilder::new(&layout.project_root, layout.crate_name.clone()) .verbose(true) + .deployment_target(ios_deployment_target) + .runner(Some(ios_runner)) .crate_dir(&layout.crate_dir) .output_dir(&layout.output_dir); let profile = if release { @@ -3476,6 +4000,8 @@ pub(crate) fn persist_mobile_spec( "function": spec.function, "iterations": spec.iterations, "warmup": spec.warmup, + "android_benchmark_timeout_secs": spec.android_benchmark_timeout_secs, + "android_heartbeat_interval_secs": spec.android_heartbeat_interval_secs, }); let contents = serde_json::to_string_pretty(&payload)?; @@ -3534,10 +4060,23 @@ pub(crate) fn persist_mobile_spec( /// Embeds the benchmark spec into Android assets and iOS bundle resources. fn embed_spec_into_apps(output_dir: &Path, spec: &RunSpec) -> Result<()> { - let embedded_spec = mobench_sdk::builders::EmbeddedBenchSpec { + #[derive(Serialize)] + struct EmbeddedRunSpec { + function: String, + iterations: u32, + warmup: u32, + #[serde(skip_serializing_if = "Option::is_none")] + android_benchmark_timeout_secs: Option, + #[serde(skip_serializing_if = "Option::is_none")] + android_heartbeat_interval_secs: Option, + } + + let embedded_spec = EmbeddedRunSpec { function: spec.function.clone(), iterations: spec.iterations, warmup: spec.warmup, + android_benchmark_timeout_secs: spec.android_benchmark_timeout_secs, + android_heartbeat_interval_secs: spec.android_heartbeat_interval_secs, }; mobench_sdk::builders::embed_bench_spec(output_dir, &embedded_spec) .map_err(|e| anyhow!("Failed to embed bench spec: {}", e)) @@ -3653,6 +4192,7 @@ fn build_summary(run_summary: &RunSummary) -> Result { .map(|s| s.max_ns) .or_else(|| entry.get("max_ns").and_then(|value| value.as_u64())), resource_usage: extract_benchmark_resource_usage(entry, perf_metrics), + failure: None, }); } @@ -3664,6 +4204,50 @@ fn build_summary(run_summary: &RunSummary) -> Result { } } + if let Some(failures) = &run_summary.benchmark_failures { + for (device, entries) in failures { + let device_index = if let Some(index) = device_summaries + .iter() + .position(|summary| summary.device == *device) + { + index + } else { + device_summaries.push(DeviceSummary { + device: device.clone(), + benchmarks: Vec::new(), + }); + device_summaries.len() - 1 + }; + let device_summary = &mut device_summaries[device_index]; + + for entry in entries { + if let Some(failure) = benchmark_failure_stats(entry) { + device_summary.benchmarks.push(BenchmarkStats { + function: entry + .get("function_name") + .or_else(|| entry.get("function")) + .and_then(|value| value.as_str()) + .unwrap_or(&run_summary.spec.function) + .to_string(), + samples: 0, + mean_ns: None, + median_ns: None, + p95_ns: None, + min_ns: None, + max_ns: None, + resource_usage: entry + .get("memory") + .and_then(extract_benchmark_resource_usage_from_memory), + failure: Some(failure), + }); + } + } + device_summary + .benchmarks + .sort_by(|a, b| a.function.cmp(&b.function)); + } + } + if device_summaries.is_empty() && let Some(local_summary) = summarize_local_report(run_summary) { @@ -3911,6 +4495,15 @@ fn print_run_completion_summary( for device_summary in &summary.summary.device_summaries { println!(" Device: {}", device_summary.device); for bench in &device_summary.benchmarks { + if let Some(failure) = &bench.failure { + println!( + " {} - failed: {}, elapsed: {}", + bench.function, + failure.kind, + format_failure_elapsed_ms(Some(failure)) + ); + continue; + } let median = bench .median_ns .map(format_duration_smart) @@ -4592,6 +5185,7 @@ fn summarize_local_report(run_summary: &RunSummary) -> Option { min_ns: Some(stats.min_ns), max_ns: Some(stats.max_ns), resource_usage: extract_benchmark_resource_usage(&run_summary.local_report, None), + failure: None, }], }) } @@ -4798,6 +5392,43 @@ fn extract_benchmark_resource_usage( (!resource_usage.is_empty()).then_some(resource_usage) } +fn extract_benchmark_resource_usage_from_memory(memory: &Value) -> Option { + let resource_usage = BenchmarkResourceUsage { + cpu_total_ms: None, + cpu_median_ms: None, + peak_memory_kb: None, + peak_memory_growth_kb: None, + process_peak_memory_kb: memory.get("process_pss_kb").and_then(json_value_to_u64), + total_pss_kb: memory.get("total_pss_kb").and_then(json_value_to_u64), + private_dirty_kb: memory.get("private_dirty_kb").and_then(json_value_to_u64), + native_heap_kb: memory.get("native_heap_kb").and_then(json_value_to_u64), + java_heap_kb: memory.get("java_heap_kb").and_then(json_value_to_u64), + }; + + (!resource_usage.is_empty()).then_some(resource_usage) +} + +fn benchmark_failure_stats(entry: &Value) -> Option { + let kind = entry.get("kind").and_then(|value| value.as_str())?; + let message = entry + .get("message") + .and_then(|value| value.as_str()) + .unwrap_or("no message") + .to_string(); + let exit_reason = entry + .get("android_exit_info") + .and_then(|info| info.get("reason")) + .and_then(|value| value.as_str()) + .map(ToOwned::to_owned); + + Some(BenchmarkFailureStats { + kind: kind.to_string(), + message, + elapsed_ms: entry.get("elapsed_ms").and_then(json_value_to_u64), + exit_reason, + }) +} + fn render_markdown_summary(summary: &SummaryReport) -> String { let mut output = String::new(); let devices = if summary.devices.is_empty() { @@ -4824,41 +5455,101 @@ fn render_markdown_summary(summary: &SummaryReport) -> String { return output; } - let _ = writeln!( - output, - "| Device | Function | Samples | Warmup | Wall mean / iter | Wall total | CPU median / iter | CPU total | CPU / wall | Peak growth | Process peak |" - ); - let _ = writeln!( - output, - "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |" - ); + let has_failures = summary.device_summaries.iter().any(|device| { + device + .benchmarks + .iter() + .any(|benchmark| benchmark.failure.is_some()) + }); + if has_failures { + let _ = writeln!( + output, + "| Device | Function | Status | Samples | Warmup | Wall mean / iter | Wall total | CPU median / iter | CPU total | CPU / wall | Peak growth | Process peak | Elapsed | Exit reason |" + ); + let _ = writeln!( + output, + "| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |" + ); + } else { + let _ = writeln!( + output, + "| Device | Function | Samples | Warmup | Wall mean / iter | Wall total | CPU median / iter | CPU total | CPU / wall | Peak growth | Process peak |" + ); + let _ = writeln!( + output, + "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |" + ); + } for device in &summary.device_summaries { for bench in &device.benchmarks { - let _ = writeln!( - output, - "| {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} |", - device.device, - bench.function, - bench.samples, - summary.warmup, - format_ms(bench.mean_ns), - format_wall_total(bench.mean_ns, bench.samples), - format_cpu_median_ms(bench.resource_usage.as_ref()), - format_cpu_total_ms(bench.resource_usage.as_ref()), - format_cpu_wall_ratio(bench.mean_ns, bench.samples, bench.resource_usage.as_ref()), - format_peak_memory( - bench - .resource_usage - .as_ref() - .and_then(BenchmarkResourceUsage::peak_memory_growth_or_legacy_kb) - ), - format_peak_memory( + if has_failures { + let _ = writeln!( + output, + "| {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} |", + device.device, + bench.function, + format_benchmark_status(bench), + bench.samples, + summary.warmup, + format_ms(bench.mean_ns), + format_wall_total(bench.mean_ns, bench.samples), + format_cpu_median_ms(bench.resource_usage.as_ref()), + format_cpu_total_ms(bench.resource_usage.as_ref()), + format_cpu_wall_ratio( + bench.mean_ns, + bench.samples, + bench.resource_usage.as_ref() + ), + format_peak_memory( + bench + .resource_usage + .as_ref() + .and_then(BenchmarkResourceUsage::peak_memory_growth_or_legacy_kb) + ), + format_peak_memory( + bench + .resource_usage + .as_ref() + .and_then(|usage| usage.process_peak_memory_kb) + ), + format_failure_elapsed_ms(bench.failure.as_ref()), bench - .resource_usage + .failure .as_ref() - .and_then(|usage| usage.process_peak_memory_kb) - ), - ); + .and_then(|failure| failure.exit_reason.as_deref()) + .unwrap_or("-"), + ); + } else { + let _ = writeln!( + output, + "| {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} |", + device.device, + bench.function, + bench.samples, + summary.warmup, + format_ms(bench.mean_ns), + format_wall_total(bench.mean_ns, bench.samples), + format_cpu_median_ms(bench.resource_usage.as_ref()), + format_cpu_total_ms(bench.resource_usage.as_ref()), + format_cpu_wall_ratio( + bench.mean_ns, + bench.samples, + bench.resource_usage.as_ref() + ), + format_peak_memory( + bench + .resource_usage + .as_ref() + .and_then(BenchmarkResourceUsage::peak_memory_growth_or_legacy_kb) + ), + format_peak_memory( + bench + .resource_usage + .as_ref() + .and_then(|usage| usage.process_peak_memory_kb) + ), + ); + } } } let _ = writeln!(output); @@ -4870,6 +5561,21 @@ fn render_markdown_summary(summary: &SummaryReport) -> String { output } +fn format_benchmark_status(bench: &BenchmarkStats) -> String { + if let Some(failure) = &bench.failure { + format!("failed ({})", failure.kind) + } else { + "ok".to_string() + } +} + +fn format_failure_elapsed_ms(failure: Option<&BenchmarkFailureStats>) -> String { + failure + .and_then(|failure| failure.elapsed_ms) + .map(|elapsed_ms| format!("{:.3}s", elapsed_ms as f64 / 1_000.0)) + .unwrap_or_else(|| "-".to_string()) +} + fn render_csv_summary(summary: &SummaryReport) -> String { let mut output = String::new(); let _ = writeln!( @@ -5202,6 +5908,8 @@ fn cmd_build( target: SdkTarget, release: bool, ios_completion_timeout_secs: Option, + ios_deployment_target: Option, + ios_runner: Option, project_root: Option, output_dir: Option, crate_path: Option, @@ -5218,6 +5926,19 @@ fn cmd_build( let effective_output_dir = output_dir.unwrap_or_else(|| layout.output_dir.clone()); let ios_completion_timeout_secs = configured_ios_completion_timeout_secs(&layout, ios_completion_timeout_secs); + let (ios_deployment_target, ios_runner) = if matches!(target, SdkTarget::Ios | SdkTarget::Both) + { + let deployment_target = + configured_ios_deployment_target(&layout, ios_deployment_target.as_deref())?; + let runner_name = ios_runner.map(ios_runner_arg_name); + let runner = configured_ios_runner(&layout, &deployment_target, runner_name)?; + (deployment_target, runner) + } else { + ( + mobench_sdk::codegen::IosDeploymentTarget::default_target(), + mobench_sdk::codegen::IosRunner::Swiftui, + ) + }; // Progress mode: simplified output if progress { @@ -5289,6 +6010,8 @@ fn cmd_build( ) .verbose(false) .dry_run(dry_run) + .deployment_target(ios_deployment_target.clone()) + .runner(Some(ios_runner)) .output_dir(&effective_output_dir) .crate_dir(&layout.crate_dir); println!("[4/5] Building iOS xcframework..."); @@ -5367,6 +6090,8 @@ fn cmd_build( ) .verbose(verbose) .dry_run(dry_run) + .deployment_target(ios_deployment_target.clone()) + .runner(Some(ios_runner)) .output_dir(&effective_output_dir) .crate_dir(&layout.crate_dir); let result = with_ios_benchmark_timeout_env(ios_completion_timeout_secs, || { @@ -5501,10 +6226,14 @@ fn cmd_package_ipa( config_path: None, })?; let effective_output_dir = output_dir.unwrap_or_else(|| layout.output_dir.clone()); + let ios_deployment_target = configured_ios_deployment_target(&layout, None)?; + let ios_runner = configured_ios_runner(&layout, &ios_deployment_target, None)?; let builder = mobench_sdk::builders::IosBuilder::new(&layout.project_root, layout.crate_name.clone()) .verbose(true) + .deployment_target(ios_deployment_target) + .runner(Some(ios_runner)) .crate_dir(&layout.crate_dir) .output_dir(&effective_output_dir); @@ -5545,10 +6274,14 @@ fn cmd_package_xcuitest( config_path: None, })?; let effective_output_dir = output_dir.unwrap_or_else(|| layout.output_dir.clone()); + let ios_deployment_target = configured_ios_deployment_target(&layout, None)?; + let ios_runner = configured_ios_runner(&layout, &ios_deployment_target, None)?; let builder = mobench_sdk::builders::IosBuilder::new(&layout.project_root, layout.crate_name.clone()) .verbose(true) + .deployment_target(ios_deployment_target) + .runner(Some(ios_runner)) .crate_dir(&layout.crate_dir) .output_dir(&effective_output_dir); @@ -6619,11 +7352,12 @@ fn resolve_devices_from_matrix( if !tag_match { continue; } - let identifier = format!("{}-{}", device.name, device.os_version); + let (identifier, os_version) = + browserstack_identifier_and_os_version(&device.name, &device.os_version); resolved.push(ResolvedMatrixDevice { name: device.name, os: device.os, - os_version: device.os_version, + os_version, identifier, tags: normalized_tags, }); @@ -6655,11 +7389,36 @@ fn resolve_devices_from_matrix( Ok(resolved) } -fn cmd_fixture_init(config_path: &Path, device_matrix_path: &Path, force: bool) -> Result<()> { - write_config_template(config_path, MobileTarget::Android, force)?; - write_device_matrix_template(device_matrix_path, force)?; - println!( - "Initialized fixture files:\n - {}\n - {}", +fn browserstack_identifier_and_os_version(name: &str, os_version: &str) -> (String, String) { + let trimmed_version = os_version.trim(); + if !trimmed_version.is_empty() { + if let Some(name_version) = parse_ios_version_from_device_identifier(name) { + let parsed_name = mobench_sdk::codegen::IosDeploymentTarget::parse(name_version); + let parsed_field = mobench_sdk::codegen::IosDeploymentTarget::parse(trimmed_version); + if let (Ok(parsed_name), Ok(parsed_field)) = (parsed_name, parsed_field) { + if parsed_name == parsed_field { + return (name.to_string(), trimmed_version.to_string()); + } + } + } + return ( + format!("{}-{}", name, trimmed_version), + trimmed_version.to_string(), + ); + } + + if let Some(parsed) = parse_ios_version_from_device_identifier(name) { + return (name.to_string(), parsed.to_string()); + } + + (name.to_string(), String::new()) +} + +fn cmd_fixture_init(config_path: &Path, device_matrix_path: &Path, force: bool) -> Result<()> { + write_config_template(config_path, MobileTarget::Android, force)?; + write_device_matrix_template(device_matrix_path, force)?; + println!( + "Initialized fixture files:\n - {}\n - {}", config_path.display(), device_matrix_path.display() ); @@ -6679,6 +7438,8 @@ fn cmd_fixture_build( release, None, None, + None, + None, output_dir, crate_path, false, @@ -6691,6 +7452,8 @@ fn cmd_fixture_build( release, None, None, + None, + None, output_dir.clone(), crate_path, false, @@ -6712,6 +7475,8 @@ fn cmd_fixture_build( release, None, None, + None, + None, output_dir.clone(), crate_path.clone(), false, @@ -6723,6 +7488,8 @@ fn cmd_fixture_build( release, None, None, + None, + None, output_dir.clone(), crate_path, false, @@ -7091,6 +7858,10 @@ pub fn bench_query_proof_generation() {} None, None, None, + None, + None, + None, + None, false, false, // release false, @@ -7166,6 +7937,10 @@ project = "proj" None, None, None, + None, + None, + None, + None, false, false, false, @@ -7231,6 +8006,10 @@ project = "proj" None, None, None, + None, + None, + None, + None, false, false, false, @@ -7345,12 +8124,78 @@ project = "proj" Some(vec!["arm64-v8a".to_string(), "x86_64".to_string()]) ); assert_eq!(layout.ios_completion_timeout_secs, Some(900)); + assert_eq!(layout.ios_deployment_target, "15.0"); + assert_eq!(layout.ios_runner, None); assert_eq!( layout.default_function.as_deref(), Some("zk_mobile_bench::bench_query_proof_generation") ); } + #[test] + fn ios_runner_selection_uses_legacy_below_ios_15() { + let temp_dir = TempDir::new().expect("temp dir"); + let (project_root, _) = write_custom_layout_project(&temp_dir); + write_file( + &project_root.join("mobench.toml"), + br#"[project] +crate = "zk-mobile-bench" +library_name = "zk_mobile_bench" + +[ios] +deployment_target = "10.0" + +[benchmarks] +default_function = "zk_mobile_bench::bench_query_proof_generation" +"#, + ) + .expect("write mobench config"); + + let layout = resolve_project_layout(ProjectLayoutOptions { + start_dir: Some(project_root.as_path()), + project_root: None, + crate_path: None, + config_path: None, + }) + .expect("resolve project layout"); + let target = configured_ios_deployment_target(&layout, None).unwrap(); + let runner = configured_ios_runner(&layout, &target, None).unwrap(); + + assert_eq!(target.to_string(), "10.0"); + assert_eq!(runner, mobench_sdk::codegen::IosRunner::UikitLegacy); + } + + #[test] + fn ios_runner_rejects_forced_swiftui_below_ios_15() { + let temp_dir = TempDir::new().expect("temp dir"); + let (project_root, _) = write_custom_layout_project(&temp_dir); + write_file( + &project_root.join("mobench.toml"), + br#"[project] +crate = "zk-mobile-bench" +library_name = "zk_mobile_bench" + +[ios] +deployment_target = "10.0" +runner = "swiftui" +"#, + ) + .expect("write mobench config"); + + let layout = resolve_project_layout(ProjectLayoutOptions { + start_dir: Some(project_root.as_path()), + project_root: None, + crate_path: None, + config_path: None, + }) + .expect("resolve project layout"); + let target = configured_ios_deployment_target(&layout, None).unwrap(); + let err = configured_ios_runner(&layout, &target, None) + .expect_err("swiftui should reject iOS 10"); + + assert!(err.to_string().contains("requires deployment target 15.0+")); + } + #[test] fn list_uses_resolved_layout_for_custom_crate() { let temp_dir = TempDir::new().expect("temp dir"); @@ -7405,6 +8250,8 @@ project = "proj" SdkTarget::Ios, false, None, + None, + None, Some(project_root), None, None, @@ -7458,6 +8305,10 @@ project = "proj" None, None, None, + None, + None, + None, + None, false, false, true, @@ -7516,6 +8367,10 @@ project = "proj" warmup: 1, devices: vec![], ios_completion_timeout_secs: None, + ios_deployment_target: None, + ios_runner: None, + android_benchmark_timeout_secs: None, + android_heartbeat_interval_secs: None, browserstack: None, ios_xcuitest: None, }; @@ -7548,6 +8403,10 @@ project = "proj" None, None, None, + None, + None, + None, + None, false, false, // release false, @@ -7739,6 +8598,34 @@ project = "proj" } } + #[test] + fn ci_run_parses_android_watchdog_settings() { + let cli = Cli::parse_from([ + "mobench", + "ci", + "run", + "--target", + "android", + "--function", + "sample_fns::fibonacci", + "--android-benchmark-timeout-secs", + "30", + "--android-heartbeat-interval-secs", + "3", + ]); + + match cli.command { + Command::Ci { + command: CiCommand::Run(args), + } => { + assert_eq!(args.target, CiTarget::Android); + assert_eq!(args.android_benchmark_timeout_secs, Some(30)); + assert_eq!(args.android_heartbeat_interval_secs, Some(3)); + } + _ => panic!("expected ci run command"), + } + } + #[test] fn build_parses_ios_completion_timeout_secs() { let cli = Cli::parse_from([ @@ -7761,6 +8648,32 @@ project = "proj" } } + #[test] + fn build_parses_ios_deployment_target_and_runner() { + let cli = Cli::parse_from([ + "mobench", + "build", + "--target", + "ios", + "--ios-deployment-target", + "10.0", + "--ios-runner", + "uikit-legacy", + ]); + + match cli.command { + Command::Build { + ios_deployment_target, + ios_runner, + .. + } => { + assert_eq!(ios_deployment_target.as_deref(), Some("10.0")); + assert_eq!(ios_runner, Some(IosRunnerArg::UikitLegacy)); + } + _ => panic!("expected build command"), + } + } + #[test] fn resolve_run_spec_reads_ios_completion_timeout_from_config() { let temp_dir = TempDir::new().expect("temp dir"); @@ -7813,6 +8726,10 @@ test_suite = "target/ios/BenchRunnerUITests.zip" None, None, Some(600), + None, + None, + None, + None, false, false, false, @@ -7820,6 +8737,8 @@ test_suite = "target/ios/BenchRunnerUITests.zip" .expect("resolve spec"); assert_eq!(spec.ios_completion_timeout_secs, Some(600)); + assert_eq!(spec.ios_deployment_target.as_deref(), Some("15.0")); + assert_eq!(spec.ios_runner.as_deref(), Some("swiftui")); assert_eq!( spec.browserstack .as_ref() @@ -7828,6 +8747,146 @@ test_suite = "target/ios/BenchRunnerUITests.zip" ); } + #[test] + fn resolve_run_spec_applies_legacy_ios_deployment_override() { + let layout = resolve_project_layout(ProjectLayoutOptions { + start_dir: None, + project_root: None, + crate_path: None, + config_path: None, + }) + .unwrap(); + + let spec = resolve_run_spec( + Some(MobileTarget::Ios), + Some("sample_fns::fibonacci".into()), + Some(1), + Some(0), + vec!["iPhone 7-10".to_string()], + &layout, + None, + None, + Vec::new(), + None, + None, + None, + Some("10.0".to_string()), + None, + None, + None, + false, + false, + false, + ) + .expect("resolve spec"); + + assert_eq!(spec.ios_deployment_target.as_deref(), Some("10.0")); + assert_eq!(spec.ios_runner.as_deref(), Some("uikit-legacy")); + } + + #[test] + fn resolve_run_spec_rejects_ios_device_below_deployment_target() { + let layout = resolve_project_layout(ProjectLayoutOptions { + start_dir: None, + project_root: None, + crate_path: None, + config_path: None, + }) + .unwrap(); + + let err = resolve_run_spec( + Some(MobileTarget::Ios), + Some("sample_fns::fibonacci".into()), + Some(1), + Some(0), + vec!["iPhone 7-10".to_string()], + &layout, + None, + None, + Vec::new(), + None, + None, + None, + Some("15.0".to_string()), + None, + None, + None, + false, + false, + false, + ) + .expect_err("iOS 10 device should reject iOS 15 app"); + + assert!( + err.to_string() + .contains("cannot run app with iOS deployment target `15.0`"), + "unexpected error: {err}" + ); + } + + #[test] + fn resolve_run_spec_reads_android_watchdog_from_config_and_cli() { + let temp_dir = TempDir::new().expect("temp dir"); + let config_path = temp_dir.path().join("bench-config.toml"); + + let config_toml = r#"target = "android" +function = "sample_fns::fibonacci" +iterations = 10 +warmup = 2 +device_matrix = "device-matrix.yaml" + +[browserstack] +app_automate_username = "user" +app_automate_access_key = "key" +project = "proj" +android_benchmark_timeout_secs = 120 +android_heartbeat_interval_secs = 7 +"#; + write_file(&config_path, config_toml.as_bytes()).expect("write config"); + write_file( + &temp_dir.path().join("device-matrix.yaml"), + br#"devices: + - name: Google Pixel 8 + os: android + os_version: "14" +"#, + ) + .expect("write matrix"); + + let layout = resolve_project_layout(ProjectLayoutOptions { + start_dir: None, + project_root: None, + crate_path: None, + config_path: None, + }) + .unwrap(); + let spec = resolve_run_spec( + Some(MobileTarget::Android), + Some("ignored::value".into()), + Some(1), + Some(0), + Vec::new(), + &layout, + Some(config_path.as_path()), + None, + Vec::new(), + None, + None, + None, + None, + None, + Some(30), + Some(3), + false, + false, + false, + ) + .expect("resolve spec"); + + assert_eq!(spec.android_benchmark_timeout_secs, Some(30)); + assert_eq!(spec.android_heartbeat_interval_secs, Some(3)); + } + #[test] fn devices_resolve_parses() { let cli = Cli::parse_from([ @@ -8156,6 +9215,18 @@ test_suite = "target/ios/BenchRunnerUITests.zip" assert_eq!(ids, vec!["Pixel 6-12.0", "Pixel 7-13.0"]); } + #[test] + fn browserstack_identifier_preserves_versioned_ios_names() { + let (identifier, os_version) = browserstack_identifier_and_os_version("iPhone 7-10", ""); + assert_eq!(identifier, "iPhone 7-10"); + assert_eq!(os_version, "10"); + + let (identifier, os_version) = + browserstack_identifier_and_os_version("iPhone 7-10", "10.0"); + assert_eq!(identifier, "iPhone 7-10"); + assert_eq!(os_version, "10.0"); + } + fn safe_config_string() -> impl Strategy { "[A-Za-z0-9_. -]{1,32}".prop_map(|s| s.trim().to_string()) } @@ -8199,6 +9270,8 @@ test_suite = "target/ios/BenchRunnerUITests.zip" app_automate_access_key: access_key, project, ios_completion_timeout_secs: None, + android_benchmark_timeout_secs: None, + android_heartbeat_interval_secs: None, }, ios_xcuitest: None, }, @@ -8362,6 +9435,7 @@ test_suite = "target/ios/BenchRunnerUITests.zip" native_heap_kb: None, java_heap_kb: None, }), + failure: None, }], }], }); @@ -8408,6 +9482,7 @@ test_suite = "target/ios/BenchRunnerUITests.zip" native_heap_kb: None, java_heap_kb: None, }), + failure: None, }], }], }); @@ -8452,6 +9527,7 @@ test_suite = "target/ios/BenchRunnerUITests.zip" native_heap_kb: None, java_heap_kb: None, }), + failure: None, }], }], }); @@ -8495,6 +9571,7 @@ test_suite = "target/ios/BenchRunnerUITests.zip" native_heap_kb: None, java_heap_kb: None, }), + failure: None, }], }], }; @@ -8537,6 +9614,7 @@ test_suite = "target/ios/BenchRunnerUITests.zip" native_heap_kb: None, java_heap_kb: None, }), + failure: None, }], }], }); @@ -8586,6 +9664,7 @@ test_suite = "target/ios/BenchRunnerUITests.zip" native_heap_kb: None, java_heap_kb: None, }), + failure: None, }], }], }); @@ -8638,6 +9717,7 @@ test_suite = "target/ios/BenchRunnerUITests.zip" native_heap_kb: None, java_heap_kb: None, }), + failure: None, }], }], }); @@ -8661,6 +9741,10 @@ test_suite = "target/ios/BenchRunnerUITests.zip" browserstack: None, ios_xcuitest: None, ios_completion_timeout_secs: None, + ios_deployment_target: None, + ios_runner: None, + android_benchmark_timeout_secs: None, + android_heartbeat_interval_secs: None, }; let run_summary = RunSummary { spec: spec.clone(), @@ -8679,6 +9763,7 @@ test_suite = "target/ios/BenchRunnerUITests.zip" ] })], )])), + benchmark_failures: None, performance_metrics: None, }; @@ -8695,6 +9780,65 @@ test_suite = "target/ios/BenchRunnerUITests.zip" assert_eq!(usage.process_peak_memory_kb, Some(1_096)); } + #[test] + fn build_summary_preserves_browserstack_failure_results() { + let spec = RunSpec { + target: MobileTarget::Android, + function: "sample_fns::sleep".into(), + iterations: 3, + warmup: 1, + devices: vec!["Vivo Y21-11.0".into()], + browserstack: None, + ios_xcuitest: None, + ios_completion_timeout_secs: None, + ios_deployment_target: None, + ios_runner: None, + android_benchmark_timeout_secs: None, + android_heartbeat_interval_secs: None, + }; + let run_summary = RunSummary { + spec: spec.clone(), + artifacts: None, + local_report: json!({}), + remote_run: None, + summary: empty_summary(&spec), + benchmark_results: None, + benchmark_failures: Some(BTreeMap::from([( + "Vivo Y21-11.0".to_string(), + vec![json!({ + "schema_version": 1, + "platform": "android", + "device": "Vivo Y21-11.0", + "function_name": "sample_fns::sleep", + "kind": "timeout", + "message": "Timed out waiting 30s for benchmark completion", + "elapsed_ms": 30_000_u64, + "memory": { + "total_pss_kb": 1024_u64 + }, + "android_exit_info": { + "reason": "low_memory", + "raw_reason": 3 + } + })], + )])), + performance_metrics: None, + }; + + let summary = build_summary(&run_summary).expect("build summary"); + let benchmark = &summary.device_summaries[0].benchmarks[0]; + let failure = benchmark.failure.as_ref().expect("failure summary"); + let markdown = render_markdown_summary(&summary); + + assert_eq!(benchmark.function, "sample_fns::sleep"); + assert_eq!(benchmark.samples, 0); + assert_eq!(failure.kind, "timeout"); + assert_eq!(failure.elapsed_ms, Some(30_000)); + assert_eq!(failure.exit_reason.as_deref(), Some("low_memory")); + assert!(markdown.contains("failed (timeout)")); + assert!(markdown.contains("low_memory")); + } + #[test] fn build_summary_prefers_measured_peak_memory_over_browserstack_perf_memory() { let spec = RunSpec { @@ -8706,6 +9850,10 @@ test_suite = "target/ios/BenchRunnerUITests.zip" browserstack: None, ios_xcuitest: None, ios_completion_timeout_secs: None, + ios_deployment_target: None, + ios_runner: None, + android_benchmark_timeout_secs: None, + android_heartbeat_interval_secs: None, }; let run_summary = RunSummary { spec: spec.clone(), @@ -8723,6 +9871,7 @@ test_suite = "target/ios/BenchRunnerUITests.zip" ] })], )])), + benchmark_failures: None, performance_metrics: Some(BTreeMap::from([( "Google Pixel 8-14.0".to_string(), browserstack::PerformanceMetrics { @@ -8819,6 +9968,10 @@ test_suite = "target/ios/BenchRunnerUITests.zip" warmup: 1, devices: vec![], ios_completion_timeout_secs: None, + ios_deployment_target: None, + ios_runner: None, + android_benchmark_timeout_secs: None, + android_heartbeat_interval_secs: None, browserstack: None, ios_xcuitest: None, }; @@ -8835,10 +9988,15 @@ test_suite = "target/ios/BenchRunnerUITests.zip" warmup: 1, devices: vec![], ios_completion_timeout_secs: None, + ios_deployment_target: None, + ios_runner: None, + android_benchmark_timeout_secs: None, + android_heartbeat_interval_secs: None, browserstack: None, ios_xcuitest: None, }), benchmark_results: None, + benchmark_failures: None, performance_metrics: None, }; run_summary.summary = build_summary(&run_summary).expect("build summary"); @@ -9019,6 +10177,7 @@ Samsung Galaxy S23-14.0,basic_benchmark::bench_checksum,5,136000000,136000000,14 std_dev_ms: None, }, resource_usage: None, + failure: None, }], iterations: 5, warmup: 1, @@ -9044,6 +10203,7 @@ Samsung Galaxy S23-14.0,basic_benchmark::bench_checksum,5,136000000,136000000,14 std_dev_ms: None, }, resource_usage: None, + failure: None, }], iterations: 5, warmup: 1, @@ -9323,6 +10483,7 @@ mod ci_merge_tests { min_ns: Some(16_000), max_ns: Some(19_000), resource_usage: None, + failure: None, }], }], }); @@ -9545,6 +10706,10 @@ mod ci_merge_tests { browserstack: None, ios_xcuitest: None, ios_completion_timeout_secs: None, + ios_deployment_target: None, + ios_runner: None, + android_benchmark_timeout_secs: None, + android_heartbeat_interval_secs: None, }; let local_report = json!({}); let run_summary = RunSummary { @@ -9571,6 +10736,7 @@ mod ci_merge_tests { } })], )])), + benchmark_failures: None, performance_metrics: None, }; @@ -9599,6 +10765,10 @@ mod ci_merge_tests { browserstack: None, ios_xcuitest: None, ios_completion_timeout_secs: None, + ios_deployment_target: None, + ios_runner: None, + android_benchmark_timeout_secs: None, + android_heartbeat_interval_secs: None, }; let run_summary = RunSummary { spec: spec.clone(), @@ -9620,6 +10790,7 @@ mod ci_merge_tests { } })], )])), + benchmark_failures: None, performance_metrics: Some(BTreeMap::from([( "iPhone 15-17.0".to_string(), browserstack::PerformanceMetrics { diff --git a/crates/mobench/src/profile.rs b/crates/mobench/src/profile.rs index 4e25d2e..4db41c8 100644 --- a/crates/mobench/src/profile.rs +++ b/crates/mobench/src/profile.rs @@ -2706,6 +2706,10 @@ fn execute_local_android_capture( warmup: DEFAULT_PROFILE_WARMUP, devices: Vec::new(), ios_completion_timeout_secs: None, + ios_deployment_target: None, + ios_runner: None, + android_benchmark_timeout_secs: None, + android_heartbeat_interval_secs: None, browserstack: None, ios_xcuitest: None, }; @@ -2976,6 +2980,10 @@ fn execute_local_ios_capture(args: &ProfileRunArgs, manifest: &mut ProfileManife warmup: DEFAULT_PROFILE_WARMUP, devices: Vec::new(), ios_completion_timeout_secs: None, + ios_deployment_target: None, + ios_runner: None, + android_benchmark_timeout_secs: None, + android_heartbeat_interval_secs: None, browserstack: None, ios_xcuitest: None, }; @@ -2986,7 +2994,7 @@ fn execute_local_ios_capture(args: &ProfileRunArgs, manifest: &mut ProfileManife ensure_local_ios_simulator_booted(&simulator)?; manifest.capture_metadata.device = Some(simulator.identifier()); - run_ios_build(&layout, false, false, None)?; + run_ios_build(&layout, false, false, None, None, None)?; let app_path = build_local_ios_simulator_app(&layout, &simulator)?; install_local_ios_app(&simulator, &app_path)?; diff --git a/crates/mobench/src/summarize.rs b/crates/mobench/src/summarize.rs index c83cec1..5c3f9c6 100644 --- a/crates/mobench/src/summarize.rs +++ b/crates/mobench/src/summarize.rs @@ -42,6 +42,19 @@ pub struct BenchmarkResult { pub timing: TimingStats, #[serde(skip_serializing_if = "Option::is_none")] pub resource_usage: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub failure: Option, +} + +/// Structured benchmark failure loaded from `failure.json`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BenchmarkFailure { + pub kind: String, + pub message: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub elapsed_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub exit_reason: Option, } /// Timing statistics across all iterations (in milliseconds). @@ -180,7 +193,12 @@ fn parse_benchmark_entry(value: &serde_json::Value) -> Result { .unwrap_or("unknown") .to_string(); - let label = humanize_benchmark_name(&name); + let failure = parse_failure(value); + let label = if failure.is_some() { + name.clone() + } else { + humanize_benchmark_name(&name) + }; let ns_to_ms = |key: &str| -> f64 { value.get(key).and_then(|v| v.as_f64()).unwrap_or(0.0) / 1_000_000.0 }; @@ -203,6 +221,7 @@ fn parse_benchmark_entry(value: &serde_json::Value) -> Result { label, timing, resource_usage: parse_resource_usage(value), + failure, }) } @@ -262,6 +281,8 @@ pub fn load_results_dir(dir: &Path) -> Result { } covered_summary_dirs.push(summary_dir.to_path_buf()); all_platforms.extend(report.platforms); + } else if let Ok(report) = parse_raw_failure_report(&path, &value) { + all_platforms.extend(report.platforms); } else { raw_candidates.push((path, value)); } @@ -271,6 +292,8 @@ pub fn load_results_dir(dir: &Path) -> Result { for (path, value) in raw_candidates { if let Ok(report) = parse_raw_bench_report(&path, &value) { all_platforms.extend(report.platforms); + } else if let Ok(report) = parse_raw_failure_report(&path, &value) { + all_platforms.extend(report.platforms); } } } @@ -502,6 +525,75 @@ fn parse_raw_bench_report(path: &Path, value: &serde_json::Value) -> Result Result { + let entries = match value { + serde_json::Value::Array(items) => items + .iter() + .filter_map(normalize_failure_entry) + .collect::>(), + _ => normalize_failure_entry(value).into_iter().collect(), + }; + if entries.is_empty() { + anyhow::bail!("Not a benchmark failure report"); + } + + let first = entries + .first() + .ok_or_else(|| anyhow::anyhow!("missing failure entries"))?; + let platform = first + .get("platform") + .and_then(|value| value.as_str()) + .map(ToOwned::to_owned) + .unwrap_or_else(|| infer_platform(path, first.get("device").and_then(|v| v.as_str()))); + let device = first + .get("device") + .and_then(|v| v.as_str()) + .map(parse_device_string) + .unwrap_or_else(|| default_device_info(&platform)); + let benchmarks = entries + .iter() + .map(parse_benchmark_entry) + .collect::>>()?; + + Ok(SummarizeReport { + platforms: vec![PlatformReport { + platform, + device, + benchmarks, + iterations: 0, + warmup: 0, + }], + }) +} + +fn normalize_failure_entry(value: &serde_json::Value) -> Option { + let object = value.as_object()?; + let kind = object.get("kind").and_then(|value| value.as_str())?; + let function = object + .get("function_name") + .or_else(|| object.get("function")) + .and_then(|value| value.as_str())?; + let mut normalized = value.clone(); + let normalized_object = normalized.as_object_mut()?; + normalized_object.insert( + "function".to_string(), + serde_json::Value::String(function.to_string()), + ); + normalized_object.insert( + "failure".to_string(), + serde_json::json!({ + "kind": kind, + "message": object.get("message").and_then(|value| value.as_str()).unwrap_or("no message"), + "elapsed_ms": object.get("elapsed_ms").and_then(|value| value.as_u64()), + "exit_reason": object + .get("android_exit_info") + .and_then(|info| info.get("reason")) + .and_then(|value| value.as_str()), + }), + ); + Some(normalized) +} + fn normalize_raw_benchmark_entry(value: &serde_json::Value) -> Option { let mut value = value.clone(); let samples = extract_raw_samples(&value); @@ -660,6 +752,36 @@ fn parse_resource_usage(value: &serde_json::Value) -> Option { .or_else(|| value.get("resources").and_then(parse_resource_usage_object)) } +fn parse_failure(value: &serde_json::Value) -> Option { + let failure = value.get("failure").unwrap_or(value); + let kind = failure.get("kind").and_then(|value| value.as_str())?; + let message = failure + .get("message") + .and_then(|value| value.as_str()) + .unwrap_or("no message") + .to_string(); + let exit_reason = failure + .get("exit_reason") + .and_then(|value| value.as_str()) + .map(ToOwned::to_owned) + .or_else(|| { + value + .get("android_exit_info") + .and_then(|info| info.get("reason")) + .and_then(|reason| reason.as_str()) + .map(ToOwned::to_owned) + }); + Some(BenchmarkFailure { + kind: kind.to_string(), + message, + elapsed_ms: failure + .get("elapsed_ms") + .and_then(json_value_to_u64) + .or_else(|| value.get("elapsed_ms").and_then(json_value_to_u64)), + exit_reason, + }) +} + fn parse_resource_usage_object(value: &serde_json::Value) -> Option { let object = value.as_object()?; @@ -795,7 +917,11 @@ fn render_platform_table(platform: &PlatformReport) -> String { .load_preset(UTF8_FULL) .set_content_arrangement(ContentArrangement::Dynamic); + let has_failures = platform.benchmarks.iter().any(|b| b.failure.is_some()); let mut headers = vec!["Benchmark", "Avg ms", "Best", "Worst", "Median", "P95"]; + if has_failures { + headers.extend(["Status", "Failure", "Exit reason"]); + } if has_resource_usage { headers.extend(["CPU total", "Peak growth", "Process peak"]); } @@ -806,14 +932,50 @@ fn render_platform_table(platform: &PlatformReport) -> String { ); for bench in &platform.benchmarks { - let mut row = vec![ - Cell::new(&bench.label), - Cell::new(format!("{:.1}", bench.timing.avg_ms)).add_attribute(Attribute::Bold), - Cell::new(format!("{:.1}", bench.timing.best_ms)), - Cell::new(format!("{:.1}", bench.timing.worst_ms)), - Cell::new(format!("{:.1}", bench.timing.median_ms)), - Cell::new(format!("{:.1}", bench.timing.p95_ms)), - ]; + let mut row = if bench.failure.is_some() { + vec![ + Cell::new(&bench.label), + Cell::new("—").add_attribute(Attribute::Bold), + Cell::new("—"), + Cell::new("—"), + Cell::new("—"), + Cell::new("—"), + ] + } else { + vec![ + Cell::new(&bench.label), + Cell::new(format!("{:.1}", bench.timing.avg_ms)).add_attribute(Attribute::Bold), + Cell::new(format!("{:.1}", bench.timing.best_ms)), + Cell::new(format!("{:.1}", bench.timing.worst_ms)), + Cell::new(format!("{:.1}", bench.timing.median_ms)), + Cell::new(format!("{:.1}", bench.timing.p95_ms)), + ] + }; + + if has_failures { + if let Some(failure) = &bench.failure { + row.push(Cell::new("failed")); + row.push(Cell::new(format!( + "{}: {} ({})", + failure.kind, + failure.message, + failure + .elapsed_ms + .map(|value| format!("{value}ms")) + .unwrap_or_else(|| "elapsed unknown".to_string()) + ))); + row.push(Cell::new( + failure + .exit_reason + .clone() + .unwrap_or_else(|| "unavailable".to_string()), + )); + } else { + row.push(Cell::new("passed")); + row.push(Cell::new("—")); + row.push(Cell::new("—")); + } + } if has_resource_usage { if let Some(ru) = &bench.resource_usage { @@ -873,29 +1035,64 @@ pub fn render_markdown(report: &SummarizeReport) -> String { .benchmarks .iter() .any(|b| b.resource_usage.is_some()); + let has_failures = platform.benchmarks.iter().any(|b| b.failure.is_some()); - if has_ru { - output.push_str( - "| Benchmark | Avg ms | Best | Worst | Median | P95 | CPU total | Peak growth | Process peak |\n", - ); - output.push_str( - "|-----------|--------|------|-------|--------|-----|-----------|-------------|--------------|\n", - ); + if has_ru || has_failures { + output.push_str("| Benchmark | Avg ms | Best | Worst | Median | P95 |"); + if has_failures { + output.push_str(" Status | Failure | Exit reason |"); + } + if has_ru { + output.push_str(" CPU total | Peak growth | Process peak |"); + } + output.push('\n'); + output.push_str("|-----------|--------|------|-------|--------|-----|"); + if has_failures { + output.push_str("--------|---------|-------------|"); + } + if has_ru { + output.push_str("-----------|-------------|--------------|"); + } + output.push('\n'); } else { output.push_str("| Benchmark | Avg ms | Best | Worst | Median | P95 |\n"); output.push_str("|-----------|--------|------|-------|--------|-----|\n"); } for bench in &platform.benchmarks { - let mut row = format!( - "| {} | **{:.1}** | {:.1} | {:.1} | {:.1} | {:.1} |", - bench.label, - bench.timing.avg_ms, - bench.timing.best_ms, - bench.timing.worst_ms, - bench.timing.median_ms, - bench.timing.p95_ms, - ); + let mut row = if bench.failure.is_some() { + format!("| {} | **—** | — | — | — | — |", bench.label) + } else { + format!( + "| {} | **{:.1}** | {:.1} | {:.1} | {:.1} | {:.1} |", + bench.label, + bench.timing.avg_ms, + bench.timing.best_ms, + bench.timing.worst_ms, + bench.timing.median_ms, + bench.timing.p95_ms, + ) + }; + + if has_failures { + if let Some(failure) = &bench.failure { + row.push_str(&format!( + " failed | {}: {} ({}) | {} |", + failure.kind, + failure.message.replace('|', "\\|"), + failure + .elapsed_ms + .map(|value| format!("{value}ms")) + .unwrap_or_else(|| "elapsed unknown".to_string()), + failure + .exit_reason + .clone() + .unwrap_or_else(|| "unavailable".to_string()) + )); + } else { + row.push_str(" passed | — | — |"); + } + } if has_ru { if let Some(ru) = &bench.resource_usage { @@ -1147,6 +1344,47 @@ mod tests { ); } + #[test] + fn test_load_results_dir_preserves_android_failure_json() { + let temp = tempfile::TempDir::new().expect("temp dir"); + let failure_dir = temp.path().join("android/passport"); + std::fs::create_dir_all(&failure_dir).expect("create failure dir"); + std::fs::write( + failure_dir.join("failure.json"), + serde_json::to_string_pretty(&json!({ + "schema_version": 1, + "platform": "android", + "device": "Vivo Y21-11.0", + "function_name": "provekit::passport", + "kind": "timeout", + "message": "Timed out waiting 30s for benchmark completion", + "elapsed_ms": 30000, + "android_exit_info": { + "reason": "low_memory", + "raw_reason": 3 + } + })) + .unwrap(), + ) + .expect("write failure"); + + let report = load_results_dir(temp.path()).expect("load failure report"); + let platform = &report.platforms[0]; + let benchmark = &platform.benchmarks[0]; + let failure = benchmark.failure.as_ref().expect("failure summary"); + assert_eq!(platform.platform, "android"); + assert_eq!(platform.device.name, "Vivo Y21"); + assert_eq!(benchmark.name, "provekit::passport"); + assert_eq!(failure.kind, "timeout"); + assert_eq!(failure.elapsed_ms, Some(30000)); + assert_eq!(failure.exit_reason.as_deref(), Some("low_memory")); + + let markdown = render_markdown(&report); + assert!(markdown.contains("provekit::passport")); + assert!(markdown.contains("timeout")); + assert!(markdown.contains("low_memory")); + } + #[test] fn test_load_results_dir_backfills_resource_usage_from_nested_summaries() { let fixture_dir = @@ -1323,6 +1561,7 @@ mod tests { native_heap_kb: Some(120000), java_heap_kb: Some(45000), }), + failure: None, }], iterations: 30, warmup: 5, @@ -1359,6 +1598,7 @@ mod tests { std_dev_ms: Some(35.2), }, resource_usage: None, + failure: None, }], iterations: 30, warmup: 5, @@ -1599,6 +1839,7 @@ mod tests { std_dev_ms: None, }, resource_usage: None, + failure: None, }], iterations: 5, warmup: 1, @@ -1624,6 +1865,7 @@ mod tests { std_dev_ms: None, }, resource_usage: None, + failure: None, }], iterations: 5, warmup: 1, diff --git a/docs/guides/browserstack-ci.md b/docs/guides/browserstack-ci.md index d0c11a8..f1efa0d 100644 --- a/docs/guides/browserstack-ci.md +++ b/docs/guides/browserstack-ci.md @@ -72,6 +72,34 @@ Invalid devices (1): - Google Pixel 7 Pro-13.0 ``` +### Legacy iOS Devices + +mobench supports two generated iOS runner templates: + +- `swiftui`: the default iOS 15+ runner used by current CI lanes. +- `uikit-legacy`: a UIKit runner without SwiftUI or Swift concurrency for lower deployment targets. + +The current/default Xcode lane is treated as iOS 15+. Older BrowserStack iOS devices require both the legacy runner and an old enough Xcode toolchain. BrowserStack App Automate currently lists iPhone 7 as iOS 10, so an iPhone 7 legacy lane needs an Xcode installation capable of building/installing for iOS 10/11/12. + +Example `mobench.toml`: + +```toml +[ios] +deployment_target = "10.0" +runner = "uikit-legacy" +``` + +Example device matrix entry: + +```yaml +devices: + - name: "iPhone 7-10" + os: "ios" + tags: ["legacy-ios"] +``` + +mobench fails early if the selected BrowserStack iOS device version is lower than the app deployment target, for example `iPhone 7-10` with `deployment_target = "15.0"`. It also checks the selected local Xcode version before iOS builds and reports when a legacy deployment target needs an older Xcode lane. + ### Verify Benchmark Setup ```bash diff --git a/docs/specs/mobench-device-farm-spec.md b/docs/specs/mobench-device-farm-spec.md new file mode 100644 index 0000000..c58b8b1 --- /dev/null +++ b/docs/specs/mobench-device-farm-spec.md @@ -0,0 +1,1084 @@ +# mobench Device Farm Provider Spec + +Status: draft +Updated: 2026-04-30 + +This spec describes a `mobench` provider for running mobile benchmarks on a +private fleet of real Android and iOS devices. The provider is intended for +devices that are unavailable or unsuitable in hosted device-cloud services. + +The device farm is narrower than a general-purpose mobile testing cloud. It +installs already-built benchmark apps, runs the generated Espresso or XCUITest +harness, collects logs and device metadata, and returns benchmark results +through an async API. + +## Goals + +- Run `mobench` benchmarks on managed physical Android and iOS devices. +- Add a `mobench`-native provider API alongside existing providers. +- Support async multi-device runs from day one. +- Return parsed benchmark results and raw debug artifacts. +- Integrate with local development and CI without requiring the farm to build + mobile artifacts. +- Provide enough operator tooling to run and maintain a small unattended device + fleet. + +## Non-Goals + +V1 does not attempt to: + +- Simulate mobile OS APIs such as wallet payments, biometrics, camera, GPS, or + sensors. +- Provide arbitrary remote device interaction. +- Execute arbitrary shell commands supplied by API callers. +- Replace all hosted device-cloud usage. +- Build or sign mobile artifacts inside the farm. +- Provide native flamegraph/profiling capture. +- Ship a full web UI before the API and operator CLI exist. + +## Provider Model + +The farm should fit the existing `mobench` lifecycle: + +1. Build platform artifacts locally or in CI. +2. Upload artifacts to the provider. +3. Schedule a provider run on one or more devices. +4. Poll until completion. +5. Fetch logs, artifacts, and parsed benchmark results. +6. Write normalized `mobench` outputs such as `summary.json`, `summary.md`, and + `results.csv`. + +The API should be native to `mobench`; it should not copy another provider's +endpoint names, payload quirks, or product terminology. + +## Architecture + +Use one unified control-plane API with platform-specific rack agents. + +Components: + +- Control plane API: REST JSON API for artifacts, runs, sessions, devices, + pools, identities, and admin operations. +- Scheduler: validates requests, reserves devices, creates per-device sessions, + and assigns sessions to agents. +- Object storage: stores uploaded apps, test bundles, raw logs, screenshots, + result bundles, and large artifacts. +- Relational database: stores projects, identities, agents, devices, pools, + runs, sessions, leases, events, metrics, and normalized result rows. +- Android agents: controller processes connected to Android phones over USB, + using `adb` and Android test tooling. +- iOS agents: controller processes connected to iPhones over USB, using Xcode + command-line tooling for install and XCUITest execution. +- Operator CLI: scriptable fleet-management tool for device inspection, + quarantine, logs, and run cancellation. +- `mobench` provider adapter: CLI integration selected by `--provider`. + +Recommended physical split: + +- Linux controllers for Android devices. +- macOS controllers for iOS devices. +- Wired Ethernet for controllers. +- Stable powered USB hubs for phones. +- Outbound-only agent connections to the control plane. + +## Device Procurement + +The pilot should optimize for end-to-end reliability before broad coverage. + +Target device classes should be defined by measurable properties instead of only +model names. + +Example Android tail-end class: + +- 2 GB RAM. +- Low-end CPU representative of the target userbase. +- 32-bit ARM userspace or 32-bit-capable SoC where relevant. +- Android version capable of running the generated app and Espresso harness. +- 16-32 GB storage. +- Enough devices for multi-device runs plus at least one spare. + +Example iOS tail-end class: + +- Lowest-end arm64 iPhone models still compatible with the supported iOS target. +- Reliable signed app installation and XCUITest execution. +- Enough devices for multi-device runs plus at least one spare. + +`mobench` may need platform-support work for older devices: + +- Android jobs targeting 32-bit devices should build and declare + `armeabi-v7a`. +- Android SDK and dependency defaults may need to remain configurable. +- iOS 32-bit devices are out of scope; low-end iOS means older supported arm64 + devices. + +## Physical Inventory + +Every physical element should map to device inventory. + +Example labeling scheme: + +```text +controller: android-rack-01, ios-rack-01 +USB hub: android-rack-01-hub-a +port: android-rack-01-hub-a-p03 +device: android-low-001 +asset tag / serial: stored in inventory +``` + +Each device record should expose enough information for an operator to find and +service the device: + +```json +{ + "device_id": "android-low-001", + "controller_id": "android-rack-01", + "usb_path": "android-rack-01-hub-a-p03", + "serial_number": "...", + "asset_tag": "...", + "physical_label": "android-low-001" +} +``` + +Power recommendations: + +- Use stable powered USB hubs. +- Prefer managed hubs with per-port power switching. +- At minimum, use controllable power at hub/controller level. +- Do not block the first pilot on per-port power switching if procurement is + uncertain. + +Network recommendations: + +- Controllers should use wired Ethernet. +- Agents should initiate outbound HTTPS long-polling to the control plane. +- The farm should not require inbound access to rack controllers. + +## Device Setup + +Android phones should be preconfigured once: + +- Developer options enabled. +- USB debugging enabled. +- Stay-awake while charging enabled. +- Animations disabled. +- Lock screen disabled where possible. +- Battery optimization disabled for the test app where possible. +- Connected to stable power. +- Enrolled in inventory with serial, model, OS version, ABI list, RAM, storage, + and physical label. + +iPhones should be preconfigured once: + +- Device IDs recorded in inventory. +- Trusted by the macOS controller. +- Prepared for command-line install/test workflows. +- Enrolled in the minimum device/provisioning setup needed for reliable signed + app installation. +- No dependency on personal developer accounts during normal farm operation. + +## Data Model + +Use a relational database for control-plane state and object storage for blobs. + +Core tables: + +- `projects` +- `identities` +- `identity_policies` +- `agents` +- `agent_heartbeats` +- `devices` +- `device_facts` +- `device_labels` +- `pools` +- `pool_memberships` +- `artifacts` +- `runs` +- `sessions` +- `leases` +- `session_events` +- `session_artifacts` +- `device_snapshots` +- `metric_samples` +- `benchmark_reports` +- `benchmark_summary_rows` + +Database records should store: + +- Run/session status transitions. +- Lease events. +- Failure reasons. +- Device facts, labels, snapshots, and health. +- Small metric samples. +- Parsed benchmark summaries. +- JSON copies of benchmark report payloads. +- Artifact metadata and object keys. + +Object storage should store: + +- App and test artifacts. +- Full device logs. +- Test runner and platform-tool output. +- Instrumentation logs. +- Screenshots/videos if enabled. +- Complete `bench-report.json` payloads. +- Zipped session artifact bundles. + +## Device Inventory and Pools + +Expose both raw device inventory and curated pools/profiles. + +API: + +```text +GET /v1/devices +GET /v1/devices/{device_id} +GET /v1/pools +GET /v1/pools/{pool_id} +``` + +Device example: + +```json +{ + "id": "android-low-001", + "display_name": "Android Low-End #1", + "target": "android", + "model": "Example Low-End Phone", + "os_version": "13", + "abi_list": ["armeabi-v7a"], + "ram_mb": 2048, + "storage_gb": 32, + "agent_id": "android-rack-01", + "usb_path": "android-rack-01-hub-a-p03", + "state": "available", + "pool_ids": ["android-tail-2gb-32bit"], + "labels": { + "tier": "tail-end" + } +} +``` + +Pool example: + +```json +{ + "pool_id": "android-tail-2gb-32bit", + "target": "android", + "description": "2 GB RAM 32-bit low-end Android devices", + "available_sessions": 3, + "capabilities": { + "ram_gb_max": 2, + "abi": "armeabi-v7a", + "tier": "tail-end" + } +} +``` + +Pool membership should combine: + +- automatic facts detected by agents +- durable pool definitions from reviewed configuration +- manual labels for benchmark intent and operational state + +Durable pool and policy definitions should be configuration-managed. Temporary +state such as quarantine, maintenance notes, and debug labels should live in the +API/admin plane. + +## Device State Machine + +Device states: + +```text +available +leased +running +recovering +quarantined +maintenance +retired +unknown +``` + +Recovery ladder: + +1. Kill stale runner/app processes. +2. Uninstall app/test packages. +3. Reconnect through platform tooling. +4. Reboot device. +5. Power-cycle USB port, hub, or controller-managed power if available. +6. Quarantine after repeated failures. + +Quarantined devices are excluded from normal pool scheduling but may remain +targetable by exact ID for diagnostics when caller policy allows exact-device +use. + +## Scheduling + +The API supports both: + +- curated pools/capability selectors +- exact physical device IDs + +CI should use pools or selectors. Exact IDs are for debugging, reproduction, +maintenance, and quarantine validation. + +A run can fan out across multiple physical devices. The control plane models +this as: + +- parent `run` +- child `session` per physical device + +Use all-or-nothing reservation for v1: + +- all requested devices must be reserved before sessions start +- queue timeout controls how long a run waits for capacity +- partial capacity produces `capacity_timeout` + +One physical device runs one session at a time. No shared-device parallelism. + +## Run and Session States + +Run statuses: + +```text +created +validating +queued +leasing +running +collecting +completed +failed +canceled +expired +``` + +Session statuses: + +```text +created +queued +leased +preparing_device +installing +running_test +collecting +completed +failed +recovering +quarantined_device +canceled +``` + +Failure codes: + +```text +artifact_invalid +capacity_timeout +install_failed +test_timeout +no_bench_report_found +device_disconnected +device_unhealthy +agent_lost +abi_mismatch +os_mismatch +runner_failed +result_parse_failed +internal_error +``` + +Timeouts should be phase-specific: + +```json +{ + "timeouts": { + "queue_secs": 900, + "device_prepare_secs": 180, + "install_secs": 300, + "test_secs": 900, + "collect_secs": 120, + "overall_secs": 1800 + } +} +``` + +Cancellation: + +```text +POST /v1/runs/{run_id}/cancel +``` + +Agents must stop the runner if possible, collect final logs, clean up app/test +packages, release leases, and mark sessions `canceled`. + +## Artifact Flow + +Use presigned object-storage URLs. Large binaries should not flow through the +control plane API process. + +Flow: + +```text +POST /v1/artifacts/initiate +PUT presigned_upload_url +POST /v1/artifacts/{id}/complete +POST /v1/runs +agent claims session and receives presigned download URLs +``` + +Artifact upload is generic, but run creation uses platform-specific roles. + +Android: + +```json +{ + "target": "android", + "artifacts": { + "app_apk": "artifact_123", + "test_apk": "artifact_456" + }, + "runner": { + "kind": "espresso" + } +} +``` + +iOS: + +```json +{ + "target": "ios", + "artifacts": { + "app_ipa": "artifact_789", + "xcuitest_zip": "artifact_abc" + }, + "runner": { + "kind": "xcuitest", + "only_testing": [ + "BenchRunnerUITests/BenchRunnerUITests/testLaunchAndCaptureBenchmarkReport" + ] + } +} +``` + +V1 signing/build policy: + +- CI or local `mobench` builds and signs artifacts before upload. +- The farm validates artifacts and installs them. +- The farm does not own mobile signing or build pipelines in v1. +- Server-side build/signing can be added later as a separate build service. + +## API Contract + +Use REST JSON with explicit versioning: + +```text +/v1/... +``` + +Payloads include: + +```json +{ + "api_version": "2026-04-30", + "result_schema": "mobench-farm-result-v1", + "bench_report_schema": "mobench-bench-report-v1" +} +``` + +Compatibility rules: + +- v1 clients ignore unknown fields +- server does not remove or rename existing v1 fields +- breaking changes require `/v2` or a new schema name + +Core endpoints: + +```text +POST /v1/artifacts/initiate +POST /v1/artifacts/{artifact_id}/complete +GET /v1/artifacts/{artifact_id} + +POST /v1/runs +GET /v1/runs/{run_id} +POST /v1/runs/{run_id}/cancel +GET /v1/runs/{run_id}/sessions +GET /v1/runs/{run_id}/results +GET /v1/runs/{run_id}/artifacts + +GET /v1/sessions/{session_id} +GET /v1/sessions/{session_id}/results +GET /v1/sessions/{session_id}/artifacts + +GET /v1/devices +GET /v1/devices/{device_id} +GET /v1/pools +GET /v1/pools/{pool_id} + +POST /v1/agents/{agent_id}/heartbeat +POST /v1/agents/{agent_id}/leases/claim +POST /v1/sessions/{session_id}/events +POST /v1/sessions/{session_id}/artifacts +POST /v1/sessions/{session_id}/metrics +``` + +Async completion: + +- polling is required in v1 +- webhooks are a possible extension + +Polling endpoints: + +```text +GET /v1/runs/{run_id} +GET /v1/runs/{run_id}/results +GET /v1/runs/{run_id}/artifacts +``` + +Future webhook shape: + +```json +{ + "webhook": { + "url": "https://ci.example/hook", + "events": ["run.completed", "run.failed"], + "secret_ref": "..." + } +} +``` + +## Run Creation Examples + +Pool-based Android run: + +```json +{ + "target": "android", + "device_request": { + "pool": "android-tail-2gb-32bit", + "count": 3, + "selector": { + "abi": "armeabi-v7a", + "ram_mb_max": 2300 + } + }, + "artifacts": { + "app_apk": "artifact_app", + "test_apk": "artifact_test" + }, + "runner": { + "kind": "espresso", + "instrumentation_args": { + "class": "dev.world.bench.MainActivityTest" + } + }, + "scheduling": { + "strategy": "all_or_nothing", + "queue_timeout_secs": 900 + }, + "timeouts": { + "queue_secs": 900, + "device_prepare_secs": 180, + "install_secs": 300, + "test_secs": 900, + "collect_secs": 120, + "overall_secs": 1800 + } +} +``` + +Exact-device iOS run: + +```json +{ + "target": "ios", + "device_request": { + "device_ids": ["ios-low-001", "ios-low-002"] + }, + "artifacts": { + "app_ipa": "artifact_app", + "xcuitest_zip": "artifact_test" + }, + "runner": { + "kind": "xcuitest", + "only_testing": [ + "BenchRunnerUITests/BenchRunnerUITests/testLaunchAndCaptureBenchmarkReport" + ] + }, + "scheduling": { + "strategy": "all_or_nothing", + "queue_timeout_secs": 900 + } +} +``` + +## Runner Security + +V1 supports fixed runner kinds only: + +- `espresso` for Android +- `xcuitest` for iOS + +Allowed arguments must be controlled and allowlisted. Do not expose arbitrary +shell execution through caller-facing APIs. + +The farm may run any APK/IPA/test bundle that matches the runner contract. +`mobench` metadata is preferred but not required for installation/execution. + +Result policy: + +- if `mobench` markers exist, parse and normalize results +- if markers do not exist, complete the run with raw logs and set parsed result + status to `no_bench_report_found` +- projects may require parsed `mobench` results for benchmark-gating workflows + +## Agent Behavior + +One cross-platform agent codebase should have platform-specific executors: + +- shared agent core: auth, heartbeat, lease claim, event upload, metric upload, + artifact download/upload, status handling +- Android executor: wraps `adb`, logcat, package install/uninstall, device facts, + and Espresso execution +- iOS executor: wraps Xcode/device tooling, install/uninstall where available, + device logs, and XCUITest execution + +Agents use polling or long-polling: + +```text +POST /v1/agents/{agent_id}/heartbeat +POST /v1/agents/{agent_id}/leases/claim +POST /v1/sessions/{session_id}/events +POST /v1/sessions/{session_id}/artifacts +POST /v1/sessions/{session_id}/metrics +``` + +Agent failure: + +- agents heartbeat frequently +- if heartbeat expires, active sessions become `agent_lost` +- leases are released only after a safety timeout +- devices managed by the lost agent become `unknown` +- devices return to scheduling only after rediscovery and health checks + +## Session Lifecycle + +Each benchmark session should start from a clean install/state. + +Typical Android lifecycle: + +1. Verify device is reachable through `adb`. +2. Capture pre-run snapshot. +3. Apply pre-run gates. +4. Uninstall old app/test packages if present. +5. Install app APK. +6. Install test APK. +7. Start logcat capture. +8. Run Espresso test. +9. Collect logs, instrumentation output, and screenshots if configured. +10. Parse or upload benchmark report artifacts. +11. Uninstall or leave installed based on maintenance policy. +12. Capture post-run snapshot. +13. Release lease. + +Typical iOS lifecycle: + +1. Verify device is visible to Xcode/device tooling. +2. Capture pre-run snapshot. +3. Apply pre-run gates. +4. Install signed app IPA. +5. Prepare XCUITest bundle. +6. Start device log capture. +7. Run XCUITest with `only_testing`. +8. Collect logs, test output, and screenshots if configured. +9. Parse or upload benchmark report artifacts. +10. Clean up where platform permits. +11. Capture post-run snapshot. +12. Release lease. + +Installation reuse is debug-only and disallowed for CI benchmark sessions. + +## Pre-Run Gates and Metrics + +Normalize device condition with hard gates where possible and record everything +else. + +Hard gates: + +- battery >= 40% or externally powered +- thermal state not hot/critical where observable +- screen unlocked and automation-ready +- free storage above configured threshold +- no active previous session processes +- device reachable through platform tooling +- expected ABI/OS/platform matches request +- artifact compatibility checks pass + +Recorded metadata: + +- battery percentage +- charging state +- thermal state +- uptime +- available RAM/storage +- OS version/build +- controller ID +- USB hub/port +- pre-run CPU/load snapshot +- device model and ABI list + +Metrics model: + +- `device_snapshot`: pre-run and post-run facts +- `metric_samples`: periodic samples during prepare/install/test, every 5-10 + seconds by default +- `benchmark_resource_usage`: benchmark-emitted resource numbers remain + authoritative for performance comparison + +Keep fleet/device health metrics separate from benchmark metrics. + +## Result Model + +Store: + +- original `BenchReport` JSON in object storage +- JSON copy in the relational database +- normalized summary rows in the relational database +- mobench-compatible aggregate grouped by device + +Minimum normalized fields: + +```text +session_id +device_id +function +iterations +warmup +sample_count +mean_ns +median_ns +p95_ns +min_ns +max_ns +cpu_total_ms +cpu_median_ms +peak_memory_kb +process_peak_memory_kb +created_at +``` + +Per-session result endpoint: + +```text +GET /v1/sessions/{session_id}/results +``` + +Run aggregate endpoint: + +```text +GET /v1/runs/{run_id}/results +``` + +Example aggregate: + +```json +{ + "run_id": "run_123", + "result_schema": "mobench-farm-result-v1", + "benchmark_results": { + "android-low-001": [ + { + "function": "sample_fns::fibonacci", + "samples": [], + "mean_ns": 123, + "median_ns": 120, + "min_ns": 110, + "max_ns": 150 + } + ] + }, + "devices": [ + { + "id": "android-low-001", + "display_name": "Android Low-End #1", + "model": "Example Low-End Phone", + "os": "android", + "os_version": "13", + "pool_ids": ["android-tail-2gb-32bit"] + } + ] +} +``` + +API payloads use stable device IDs as primary identifiers and include +human-readable labels for display. + +## Authentication and Authorization + +Support distinct identity classes: + +- CI identities +- internal scoped API tokens +- external scoped API tokens +- rack agent credentials +- human/admin identities + +External tokens are first-class principals. They can have explicit limits or +unbounded access when intentionally granted. + +Identity policy example: + +```json +{ + "project_id": "example-project", + "allowed_pools": ["android-tail", "ios-tail"], + "allowed_targets": ["android", "ios"], + "max_concurrent_runs": 2, + "max_concurrent_sessions": 6, + "max_queue_seconds": 1800, + "priority": "normal", + "retention_days": 30, + "can_use_exact_device_ids": false, + "can_create_unbounded_runs": false, + "expires_at": "2026-12-31T23:59:59Z" +} +``` + +Authorization is project-scoped from v1: + +- project owns runs, artifacts, and results +- project can access selected device pools +- project has concurrency quotas +- project has usage quotas, optionally unlimited +- project has artifact retention policy +- project has allowed CI identities and API tokens + +The same physical fleet can be shared across internal and external users. +Policies should live on auth identities and projects rather than requiring +separate hardware. + +## Retention + +Suggested defaults: + +- raw logs/artifacts: 30 days +- parsed benchmark summaries: longer-lived, project-policy controlled +- failed-run debug bundles: 30 days unless project policy shortens or extends +- external tokens may default to shorter retention unless explicitly raised + +Retention should be enforced through object-storage lifecycle policies plus +database cleanup jobs. + +## mobench CLI Integration + +Add a general provider model. + +Example CLI: + +```bash +cargo mobench run \ + --provider private_farm \ + --target android \ + --function sample_fns::fibonacci \ + --pool android-tail-2gb-32bit \ + --count 3 \ + --release \ + --fetch \ + --output target/mobench/results.json +``` + +Keep `--devices` compatibility, but add farm-native selectors: + +```bash +cargo mobench run --provider private_farm --target android --pool android-tail-2gb-32bit --count 3 +cargo mobench run --provider private_farm --target android --device-id android-low-001 +cargo mobench run --provider private_farm --target android --selector abi=armeabi-v7a,ram_mb_max=2300 --count 2 +``` + +Provider config: + +```toml +[providers.browserstack] +kind = "browserstack" +username_env = "BROWSERSTACK_USERNAME" +access_key_env = "BROWSERSTACK_ACCESS_KEY" + +[providers.private_farm] +kind = "mobench-farm" +base_url = "https://mobench-farm.example" +token_env = "MOBENCH_FARM_TOKEN" + +[run] +provider = "private_farm" +``` + +Provider behavior: + +- Hosted providers and farm providers remain separate. +- A single run targets one provider in v1. +- Device matrices may include provider-specific profiles. +- Cross-provider comparison happens in reporting. + +## CI Integration + +V1 flow: + +1. CI runner builds Android/iOS artifacts using existing `mobench` build paths. +2. `mobench` initiates artifact upload and receives presigned URLs. +3. CI uploads artifacts to object storage. +4. `mobench` creates a farm run. +5. `mobench --fetch` polls the farm until completion. +6. `mobench` downloads or receives aggregate results. +7. Existing outputs are written: + - `summary.json` + - `summary.md` + - `results.csv` + - optional JUnit/check-run output + +CI cancellation should call farm cancellation so devices are not held until +timeout. + +## Operator CLI + +V1 should ship an operator CLI before a web UI. + +Required commands: + +```bash +farmctl agents list +farmctl devices list --pool android-tail-2gb-32bit +farmctl devices inspect android-low-001 +farmctl devices quarantine android-low-001 --reason "USB disconnect loop" +farmctl devices unquarantine android-low-001 +farmctl runs list --status running +farmctl runs cancel run_123 +farmctl sessions logs ses_123 +farmctl pools list +``` + +The CLI should be usable by the person physically maintaining the rack. + +## Observability + +Control plane: + +- structured JSON logs +- request IDs +- run/session event timeline +- metrics for queue depth, run duration, failure codes, artifact volume, and + API errors + +Agents: + +- structured logs +- heartbeat metrics +- device discovery events +- command duration and exit status +- recovery attempts and quarantine causes + +Alerts: + +- agent offline +- stuck queue +- high install/test failure rate +- repeated device disconnects +- high quarantined device count +- object storage growth above threshold +- runs stuck beyond timeout + +## Security Considerations + +The farm installs arbitrary mobile binaries onto real devices. The security +model must assume uploaded apps can be malicious or broken. + +V1 controls: + +- no arbitrary shell command execution for callers +- fixed runner kinds only +- per-project and per-identity quotas +- per-agent credentials with independent revocation +- short-lived presigned URLs +- audit logs for run creation, cancellation, artifact access, token use, and + admin operations +- outbound-only agent connectivity +- scoped external tokens with expiration and optional IP allowlists +- secrets stored in a managed secret store + +## Rollout Plan + +Phase 0: local spike + +- one Android phone connected to a developer machine +- manually install/run generated APK/test APK +- prove log/result extraction on the target device class +- identify `mobench` changes needed for `armeabi-v7a` or older SDK support + +Phase 1: Android rack pilot + +- one Android controller +- 2-3 Android phones plus spare +- minimal agent +- local or development control plane +- repeatable multi-device Espresso jobs + +Phase 2: iOS rack pilot + +- one macOS controller +- 1-2 iPhones plus spare +- XCUITest execution through the same control plane model +- validate signing/provisioning assumptions + +Phase 3: unified provider integration + +- add `--provider` model to `mobench` +- add farm provider config +- add artifact upload/run/poll/fetch adapter +- write existing CI output contract + +Phase 4: CI pilot + +- run farm-backed Android and iOS jobs from CI +- support cancellation +- publish summaries/checks +- monitor a soak period + +Phase 5: external access pilot + +- create scoped API tokens for selected external users +- validate quotas, retention, audit logs, and support process + +Phase 6: fleet expansion + +- add more devices matching measured target classes +- tune controller-to-phone ratios +- add per-port power switching if pilot data justifies it + +## Pilot Success Criteria + +The pilot is successful when: + +- CI can submit Android and iOS farm runs through `mobench`. +- The farm runs multi-device jobs unattended. +- Results include parsed benchmark numbers grouped by stable device ID. +- Raw logs and session artifacts are available for failed runs. +- Common device failures trigger recovery or quarantine without blocking the + whole fleet. +- No manual intervention is needed across a meaningful soak window. +- Operators can identify and service a physical phone from API/CLI inventory + data. + +## Open Questions + +These should be resolved before purchase or production deployment: + +- Which exact Android and iOS models best represent the target device classes? +- What is the acceptable budget for controllers, phones, hubs, mounts, power, + and spares? +- What device provisioning setup should be used for iOS test devices? +- What retention and access policies should external identities receive by + default? +- Should the control plane be private-network-only or expose a public API with + strict auth and rate limits? + diff --git a/templates/ios/BenchRunner/BenchRunner/UIKitLegacyRunner.swift.template b/templates/ios/BenchRunner/BenchRunner/UIKitLegacyRunner.swift.template new file mode 100644 index 0000000..7797c53 --- /dev/null +++ b/templates/ios/BenchRunner/BenchRunner/UIKitLegacyRunner.swift.template @@ -0,0 +1,163 @@ +import UIKit + +private struct ProfileLaunchOptions { + let benchDelayMs: UInt64 + let resultHoldMs: UInt64 + let repeatUntilMs: UInt64 + let warmupOnly: Bool + + static func resolved() -> ProfileLaunchOptions { + let info = ProcessInfo.processInfo + + var benchDelayMs = UInt64(info.environment["MOBENCH_BENCH_DELAY_MS"] ?? "0") ?? 0 + var resultHoldMs = UInt64( + info.environment["MOBENCH_PROFILE_RESULT_HOLD_MS"] ?? "5000" + ) ?? 5000 + var repeatUntilMs = UInt64( + info.environment["MOBENCH_PROFILE_REPEAT_UNTIL_MS"] ?? "0" + ) ?? 0 + var warmupOnly = info.environment["MOBENCH_PROFILE_WARMUP_ONLY"] == "1" + + for arg in info.arguments { + if arg.hasPrefix("--mobench-profile-bench-delay-ms="), + let value = arg.split(separator: "=", maxSplits: 1).last, + let parsed = UInt64(value) { + benchDelayMs = parsed + } else if arg.hasPrefix("--mobench-profile-result-hold-ms="), + let value = arg.split(separator: "=", maxSplits: 1).last, + let parsed = UInt64(value) { + resultHoldMs = parsed + } else if arg.hasPrefix("--mobench-profile-repeat-until-ms="), + let value = arg.split(separator: "=", maxSplits: 1).last, + let parsed = UInt64(value) { + repeatUntilMs = parsed + } else if arg == "--mobench-profile-warmup-only" + || arg == "--mobench-profile-warmup-only=1" { + warmupOnly = true + } + } + + NSLog( + "[BenchRunner] Profile launch options: delayMs=%llu, repeatUntilMs=%llu, resultHoldMs=%llu, warmupOnly=%@", + benchDelayMs, + repeatUntilMs, + resultHoldMs, + warmupOnly ? "true" : "false" + ) + + return ProfileLaunchOptions( + benchDelayMs: benchDelayMs, + resultHoldMs: resultHoldMs, + repeatUntilMs: repeatUntilMs, + warmupOnly: warmupOnly + ) + } +} + +@main +final class AppDelegate: UIResponder, UIApplicationDelegate { + var window: UIWindow? + + func application( + _ application: UIApplication, + didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? + ) -> Bool { + let window = UIWindow(frame: UIScreen.main.bounds) + window.rootViewController = BenchmarkViewController() + window.makeKeyAndVisible() + self.window = window + return true + } +} + +final class BenchmarkViewController: UIViewController { + private let reportView = UITextView() + private let completionLabel = UILabel() + private let jsonLabel = UILabel() + + override func viewDidLoad() { + super.viewDidLoad() + view.backgroundColor = .white + + reportView.translatesAutoresizingMaskIntoConstraints = false + reportView.isEditable = false + reportView.font = UIFont(name: "Menlo", size: 14) ?? UIFont.systemFont(ofSize: 14) + reportView.text = "Running benchmarks..." + reportView.accessibilityIdentifier = "benchmarkReport" + view.addSubview(reportView) + + completionLabel.translatesAutoresizingMaskIntoConstraints = false + completionLabel.text = "" + completionLabel.accessibilityIdentifier = "benchmarkCompleted" + completionLabel.isAccessibilityElement = false + completionLabel.textColor = .clear + view.addSubview(completionLabel) + + jsonLabel.translatesAutoresizingMaskIntoConstraints = false + jsonLabel.text = "" + jsonLabel.accessibilityIdentifier = "benchmarkReportJSON" + jsonLabel.isAccessibilityElement = true + jsonLabel.textColor = .clear + view.addSubview(jsonLabel) + + NSLayoutConstraint.activate([ + reportView.leadingAnchor.constraint(equalTo: view.leadingAnchor, constant: 16), + reportView.trailingAnchor.constraint(equalTo: view.trailingAnchor, constant: -16), + reportView.topAnchor.constraint(equalTo: view.topAnchor, constant: 16), + reportView.bottomAnchor.constraint(equalTo: view.bottomAnchor, constant: -16), + completionLabel.leadingAnchor.constraint(equalTo: view.leadingAnchor), + completionLabel.topAnchor.constraint(equalTo: view.topAnchor), + completionLabel.widthAnchor.constraint(equalToConstant: 1), + completionLabel.heightAnchor.constraint(equalToConstant: 1), + jsonLabel.leadingAnchor.constraint(equalTo: view.leadingAnchor), + jsonLabel.topAnchor.constraint(equalTo: completionLabel.bottomAnchor), + jsonLabel.widthAnchor.constraint(equalToConstant: 1), + jsonLabel.heightAnchor.constraint(equalToConstant: 1), + ]) + + runBenchmark() + } + + private func runBenchmark() { + DispatchQueue.global(qos: .userInitiated).async { + let options = ProfileLaunchOptions.resolved() + if options.benchDelayMs > 0 { + Thread.sleep(forTimeInterval: Double(options.benchDelayMs) / 1_000.0) + } + + let repeatDeadline = Date().addingTimeInterval( + Double(options.repeatUntilMs) / 1_000.0 + ) + var repeatedRuns = 1 + var result = {{PROJECT_NAME_PASCAL}}FFI.runCurrentBenchmark() + while !options.warmupOnly && options.repeatUntilMs > 0 && Date() < repeatDeadline { + result = {{PROJECT_NAME_PASCAL}}FFI.runCurrentBenchmark() + repeatedRuns += 1 + } + + DispatchQueue.main.async { + self.reportView.text = result.displayText + self.jsonLabel.text = result.jsonReport + self.jsonLabel.accessibilityLabel = result.jsonReport + self.completionLabel.text = "completed" + self.completionLabel.isAccessibilityElement = true + } + + NSLog("BENCH_REPORT_JSON_START") + NSLog("%@", result.jsonReport) + NSLog("BENCH_REPORT_JSON_END") + if repeatedRuns > 1 { + NSLog("Repeated benchmark %d time(s) during profile capture", repeatedRuns) + } + + if options.warmupOnly { + NSLog("Warmup-only profile run complete") + return + } + + NSLog("Displaying results for \(options.resultHoldMs) ms for capture output...") + Thread.sleep(forTimeInterval: Double(options.resultHoldMs) / 1_000.0) + NSLog("Display hold complete") + } + } +} diff --git a/templates/ios/BenchRunner/project.yml.template b/templates/ios/BenchRunner/project.yml.template index 7ccad1a..cdf35c5 100644 --- a/templates/ios/BenchRunner/project.yml.template +++ b/templates/ios/BenchRunner/project.yml.template @@ -8,7 +8,7 @@ targets: {{PROJECT_NAME_PASCAL}}: type: application platform: iOS - deploymentTarget: "15.0" + deploymentTarget: "{{IOS_DEPLOYMENT_TARGET}}" sources: - path: {{PROJECT_NAME_PASCAL}} resources: @@ -33,7 +33,7 @@ targets: {{PROJECT_NAME_PASCAL}}UITests: type: bundle.ui-testing platform: iOS - deploymentTarget: "15.0" + deploymentTarget: "{{IOS_DEPLOYMENT_TARGET}}" sources: - path: {{PROJECT_NAME_PASCAL}}UITests info: