From d655eb54fd44d3600a2401f16e4cec45159e04a2 Mon Sep 17 00:00:00 2001 From: AndreasIgel Date: Wed, 2 Sep 2026 21:06:28 +0200 Subject: [PATCH 01/11] Adding a script for performance analysis and having multiple runs --- .../scripts/run_full_comparison.py | 10 ++ .../scripts/run_stability_analysis.sh | 119 ++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100755 performance-test/scripts/run_stability_analysis.sh diff --git a/performance-test/scripts/run_full_comparison.py b/performance-test/scripts/run_full_comparison.py index daaea2fb..1ef9b8a1 100755 --- a/performance-test/scripts/run_full_comparison.py +++ b/performance-test/scripts/run_full_comparison.py @@ -95,6 +95,14 @@ def main() -> None: "avoiding the overhead of the processor's internal performance tracker. " "This ensures a fair comparison without measurement overhead.", ) + parser.add_argument( + "--label-suffix", + type=str, + default="", + help="Suffix appended to the label for each builder type " + "(e.g. --label-suffix stability gives 'sb-5runs-stability'). " + "Useful to distinguish different measurement campaigns.", + ) args = parser.parse_args() num_runs = args.runs @@ -104,6 +112,8 @@ def main() -> None: for bt in BUILDER_TYPES: label = f"{LABEL_PREFIX[bt]}-{num_runs}runs" + if args.label_suffix: + label += f"-{args.label_suffix}" print_section(f"{bt} (label: {label})") # 1. Generate classes diff --git a/performance-test/scripts/run_stability_analysis.sh b/performance-test/scripts/run_stability_analysis.sh new file mode 100755 index 00000000..cd3bec53 --- /dev/null +++ b/performance-test/scripts/run_stability_analysis.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env zsh +# Run stability analysis: 4 builder types × 3 run counts (5, 10, 15) +# Then compare each builder type across run counts to assess statistical stability. +# +# Usage: +# zsh performance-test/scripts/run_stability_analysis.sh +# +# Output directories: +# performance-reports/{sb,mb,rb,lombok}-{5,10,15}runs-stability/ +# +# Expected runtime: ~20-25 minutes + +set -euo pipefail + +echo "============================================================" +echo " Stability analysis started: $(date '+%Y-%m-%d %H:%M:%S')" +echo "============================================================" +echo + +START_TIME=$(date +%s) +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +BASE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$BASE_DIR/.." + +# --- macOS performance optimizations --- +# Prevent system sleep while the analysis is running +if command -v caffeinate &>/dev/null; then + caffeinate -dimsu -w $$ & +fi + +# Run the script and all children at elevated priority +renice -n -10 -p $$ 2>/dev/null || true + +echo "============================================================" +echo " STABILITY ANALYSIS: 4 builders × 3 run counts (5/10/15)" +echo "============================================================" +echo " macOS optimizations:" +echo " - System sleep disabled (caffeinate)" +echo " - Process priority elevated (renice -10)" +echo " For maximum performance, also consider:" +echo " - Close other CPU-intensive apps (browsers, IDEs)" +echo " - Ensure adequate cooling (laptop on hard surface)" +echo " - Plug in power adapter (prevents thermal throttling)" +echo "============================================================" +echo + +echo "============================================================" +echo " JVM / BUILD CONTEXT" +echo "============================================================" +JAVA_VERSION=$(java -version 2>&1 | head -n 1) +MAVEN_VERSION=$(mvn -version 2>&1 | head -n 1) +JVM_FLAGS=$(java -XX:+PrintFlagsFinal -version 2>/dev/null) +HEAP_MAX=$(echo "$JVM_FLAGS" | awk '/MaxHeapSize/ {printf "%.0f", $4/1024/1024; exit}') +HEAP_INIT=$(echo "$JVM_FLAGS" | awk '/InitialHeapSize/ {printf "%.0f", $4/1024/1024; exit}') +META_MAX=$(echo "$JVM_FLAGS" | awk '/MaxMetaspaceSize/ { + if ($4 == "18446744073709551615") { + print "unlimited" + } else { + printf "%.0fm", $4/1024/1024 + } + exit +}') +GC=$(java -XX:+PrintCommandLineFlags -version 2>&1 | tr ' ' '\n' | grep '^-XX:+Use.*GC' | sed -e 's/-XX:+Use//' -e 's/GC$//' | head -n 1) +echo " Java: $JAVA_VERSION" +echo " Maven: $MAVEN_VERSION" +echo " Heap: -Xms=${HEAP_INIT}m -Xmx=${HEAP_MAX}m" +echo " Metaspace: -XX:MaxMetaspaceSize=${META_MAX}" +echo " GC: ${GC:-default}" +echo "============================================================" +echo + +for RUNS in 5 10 15; do + echo "============================================================" + echo " Running all 4 builders with ${RUNS} runs (--no-tracking)" + echo "============================================================" + python3 performance-test/scripts/run_full_comparison.py \ + --runs "$RUNS" --no-tracking --label-suffix stability + echo +done + +echo "============================================================" +echo " STABILITY COMPARISONS (same builder across run counts)" +echo "============================================================" +echo + +echo "=== STABILITY: simple-builder (5 vs 10 vs 15) ===" +python3 performance-test/scripts/compare_performance.py \ + sb-5runs-stability/summary.json \ + sb-10runs-stability/summary.json \ + sb-15runs-stability/summary.json +echo + +echo "=== STABILITY: simple-minimal-builder (5 vs 10 vs 15) ===" +python3 performance-test/scripts/compare_performance.py \ + mb-5runs-stability/summary.json \ + mb-10runs-stability/summary.json \ + mb-15runs-stability/summary.json +echo + +echo "=== STABILITY: record-builder (5 vs 10 vs 15) ===" +python3 performance-test/scripts/compare_performance.py \ + rb-5runs-stability/summary.json \ + rb-10runs-stability/summary.json \ + rb-15runs-stability/summary.json +echo + +echo "=== STABILITY: lombok (5 vs 10 vs 15) ===" +python3 performance-test/scripts/compare_performance.py \ + lombok-5runs-stability/summary.json \ + lombok-10runs-stability/summary.json \ + lombok-15runs-stability/summary.json +echo + +echo "============================================================" +echo " Stability analysis complete." +echo " Started: $(date -r "$START_TIME" '+%Y-%m-%d %H:%M:%S')" +echo " Finished: $(date '+%Y-%m-%d %H:%M:%S')" +echo " Reports: performance-test/performance-reports/*-stability/" +echo "============================================================" From 1b7a59a6a1f281629b394062a39d170262d5c827 Mon Sep 17 00:00:00 2001 From: AndreasIgel Date: Wed, 2 Sep 2026 21:12:29 +0200 Subject: [PATCH 02/11] Removing stability analysis because this is not needed in future --- .../scripts/run_stability_analysis.sh | 119 ------------------ 1 file changed, 119 deletions(-) delete mode 100755 performance-test/scripts/run_stability_analysis.sh diff --git a/performance-test/scripts/run_stability_analysis.sh b/performance-test/scripts/run_stability_analysis.sh deleted file mode 100755 index cd3bec53..00000000 --- a/performance-test/scripts/run_stability_analysis.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env zsh -# Run stability analysis: 4 builder types × 3 run counts (5, 10, 15) -# Then compare each builder type across run counts to assess statistical stability. -# -# Usage: -# zsh performance-test/scripts/run_stability_analysis.sh -# -# Output directories: -# performance-reports/{sb,mb,rb,lombok}-{5,10,15}runs-stability/ -# -# Expected runtime: ~20-25 minutes - -set -euo pipefail - -echo "============================================================" -echo " Stability analysis started: $(date '+%Y-%m-%d %H:%M:%S')" -echo "============================================================" -echo - -START_TIME=$(date +%s) -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -BASE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" -cd "$BASE_DIR/.." - -# --- macOS performance optimizations --- -# Prevent system sleep while the analysis is running -if command -v caffeinate &>/dev/null; then - caffeinate -dimsu -w $$ & -fi - -# Run the script and all children at elevated priority -renice -n -10 -p $$ 2>/dev/null || true - -echo "============================================================" -echo " STABILITY ANALYSIS: 4 builders × 3 run counts (5/10/15)" -echo "============================================================" -echo " macOS optimizations:" -echo " - System sleep disabled (caffeinate)" -echo " - Process priority elevated (renice -10)" -echo " For maximum performance, also consider:" -echo " - Close other CPU-intensive apps (browsers, IDEs)" -echo " - Ensure adequate cooling (laptop on hard surface)" -echo " - Plug in power adapter (prevents thermal throttling)" -echo "============================================================" -echo - -echo "============================================================" -echo " JVM / BUILD CONTEXT" -echo "============================================================" -JAVA_VERSION=$(java -version 2>&1 | head -n 1) -MAVEN_VERSION=$(mvn -version 2>&1 | head -n 1) -JVM_FLAGS=$(java -XX:+PrintFlagsFinal -version 2>/dev/null) -HEAP_MAX=$(echo "$JVM_FLAGS" | awk '/MaxHeapSize/ {printf "%.0f", $4/1024/1024; exit}') -HEAP_INIT=$(echo "$JVM_FLAGS" | awk '/InitialHeapSize/ {printf "%.0f", $4/1024/1024; exit}') -META_MAX=$(echo "$JVM_FLAGS" | awk '/MaxMetaspaceSize/ { - if ($4 == "18446744073709551615") { - print "unlimited" - } else { - printf "%.0fm", $4/1024/1024 - } - exit -}') -GC=$(java -XX:+PrintCommandLineFlags -version 2>&1 | tr ' ' '\n' | grep '^-XX:+Use.*GC' | sed -e 's/-XX:+Use//' -e 's/GC$//' | head -n 1) -echo " Java: $JAVA_VERSION" -echo " Maven: $MAVEN_VERSION" -echo " Heap: -Xms=${HEAP_INIT}m -Xmx=${HEAP_MAX}m" -echo " Metaspace: -XX:MaxMetaspaceSize=${META_MAX}" -echo " GC: ${GC:-default}" -echo "============================================================" -echo - -for RUNS in 5 10 15; do - echo "============================================================" - echo " Running all 4 builders with ${RUNS} runs (--no-tracking)" - echo "============================================================" - python3 performance-test/scripts/run_full_comparison.py \ - --runs "$RUNS" --no-tracking --label-suffix stability - echo -done - -echo "============================================================" -echo " STABILITY COMPARISONS (same builder across run counts)" -echo "============================================================" -echo - -echo "=== STABILITY: simple-builder (5 vs 10 vs 15) ===" -python3 performance-test/scripts/compare_performance.py \ - sb-5runs-stability/summary.json \ - sb-10runs-stability/summary.json \ - sb-15runs-stability/summary.json -echo - -echo "=== STABILITY: simple-minimal-builder (5 vs 10 vs 15) ===" -python3 performance-test/scripts/compare_performance.py \ - mb-5runs-stability/summary.json \ - mb-10runs-stability/summary.json \ - mb-15runs-stability/summary.json -echo - -echo "=== STABILITY: record-builder (5 vs 10 vs 15) ===" -python3 performance-test/scripts/compare_performance.py \ - rb-5runs-stability/summary.json \ - rb-10runs-stability/summary.json \ - rb-15runs-stability/summary.json -echo - -echo "=== STABILITY: lombok (5 vs 10 vs 15) ===" -python3 performance-test/scripts/compare_performance.py \ - lombok-5runs-stability/summary.json \ - lombok-10runs-stability/summary.json \ - lombok-15runs-stability/summary.json -echo - -echo "============================================================" -echo " Stability analysis complete." -echo " Started: $(date -r "$START_TIME" '+%Y-%m-%d %H:%M:%S')" -echo " Finished: $(date '+%Y-%m-%d %H:%M:%S')" -echo " Reports: performance-test/performance-reports/*-stability/" -echo "============================================================" From 90dc3abaa1335582ae975deff30a0acd594419db Mon Sep 17 00:00:00 2001 From: AndreasIgel Date: Sun, 6 Sep 2026 20:50:54 +0200 Subject: [PATCH 03/11] Adding a check for expected annotations on generated code so that analysis runs could not be done on different code-geration-runs before --- .../scripts/run_performance_measurement.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/performance-test/scripts/run_performance_measurement.py b/performance-test/scripts/run_performance_measurement.py index a691302b..96afde70 100644 --- a/performance-test/scripts/run_performance_measurement.py +++ b/performance-test/scripts/run_performance_measurement.py @@ -454,6 +454,34 @@ def main() -> None: print(f"Report directory: {report_dir}") print() + # Pre-flight check: verify generated sources use the expected annotation. + # If generate_classes.py was run for a different builder type, the Maven + # profile won't match and compilation will fail or produce wrong results. + expected_annotation = BUILDER_TYPE_ANNOTATION.get(builder_type) + if expected_annotation: + src_dir = BASE_DIR / "src" / "main" / "java" + if src_dir.exists(): + actual = None + for f in src_dir.rglob("*.java"): + try: + text = f.read_text() + except OSError: + continue + for ann in BUILDER_TYPE_ANNOTATION.values(): + if ann in text: + actual = ann + break + if actual is not None: + break + if actual is not None and actual != expected_annotation: + print(f"ERROR: Generated sources use {actual} but --builder-type is " + f"{builder_type} (expects {expected_annotation}).") + print(f"Run generate_classes.py first: " + f"python3 scripts/generate_classes.py --builder-type {builder_type} --force") + sys.exit(1) + + + max_retries = args.max_retries runs: list[dict] = [] total_attempts = 0 From 925938f0938d36cad4c63713ecb17da4c87f3f16 Mon Sep 17 00:00:00 2001 From: AndreasIgel Date: Sun, 6 Sep 2026 21:28:50 +0200 Subject: [PATCH 04/11] Adding first version of performance analysis result --- README.md | 2 + performance-test/docs/PERFORMANCE_ANALYSIS.md | 78 +++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/README.md b/README.md index 39902af9..13f18bfd 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,8 @@ Simple Builders generates fluent, type-safe builders for your **existing** class Use Simple Builders when you want fluent, type-safe builders for the classes and records you already have, generated as plain readable source, with no bytecode manipulation and no IDE plugin — and no lock-in: because the builders are ordinary generated Java, you can drop the dependency at any time by copying the generated builder classes into your own sources, and they keep working. +For performance benchmark results comparing Simple Builders, Simple Minimal Builder, RecordBuilder, and Lombok, see the [Performance Analysis Guide](performance-test/docs/PERFORMANCE_ANALYSIS.md#benchmark-results). + ### Doing what other builders advertise — the Simple Builders way - **Required fields:** Primitive fields and fields annotated with an annotation named `NotNull` or `NonNull` are non-nullable; constructor parameters are builder inputs. `build()` enforces the required/non-null contract with `IllegalStateException` ([configuration details](#required-fields-and-null-safety)). diff --git a/performance-test/docs/PERFORMANCE_ANALYSIS.md b/performance-test/docs/PERFORMANCE_ANALYSIS.md index 985c2e50..ac16491a 100644 --- a/performance-test/docs/PERFORMANCE_ANALYSIS.md +++ b/performance-test/docs/PERFORMANCE_ANALYSIS.md @@ -127,3 +127,81 @@ performance-test/ "wallTimeOnly": true } ``` + +## Benchmark Results + +The following results are from a 10-iteration stability run using +[`run_full_comparison.py`](../scripts/run_full_comparison.py) +`--runs 10 --no-tracking --label-suffix stability`. +All frameworks were measured with wall-time only (no JSON tracking overhead) to +ensure a fair comparison. Note that the builder counts differ: Simple Builders +and Lombok generate one builder per class, while RecordBuilder generates fewer +(295) because the test dataset contains fewer records than plain classes. + +### Wall-Time Comparison (10 runs, wall-time only) + +| Framework | Builders | Wall Avg (s) | Wall Min (s) | Wall Max (s) | Per Builder (ms) | +|-----------|----------|-------------|-------------|-------------|-----------------| +| Simple Builder (`@SimpleBuilder`) | 1079 | 53.1 | 52.8 | 53.4 | 48.0 | +| Simple Minimal Builder (`@SimpleMinimalBuilder`) | 1079 | 22.0 | 21.5 | 22.4 | 19.3 | +| RecordBuilder (`@RecordBuilder`) | 295 | 6.6 | 6.5 | 6.7 | 18.5 | +| Lombok (`@Builder`) | 1077 | 6.9 | 6.7 | 7.1 | 5.3 | + +Key observations: + +- **Lombok** is fastest per builder but instruments bytecode at compile time + rather than generating separate source files, so the comparison is not + apples-to-apples. +- **RecordBuilder** generates far fewer builders (295 vs 1079) because the test + dataset contains fewer records than plain classes. However, its per-builder + cost (~18.5 ms) is nearly identical to Simple Minimal Builder (~19.3 ms) — the + wall-time difference is almost entirely due to the lower builder count, not + per-builder efficiency. +- **Simple Minimal Builder** is ~2.4x faster than Simple Builder. The speedup + comes from two factors: fewer generated methods (no collection helpers, + conditional logic, supplier/consumer setters, Javadoc, etc.) and + correspondingly less source formatting work. +- **Simple Builder** is the slowest due to its full feature set. The per-builder + cost (~48 ms) is dominated by code generation and formatting. + +### Processor-Internal Breakdown (with JSON tracking) + +For deeper insight into where time is spent inside the Simple Builders +processor, enable the processor's internal performance tracker with +`-Asimplebuilder.performanceTracking=true`. This adds minor overhead but +provides phase-level breakdowns in the JSON report. + +A representative 10-run measurement of Simple Builder (full features, JDT +formatting) shows the following phase distribution: + +| Phase | Avg (s) | Share | +|-------|---------|-------| +| Config Resolution | 0.08 | <1% | +| Builder Def Extraction | 0.73 | ~2% | +| DTO Mapping | 0.03 | <1% | +| Code Generation | 42.3 | ~93% | +| **Processor Total** | **45.3** | | +| **Wall Total** | **54.0** | | + +Code generation dominates at ~93% of processor time. This includes Roaster +source construction and source formatting. A 5-run comparison of the three +formatting modes (Simple Builder, 1079 builders) shows the following: + +| Formatting Mode | Wall Avg (s) | Per Builder (ms) | vs NONE | +|-----------------|-------------|-----------------|---------| +| NONE (raw Roaster) | 74.8 | 67.6 | — | +| LIGHTWEIGHT | 78.5 | 71.1 | +5% | +| JDT (default) | 81.8 | 74.1 | +9% | + +Formatting adds ~7s total (~6.5 ms/builder) for JDT over NONE. The lightweight +formatter adds about half that cost. While measurable, formatting is a +secondary factor compared to the difference in generated method count between +Simple Builder and Simple Minimal Builder (~31s), which is driven primarily by +the number of methods and collection helpers generated. + +### Running the Benchmarks + +To reproduce the wall-time comparison, see [Quick Start](#quick-start) above. +For processor-internal breakdowns, run +[`run_performance_measurement.py`](../scripts/run_performance_measurement.py) +without `--no-tracking` to enable JSON reporting. From 2701147853733c52a2a206838b0b99ab9280e1c7 Mon Sep 17 00:00:00 2001 From: AndreasIgel Date: Sun, 6 Sep 2026 21:29:40 +0200 Subject: [PATCH 05/11] Revert "Removing stability analysis because this is not needed in future" This reverts commit 1b7a59a6a1f281629b394062a39d170262d5c827. --- .../scripts/run_stability_analysis.sh | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100755 performance-test/scripts/run_stability_analysis.sh diff --git a/performance-test/scripts/run_stability_analysis.sh b/performance-test/scripts/run_stability_analysis.sh new file mode 100755 index 00000000..cd3bec53 --- /dev/null +++ b/performance-test/scripts/run_stability_analysis.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env zsh +# Run stability analysis: 4 builder types × 3 run counts (5, 10, 15) +# Then compare each builder type across run counts to assess statistical stability. +# +# Usage: +# zsh performance-test/scripts/run_stability_analysis.sh +# +# Output directories: +# performance-reports/{sb,mb,rb,lombok}-{5,10,15}runs-stability/ +# +# Expected runtime: ~20-25 minutes + +set -euo pipefail + +echo "============================================================" +echo " Stability analysis started: $(date '+%Y-%m-%d %H:%M:%S')" +echo "============================================================" +echo + +START_TIME=$(date +%s) +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +BASE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$BASE_DIR/.." + +# --- macOS performance optimizations --- +# Prevent system sleep while the analysis is running +if command -v caffeinate &>/dev/null; then + caffeinate -dimsu -w $$ & +fi + +# Run the script and all children at elevated priority +renice -n -10 -p $$ 2>/dev/null || true + +echo "============================================================" +echo " STABILITY ANALYSIS: 4 builders × 3 run counts (5/10/15)" +echo "============================================================" +echo " macOS optimizations:" +echo " - System sleep disabled (caffeinate)" +echo " - Process priority elevated (renice -10)" +echo " For maximum performance, also consider:" +echo " - Close other CPU-intensive apps (browsers, IDEs)" +echo " - Ensure adequate cooling (laptop on hard surface)" +echo " - Plug in power adapter (prevents thermal throttling)" +echo "============================================================" +echo + +echo "============================================================" +echo " JVM / BUILD CONTEXT" +echo "============================================================" +JAVA_VERSION=$(java -version 2>&1 | head -n 1) +MAVEN_VERSION=$(mvn -version 2>&1 | head -n 1) +JVM_FLAGS=$(java -XX:+PrintFlagsFinal -version 2>/dev/null) +HEAP_MAX=$(echo "$JVM_FLAGS" | awk '/MaxHeapSize/ {printf "%.0f", $4/1024/1024; exit}') +HEAP_INIT=$(echo "$JVM_FLAGS" | awk '/InitialHeapSize/ {printf "%.0f", $4/1024/1024; exit}') +META_MAX=$(echo "$JVM_FLAGS" | awk '/MaxMetaspaceSize/ { + if ($4 == "18446744073709551615") { + print "unlimited" + } else { + printf "%.0fm", $4/1024/1024 + } + exit +}') +GC=$(java -XX:+PrintCommandLineFlags -version 2>&1 | tr ' ' '\n' | grep '^-XX:+Use.*GC' | sed -e 's/-XX:+Use//' -e 's/GC$//' | head -n 1) +echo " Java: $JAVA_VERSION" +echo " Maven: $MAVEN_VERSION" +echo " Heap: -Xms=${HEAP_INIT}m -Xmx=${HEAP_MAX}m" +echo " Metaspace: -XX:MaxMetaspaceSize=${META_MAX}" +echo " GC: ${GC:-default}" +echo "============================================================" +echo + +for RUNS in 5 10 15; do + echo "============================================================" + echo " Running all 4 builders with ${RUNS} runs (--no-tracking)" + echo "============================================================" + python3 performance-test/scripts/run_full_comparison.py \ + --runs "$RUNS" --no-tracking --label-suffix stability + echo +done + +echo "============================================================" +echo " STABILITY COMPARISONS (same builder across run counts)" +echo "============================================================" +echo + +echo "=== STABILITY: simple-builder (5 vs 10 vs 15) ===" +python3 performance-test/scripts/compare_performance.py \ + sb-5runs-stability/summary.json \ + sb-10runs-stability/summary.json \ + sb-15runs-stability/summary.json +echo + +echo "=== STABILITY: simple-minimal-builder (5 vs 10 vs 15) ===" +python3 performance-test/scripts/compare_performance.py \ + mb-5runs-stability/summary.json \ + mb-10runs-stability/summary.json \ + mb-15runs-stability/summary.json +echo + +echo "=== STABILITY: record-builder (5 vs 10 vs 15) ===" +python3 performance-test/scripts/compare_performance.py \ + rb-5runs-stability/summary.json \ + rb-10runs-stability/summary.json \ + rb-15runs-stability/summary.json +echo + +echo "=== STABILITY: lombok (5 vs 10 vs 15) ===" +python3 performance-test/scripts/compare_performance.py \ + lombok-5runs-stability/summary.json \ + lombok-10runs-stability/summary.json \ + lombok-15runs-stability/summary.json +echo + +echo "============================================================" +echo " Stability analysis complete." +echo " Started: $(date -r "$START_TIME" '+%Y-%m-%d %H:%M:%S')" +echo " Finished: $(date '+%Y-%m-%d %H:%M:%S')" +echo " Reports: performance-test/performance-reports/*-stability/" +echo "============================================================" From 98690705218c33a778bfcc7d01c6576b341a00e0 Mon Sep 17 00:00:00 2001 From: AndreasIgel Date: Sun, 6 Sep 2026 22:00:44 +0200 Subject: [PATCH 06/11] Adding possibility to define formatting-mode when running performance-measurement --- .../scripts/run_performance_measurement.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/performance-test/scripts/run_performance_measurement.py b/performance-test/scripts/run_performance_measurement.py index 96afde70..b2142ad0 100644 --- a/performance-test/scripts/run_performance_measurement.py +++ b/performance-test/scripts/run_performance_measurement.py @@ -134,7 +134,7 @@ def parse_compiler_time(output: str) -> Optional[float]: def run_one(run_index: int, profile: str, is_simple_builders: bool, report_dir: Path, - builder_type: str = "") -> Optional[dict]: + builder_type: str = "", formatting_mode: str = "") -> Optional[dict]: """Run a single clean compile and return the parsed JSON report (or wall-time-only dict). The report file uses the run_index in its name so that retries overwrite the failed @@ -158,6 +158,8 @@ def run_one(run_index: int, profile: str, is_simple_builders: bool, report_dir: "-Dsimplebuilder.performanceTracking=true", f"-Dsimplebuilder.performanceOutputFile={report_file}", ]) + if formatting_mode: + cmd.append(f"-Dsimplebuilder.formattingMode={formatting_mode}") result = subprocess.run( cmd, @@ -432,6 +434,15 @@ def main() -> None: "lombok/record-builder), avoiding the overhead of the processor's " "internal performance tracker.", ) + parser.add_argument( + "--formatting-mode", + type=str, + default="", + choices=["", "jdt", "lightweight", "none"], + help="Override the formatting mode for simple-builders types via " + "-Asimplebuilder.formattingMode. Only affects simple-builder and " + "simple-minimal-builder. Default: empty (use default from profile or JDT, if not defined in profile).", + ) args = parser.parse_args() num_runs = args.runs @@ -494,7 +505,8 @@ def main() -> None: if attempt > 1: print(f" Run {i}: retry {attempt - 1}/{max_retries}...", flush=True) run_start = time.time() - data = run_one(i, profile, use_tracking, report_dir, builder_type) + data = run_one(i, profile, use_tracking, report_dir, builder_type, + args.formatting_mode) if data is not None: if "_wallTimeSeconds" not in data: data["_wallTimeSeconds"] = time.time() - run_start From c5d4016e76ec3ca45f298504ddc511329474f7c8 Mon Sep 17 00:00:00 2001 From: AndreasIgel Date: Sat, 12 Sep 2026 12:32:02 +0200 Subject: [PATCH 07/11] Fixing bugs in run_performance_measurements --- .../scripts/run_performance_measurement.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/performance-test/scripts/run_performance_measurement.py b/performance-test/scripts/run_performance_measurement.py index b2142ad0..c62db777 100644 --- a/performance-test/scripts/run_performance_measurement.py +++ b/performance-test/scripts/run_performance_measurement.py @@ -133,7 +133,7 @@ def parse_compiler_time(output: str) -> Optional[float]: return None -def run_one(run_index: int, profile: str, is_simple_builders: bool, report_dir: Path, +def run_one(run_index: int, profile: str, use_tracking: bool, report_dir: Path, builder_type: str = "", formatting_mode: str = "") -> Optional[dict]: """Run a single clean compile and return the parsed JSON report (or wall-time-only dict). @@ -153,13 +153,14 @@ def run_one(run_index: int, profile: str, is_simple_builders: bool, report_dir: "-Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS", "--no-transfer-progress", ] - if is_simple_builders: + is_simple_builders = builder_type in SIMPLE_BUILDERS_TYPES + if use_tracking: cmd.extend([ "-Dsimplebuilder.performanceTracking=true", f"-Dsimplebuilder.performanceOutputFile={report_file}", ]) - if formatting_mode: - cmd.append(f"-Dsimplebuilder.formattingMode={formatting_mode}") + if is_simple_builders and formatting_mode: + cmd.append(f"-Dsimplebuilder.formattingMode={formatting_mode}") result = subprocess.run( cmd, @@ -182,7 +183,7 @@ def run_one(run_index: int, profile: str, is_simple_builders: bool, report_dir: builder_count = count_annotated_sources(annotation) compiler_time = parse_compiler_time(result.stdout + result.stderr) - if is_simple_builders: + if use_tracking: if not report_file.exists(): print(f" Run {run_index}: compiled OK but no JSON report found ({elapsed:.1f}s)") return None From 2ce8de8b24a047fcfdbcbc6e6a100fafb5aa6f44 Mon Sep 17 00:00:00 2001 From: AndreasIgel Date: Sat, 12 Sep 2026 12:33:22 +0200 Subject: [PATCH 08/11] adding script for running a full analysis with comparing the results of all builder types --- performance-test/scripts/run_full_analysis.sh | 136 ++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100755 performance-test/scripts/run_full_analysis.sh diff --git a/performance-test/scripts/run_full_analysis.sh b/performance-test/scripts/run_full_analysis.sh new file mode 100755 index 00000000..8fd04e46 --- /dev/null +++ b/performance-test/scripts/run_full_analysis.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env zsh +# Run the full performance analysis for simple-builders. +# +# This is the single entry point referenced in PERFORMANCE_ANALYSIS.md. +# It runs two analyses: +# +# 1. Cross-framework comparison (wall-time only): +# simple-builder, simple-minimal-builder, record-builder, lombok +# All measured with --no-tracking for a fair wall-time comparison. +# +# 2. Formatting-mode analysis (with processor metrics): +# simple-builder with jdt, lightweight, and none formatting modes. +# Tracking is enabled so JSON processor metrics are available for +# deeper insight into formatting overhead. +# +# Usage: +# zsh performance-test/scripts/run_full_analysis.sh +# RUNS=5 zsh performance-test/scripts/run_full_analysis.sh # override run count (default: 10) +# +# Output directories: +# performance-reports/{sb,mb,rb,lombok}-{N}runs/ +# performance-reports/fmt-{jdt,lightweight,none}-{N}runs/ +# +# Expected runtime: ~30-40 minutes (depends on RUNS and hardware) + +set -euo pipefail + +RUNS="${RUNS:-10}" + +echo "============================================================" +echo " FULL PERFORMANCE ANALYSIS started: $(date '+%Y-%m-%d %H:%M:%S')" +echo " Runs per measurement: ${RUNS}" +echo "============================================================" +echo + +START_TIME=$(date +%s) +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +BASE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$BASE_DIR/.." + +# --- macOS performance optimizations --- +# Prevent system sleep while the analysis is running +if command -v caffeinate &>/dev/null; then + caffeinate -dimsu -w $$ & +fi + +# Run the script and all children at elevated priority +renice -n -10 -p $$ 2>/dev/null || true + +echo "============================================================" +echo " JVM / BUILD CONTEXT" +echo "============================================================" +JAVA_VERSION=$(java -version 2>&1 | head -n 1) +MAVEN_VERSION=$(mvn -version 2>&1 | head -n 1) +JVM_FLAGS=$(java -XX:+PrintFlagsFinal -version 2>/dev/null) +HEAP_MAX=$(echo "$JVM_FLAGS" | awk '/MaxHeapSize/ {printf "%.0f", $4/1024/1024; exit}') +HEAP_INIT=$(echo "$JVM_FLAGS" | awk '/InitialHeapSize/ {printf "%.0f", $4/1024/1024; exit}') +META_MAX=$(echo "$JVM_FLAGS" | awk '/MaxMetaspaceSize/ { + if ($4 == "18446744073709551615") { + print "unlimited" + } else { + printf "%.0fm", $4/1024/1024 + } + exit +}') +GC=$(java -XX:+PrintCommandLineFlags -version 2>&1 | tr ' ' '\n' | grep '^-XX:+Use.*GC' | sed -e 's/-XX:+Use//' -e 's/GC$//' | head -n 1) +echo " Java: $JAVA_VERSION" +echo " Maven: $MAVEN_VERSION" +echo " Heap: -Xms=${HEAP_INIT}m -Xmx=${HEAP_MAX}m" +echo " Metaspace: -XX:MaxMetaspaceSize=${META_MAX}" +echo " GC: ${GC:-default}" +echo "============================================================" +echo + +# ============================================================ +# Part 1: Cross-framework comparison (wall-time only) +# ============================================================ +echo "============================================================" +echo " PART 1: Cross-framework comparison (${RUNS} runs, --no-tracking)" +echo " Builders: simple-builder, simple-minimal-builder, record-builder, lombok" +echo "============================================================" +echo + +python3 performance-test/scripts/run_full_comparison.py \ + --runs "$RUNS" --no-tracking +echo + +# ============================================================ +# Part 2: Formatting-mode analysis (with processor metrics) +# ============================================================ +echo "============================================================" +echo " PART 2: Formatting-mode analysis (${RUNS} runs, with tracking)" +echo " Builder: simple-builder" +echo " Modes: jdt, lightweight, none" +echo "============================================================" +echo + +# Generate simple-builder sources once for all formatting-mode runs +python3 performance-test/scripts/generate_classes.py \ + --builder-type simple-builder --force +echo + +for MODE in jdt lightweight none; do + echo "============================================================" + echo " simple-builder, formattingMode=${MODE}, ${RUNS} runs (with tracking)" + echo "============================================================" + python3 performance-test/scripts/run_performance_measurement.py \ + --runs "$RUNS" \ + --label "fmt-${MODE}-${RUNS}runs" \ + --builder-type simple-builder \ + --formatting-mode "$MODE" + echo +done + +echo "============================================================" +echo " FORMATTING-MODE COMPARISON (with processor metrics)" +echo "============================================================" +echo +echo "=== FORMATTING MODES: jdt vs lightweight vs none ===" +python3 performance-test/scripts/compare_performance.py \ + "fmt-jdt-${RUNS}runs/summary.json" \ + "fmt-lightweight-${RUNS}runs/summary.json" \ + "fmt-none-${RUNS}runs/summary.json" +echo + +# ============================================================ +# Summary +# ============================================================ +echo "============================================================" +echo " Full performance analysis complete." +echo " Started: $(date -r "$START_TIME" '+%Y-%m-%d %H:%M:%S')" +echo " Finished: $(date '+%Y-%m-%d %H:%M:%S')" +echo " Reports:" +echo " Cross-framework: performance-test/performance-reports/{sb,mb,rb,lombok}-${RUNS}runs/" +echo " Formatting-mode: performance-test/performance-reports/fmt-{jdt,lightweight,none}-${RUNS}runs/" +echo "============================================================" From 6b43a79b379856c4ec64fedf66cbf1dc7a266779 Mon Sep 17 00:00:00 2001 From: AndreasIgel Date: Sat, 12 Sep 2026 12:33:44 +0200 Subject: [PATCH 09/11] Removing stability analysis because this is not needed anymore --- .../scripts/run_stability_analysis.sh | 119 ------------------ 1 file changed, 119 deletions(-) delete mode 100755 performance-test/scripts/run_stability_analysis.sh diff --git a/performance-test/scripts/run_stability_analysis.sh b/performance-test/scripts/run_stability_analysis.sh deleted file mode 100755 index cd3bec53..00000000 --- a/performance-test/scripts/run_stability_analysis.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env zsh -# Run stability analysis: 4 builder types × 3 run counts (5, 10, 15) -# Then compare each builder type across run counts to assess statistical stability. -# -# Usage: -# zsh performance-test/scripts/run_stability_analysis.sh -# -# Output directories: -# performance-reports/{sb,mb,rb,lombok}-{5,10,15}runs-stability/ -# -# Expected runtime: ~20-25 minutes - -set -euo pipefail - -echo "============================================================" -echo " Stability analysis started: $(date '+%Y-%m-%d %H:%M:%S')" -echo "============================================================" -echo - -START_TIME=$(date +%s) -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -BASE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" -cd "$BASE_DIR/.." - -# --- macOS performance optimizations --- -# Prevent system sleep while the analysis is running -if command -v caffeinate &>/dev/null; then - caffeinate -dimsu -w $$ & -fi - -# Run the script and all children at elevated priority -renice -n -10 -p $$ 2>/dev/null || true - -echo "============================================================" -echo " STABILITY ANALYSIS: 4 builders × 3 run counts (5/10/15)" -echo "============================================================" -echo " macOS optimizations:" -echo " - System sleep disabled (caffeinate)" -echo " - Process priority elevated (renice -10)" -echo " For maximum performance, also consider:" -echo " - Close other CPU-intensive apps (browsers, IDEs)" -echo " - Ensure adequate cooling (laptop on hard surface)" -echo " - Plug in power adapter (prevents thermal throttling)" -echo "============================================================" -echo - -echo "============================================================" -echo " JVM / BUILD CONTEXT" -echo "============================================================" -JAVA_VERSION=$(java -version 2>&1 | head -n 1) -MAVEN_VERSION=$(mvn -version 2>&1 | head -n 1) -JVM_FLAGS=$(java -XX:+PrintFlagsFinal -version 2>/dev/null) -HEAP_MAX=$(echo "$JVM_FLAGS" | awk '/MaxHeapSize/ {printf "%.0f", $4/1024/1024; exit}') -HEAP_INIT=$(echo "$JVM_FLAGS" | awk '/InitialHeapSize/ {printf "%.0f", $4/1024/1024; exit}') -META_MAX=$(echo "$JVM_FLAGS" | awk '/MaxMetaspaceSize/ { - if ($4 == "18446744073709551615") { - print "unlimited" - } else { - printf "%.0fm", $4/1024/1024 - } - exit -}') -GC=$(java -XX:+PrintCommandLineFlags -version 2>&1 | tr ' ' '\n' | grep '^-XX:+Use.*GC' | sed -e 's/-XX:+Use//' -e 's/GC$//' | head -n 1) -echo " Java: $JAVA_VERSION" -echo " Maven: $MAVEN_VERSION" -echo " Heap: -Xms=${HEAP_INIT}m -Xmx=${HEAP_MAX}m" -echo " Metaspace: -XX:MaxMetaspaceSize=${META_MAX}" -echo " GC: ${GC:-default}" -echo "============================================================" -echo - -for RUNS in 5 10 15; do - echo "============================================================" - echo " Running all 4 builders with ${RUNS} runs (--no-tracking)" - echo "============================================================" - python3 performance-test/scripts/run_full_comparison.py \ - --runs "$RUNS" --no-tracking --label-suffix stability - echo -done - -echo "============================================================" -echo " STABILITY COMPARISONS (same builder across run counts)" -echo "============================================================" -echo - -echo "=== STABILITY: simple-builder (5 vs 10 vs 15) ===" -python3 performance-test/scripts/compare_performance.py \ - sb-5runs-stability/summary.json \ - sb-10runs-stability/summary.json \ - sb-15runs-stability/summary.json -echo - -echo "=== STABILITY: simple-minimal-builder (5 vs 10 vs 15) ===" -python3 performance-test/scripts/compare_performance.py \ - mb-5runs-stability/summary.json \ - mb-10runs-stability/summary.json \ - mb-15runs-stability/summary.json -echo - -echo "=== STABILITY: record-builder (5 vs 10 vs 15) ===" -python3 performance-test/scripts/compare_performance.py \ - rb-5runs-stability/summary.json \ - rb-10runs-stability/summary.json \ - rb-15runs-stability/summary.json -echo - -echo "=== STABILITY: lombok (5 vs 10 vs 15) ===" -python3 performance-test/scripts/compare_performance.py \ - lombok-5runs-stability/summary.json \ - lombok-10runs-stability/summary.json \ - lombok-15runs-stability/summary.json -echo - -echo "============================================================" -echo " Stability analysis complete." -echo " Started: $(date -r "$START_TIME" '+%Y-%m-%d %H:%M:%S')" -echo " Finished: $(date '+%Y-%m-%d %H:%M:%S')" -echo " Reports: performance-test/performance-reports/*-stability/" -echo "============================================================" From dca01944e7b4d1b4192c174952f7bc4f95476ae2 Mon Sep 17 00:00:00 2001 From: AndreasIgel Date: Sat, 12 Sep 2026 14:31:47 +0200 Subject: [PATCH 10/11] Updating performance analysis documentation --- performance-test/docs/PERFORMANCE_ANALYSIS.md | 83 ++++++++++++------- 1 file changed, 52 insertions(+), 31 deletions(-) diff --git a/performance-test/docs/PERFORMANCE_ANALYSIS.md b/performance-test/docs/PERFORMANCE_ANALYSIS.md index ac16491a..41417337 100644 --- a/performance-test/docs/PERFORMANCE_ANALYSIS.md +++ b/performance-test/docs/PERFORMANCE_ANALYSIS.md @@ -15,6 +15,7 @@ processor to measure processing time. Three scripts work together: | `run_performance_measurement.py` | Run N compilations and aggregate timing results | | `compare_performance.py` | Compare results from multiple measurement runs side-by-side | | `run_full_comparison.py` | Run all frameworks end-to-end and compare (convenience) | +| `run_full_analysis.sh` | Full analysis: cross-framework comparison plus formatting-mode analysis (recommended entry point) | ## Supported Builder Types @@ -30,7 +31,19 @@ metrics. Builder types without JSON reports only measure overall wall time. ## Quick Start -Run all four frameworks with N runs each, then compare: +For the complete analysis (cross-framework comparison plus formatting-mode +breakdown), run the top-level script: + +```bash +./scripts/run_full_analysis.sh +``` + +It runs all four frameworks with wall-time only, then runs simple-builder with +each formatting mode (`jdt`, `lightweight`, `none`) and JSON tracking enabled, +and finally prints comparisons. `RUNS=5 ./scripts/run_full_analysis.sh` +overrides the default of 10 runs per measurement. + +To run only the cross-framework comparison: ```bash python3 scripts/run_full_comparison.py --runs 10 @@ -56,6 +69,10 @@ python3 scripts/generate_classes.py --builder-type simple-builder --force python3 scripts/run_performance_measurement.py --runs 30 --label sb-30runs --builder-type simple-builder ``` +`run_performance_measurement.py` also accepts `--formatting-mode ` +to control source formatting for simple-builders types, and `--no-tracking` to +skip JSON processor metrics (wall-time only). + Results are written to `performance-test/performance-reports/