|
3 | 3 | //! |
4 | 4 | //! Run cargo run --bin benchmark --release to compile and run the benchmark. |
5 | 5 | //! You can use it in a shell script to compare performance between two git revisions. |
| 6 | +//! To profile the CPU usage of the benchmark, run: |
| 7 | +//! |
| 8 | +//! ```sh |
| 9 | +//! cargo build --profile profiling --bin benchmark |
| 10 | +//! samply record target/profiling/benchmark --profile-long |
| 11 | +//! ``` |
6 | 12 | //! |
7 | 13 | //! For example, to compare between this commit and the previous one: |
8 | 14 | //! |
|
14 | 20 | //! git checkout - |
15 | 21 | //! ``` |
16 | 22 | use gdscript_formatter::{FormatterConfiguration, RenderElement, format_gdscript_with_buffers}; |
17 | | -use std::{fs, time::Instant}; |
| 23 | +use std::{ |
| 24 | + env, fs, |
| 25 | + hint::black_box, |
| 26 | + time::{Duration, Instant}, |
| 27 | +}; |
18 | 28 |
|
19 | | -const ITERATIONS: u16 = 40; |
20 | | - |
21 | | -fn main() -> Result<(), Box<dyn std::error::Error>> { |
22 | | - let short_content = fs::read_to_string("benchmarks/gdscript_files/short.gd")?; |
23 | | - let long_content = fs::read_to_string("benchmarks/gdscript_files/long.gd")?; |
24 | | - let config = FormatterConfiguration::default(); |
| 29 | +const PROFILE_ITERATIONS: usize = 5_000; |
| 30 | +const WARMUP_DURATION: Duration = Duration::from_millis(200); |
| 31 | +const SAMPLE_DURATION: Duration = Duration::from_millis(200); |
| 32 | +const SAMPLE_COUNT: usize = 10; |
25 | 33 |
|
26 | | - println!("Running GDScript Formatter Benchmark..."); |
| 34 | +/// A struct that stores data and has functions to benchmark the GDScript |
| 35 | +/// formatter and measure performance. |
| 36 | +struct BenchmarkRunner { |
| 37 | + render_elements: Vec<RenderElement>, |
| 38 | + output: String, |
| 39 | +} |
27 | 40 |
|
28 | | - let mut render_elements: Vec<RenderElement> = Vec::new(); |
29 | | - let mut output = String::new(); |
| 41 | +struct BenchmarkMeasurement { |
| 42 | + median_seconds_per_iteration: f64, |
| 43 | + total_iterations: usize, |
| 44 | +} |
30 | 45 |
|
31 | | - println!("Running short file warmup (10 iterations)"); |
32 | | - for _ in 0..10 { |
33 | | - format_gdscript_with_buffers(&short_content, &config, &mut render_elements, &mut output)?; |
| 46 | +impl BenchmarkRunner { |
| 47 | + fn new() -> Self { |
| 48 | + Self { |
| 49 | + render_elements: Vec::new(), |
| 50 | + output: String::new(), |
| 51 | + } |
34 | 52 | } |
35 | 53 |
|
36 | | - println!("Benchmarking short file ({} iterations)", ITERATIONS); |
37 | | - let mut start = Instant::now(); |
38 | | - for _ in 0..ITERATIONS { |
39 | | - format_gdscript_with_buffers(&short_content, &config, &mut render_elements, &mut output)?; |
| 54 | + fn format(&mut self, source: &str, config: &FormatterConfiguration) -> Result<(), String> { |
| 55 | + format_gdscript_with_buffers( |
| 56 | + black_box(source), |
| 57 | + black_box(config), |
| 58 | + &mut self.render_elements, |
| 59 | + &mut self.output, |
| 60 | + )?; |
| 61 | + black_box(&self.output); |
| 62 | + Ok(()) |
40 | 63 | } |
41 | | - let duration_short_file = start.elapsed(); |
42 | 64 |
|
43 | | - println!("Benchmarking long file ({} iterations)...", ITERATIONS); |
44 | | - start = Instant::now(); |
45 | | - for _ in 0..ITERATIONS { |
46 | | - format_gdscript_with_buffers(&long_content, &config, &mut render_elements, &mut output)?; |
47 | | - } |
48 | | - let long_time = start.elapsed(); |
| 65 | + fn measure( |
| 66 | + &mut self, |
| 67 | + source: &str, |
| 68 | + config: &FormatterConfiguration, |
| 69 | + ) -> Result<BenchmarkMeasurement, String> { |
| 70 | + let warmup_start = Instant::now(); |
| 71 | + while warmup_start.elapsed() < WARMUP_DURATION { |
| 72 | + self.format(source, config)?; |
| 73 | + } |
49 | 74 |
|
50 | | - let safe_config = FormatterConfiguration { |
51 | | - safe: true, |
52 | | - ..config |
53 | | - }; |
| 75 | + let mut seconds_per_iteration = Vec::with_capacity(SAMPLE_COUNT); |
| 76 | + let mut total_iterations = 0; |
| 77 | + let mut sample_index = 0; |
| 78 | + while sample_index < SAMPLE_COUNT { |
| 79 | + let sample_start = Instant::now(); |
| 80 | + let mut sample_iterations = 0; |
| 81 | + while sample_start.elapsed() < SAMPLE_DURATION { |
| 82 | + self.format(source, config)?; |
| 83 | + sample_iterations += 1; |
| 84 | + } |
| 85 | + let sample_seconds = sample_start.elapsed().as_secs_f64(); |
| 86 | + seconds_per_iteration.push(sample_seconds / sample_iterations as f64); |
| 87 | + total_iterations += sample_iterations; |
| 88 | + sample_index += 1; |
| 89 | + } |
54 | 90 |
|
55 | | - println!( |
56 | | - "Benchmarking short file with safe mode ({} iterations)...", |
57 | | - ITERATIONS |
58 | | - ); |
59 | | - start = Instant::now(); |
60 | | - for _ in 0..ITERATIONS { |
61 | | - format_gdscript_with_buffers( |
62 | | - &short_content, |
63 | | - &safe_config, |
64 | | - &mut render_elements, |
65 | | - &mut output, |
66 | | - )?; |
| 91 | + let mut current_index = 1; |
| 92 | + while current_index < seconds_per_iteration.len() { |
| 93 | + let mut sorted_index = current_index; |
| 94 | + while sorted_index > 0 |
| 95 | + && seconds_per_iteration[sorted_index] < seconds_per_iteration[sorted_index - 1] |
| 96 | + { |
| 97 | + seconds_per_iteration.swap(sorted_index, sorted_index - 1); |
| 98 | + sorted_index -= 1; |
| 99 | + } |
| 100 | + current_index += 1; |
| 101 | + } |
| 102 | + |
| 103 | + Ok(BenchmarkMeasurement { |
| 104 | + median_seconds_per_iteration: seconds_per_iteration[SAMPLE_COUNT / 2], |
| 105 | + total_iterations, |
| 106 | + }) |
67 | 107 | } |
68 | | - let duration_short_file_safe = start.elapsed(); |
| 108 | +} |
69 | 109 |
|
70 | | - println!( |
71 | | - "Benchmarking long file with safe mode ({} iterations)...", |
72 | | - ITERATIONS |
73 | | - ); |
74 | | - start = Instant::now(); |
75 | | - for _ in 0..ITERATIONS { |
76 | | - format_gdscript_with_buffers( |
77 | | - &long_content, |
78 | | - &safe_config, |
79 | | - &mut render_elements, |
80 | | - &mut output, |
81 | | - )?; |
| 110 | +fn main() -> Result<(), Box<dyn std::error::Error>> { |
| 111 | + let short_content = fs::read_to_string("benchmarks/gdscript_files/short.gd")?; |
| 112 | + let long_content = fs::read_to_string("benchmarks/gdscript_files/long.gd")?; |
| 113 | + let config = FormatterConfiguration::default(); |
| 114 | + |
| 115 | + let mut arguments = env::args(); |
| 116 | + arguments.next(); |
| 117 | + if let Some(argument) = arguments.next() { |
| 118 | + if argument != "--profile-long" { |
| 119 | + return Err(format!("Unknown benchmark argument: {argument}").into()); |
| 120 | + } |
| 121 | + let iterations = if let Some(value) = arguments.next() { |
| 122 | + value.parse::<usize>()? |
| 123 | + } else { |
| 124 | + PROFILE_ITERATIONS |
| 125 | + }; |
| 126 | + if arguments.next().is_some() { |
| 127 | + return Err("Usage: benchmark --profile-long [iterations]".into()); |
| 128 | + } |
| 129 | + |
| 130 | + let mut runner = BenchmarkRunner::new(); |
| 131 | + for _ in 0..10 { |
| 132 | + runner.format(&long_content, &config)?; |
| 133 | + } |
| 134 | + println!("Profiling long GDScript file ({iterations} iterations)..."); |
| 135 | + for _ in 0..iterations { |
| 136 | + runner.format(&long_content, &config)?; |
| 137 | + } |
| 138 | + return Ok(()); |
82 | 139 | } |
83 | | - let long_time_safe = start.elapsed(); |
84 | 140 |
|
85 | | - let average_time_short = duration_short_file.as_micros() as f64 / f64::from(ITERATIONS); |
86 | | - let average_time_long = long_time.as_micros() as f64 / f64::from(ITERATIONS); |
87 | | - let average_time_safe_short = |
88 | | - duration_short_file_safe.as_micros() as f64 / f64::from(ITERATIONS); |
89 | | - let average_time_safe_long = long_time_safe.as_micros() as f64 / f64::from(ITERATIONS); |
| 141 | + println!("Running GDScript Formatter Benchmark..."); |
| 142 | + |
| 143 | + let safe_config = FormatterConfiguration { |
| 144 | + safe: true, |
| 145 | + ..config.clone() |
| 146 | + }; |
| 147 | + let mut runner = BenchmarkRunner::new(); |
| 148 | + println!("Benchmarking short file..."); |
| 149 | + let short = runner.measure(&short_content, &config)?; |
| 150 | + println!("Benchmarking short file with safe mode..."); |
| 151 | + let short_safe = runner.measure(&short_content, &safe_config)?; |
| 152 | + println!("Benchmarking long file..."); |
| 153 | + let long = runner.measure(&long_content, &config)?; |
| 154 | + println!("Benchmarking long file with safe mode..."); |
| 155 | + let long_safe = runner.measure(&long_content, &safe_config)?; |
90 | 156 |
|
91 | 157 | let short_slowdown = |
92 | | - ((average_time_safe_short - average_time_short) / average_time_short) * 100.0; |
93 | | - let long_slowdown = ((average_time_safe_long - average_time_long) / average_time_long) * 100.0; |
| 158 | + (short_safe.median_seconds_per_iteration / short.median_seconds_per_iteration - 1.0) |
| 159 | + * 100.0; |
| 160 | + let long_slowdown = |
| 161 | + (long_safe.median_seconds_per_iteration / long.median_seconds_per_iteration - 1.0) * 100.0; |
94 | 162 |
|
95 | 163 | println!("\nBenchmark Results:"); |
96 | 164 | println!("================="); |
97 | 165 | println!( |
98 | | - "Short file ({} iterations): {:?} (avg: {:.2}ms per iteration)", |
99 | | - ITERATIONS, |
100 | | - duration_short_file, |
101 | | - average_time_short / 1000.0 |
| 166 | + "Short file ({} iterations): {:.3}ms median per iteration", |
| 167 | + short.total_iterations, |
| 168 | + short.median_seconds_per_iteration * 1000.0 |
102 | 169 | ); |
103 | 170 | println!( |
104 | | - "Long file ({} iterations): {:?} (avg: {:.2}ms per iteration)", |
105 | | - ITERATIONS, |
106 | | - long_time, |
107 | | - average_time_long / 1000.0 |
| 171 | + "Long file ({} iterations): {:.3}ms median per iteration", |
| 172 | + long.total_iterations, |
| 173 | + long.median_seconds_per_iteration * 1000.0 |
108 | 174 | ); |
109 | 175 | println!( |
110 | | - "Short file with safe mode ({} iterations): {:?} (avg: {:.2}ms per iteration, {:.1}% slower)", |
111 | | - ITERATIONS, |
112 | | - duration_short_file_safe, |
113 | | - average_time_safe_short / 1000.0, |
| 176 | + "Short file with safe mode ({} iterations): {:.3}ms median per iteration, {:.1}% slower", |
| 177 | + short_safe.total_iterations, |
| 178 | + short_safe.median_seconds_per_iteration * 1000.0, |
114 | 179 | short_slowdown |
115 | 180 | ); |
116 | 181 | println!( |
117 | | - "Long file with safe mode ({} iterations): {:?} (avg: {:.2}ms per iteration, {:.1}% slower)", |
118 | | - ITERATIONS, |
119 | | - long_time_safe, |
120 | | - average_time_safe_long / 1000.0, |
| 182 | + "Long file with safe mode ({} iterations): {:.3}ms median per iteration, {:.1}% slower", |
| 183 | + long_safe.total_iterations, |
| 184 | + long_safe.median_seconds_per_iteration * 1000.0, |
121 | 185 | long_slowdown |
122 | 186 | ); |
123 | 187 |
|
|
0 commit comments