From 0d9727ff6cf3fd4da543090a1e2a4b6d4b10da0f Mon Sep 17 00:00:00 2001 From: Jonathan del Strother Date: Thu, 11 Dec 2025 16:41:41 +0000 Subject: [PATCH 1/3] Avoid empty job groups when there are more workers than jobs When there's a mix of timed & untimed files, the previous behaviour could produce a list of job groups with some empty elements. This could cause, eg, `flatware rspec ./spec/mailers` to send an empty file list to one of the spawned workers, causing that worker to run every single spec under ./spec --- lib/flatware/rspec/job_builder.rb | 1 + spec/flatware/rspec/job_builder_spec.rb | 23 ++++++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/flatware/rspec/job_builder.rb b/lib/flatware/rspec/job_builder.rb index 1ae9756..fdce9f4 100644 --- a/lib/flatware/rspec/job_builder.rb +++ b/lib/flatware/rspec/job_builder.rb @@ -48,6 +48,7 @@ def balance_jobs(bucket_count:, timed_files:, untimed_files:) .zip( round_robin(bucket_count, untimed_files) ).map(&:flatten) + .reject(&:empty?) .map { |files| Job.new(files, args) } end diff --git a/spec/flatware/rspec/job_builder_spec.rb b/spec/flatware/rspec/job_builder_spec.rb index 6e35d14..6a8b114 100644 --- a/spec/flatware/rspec/job_builder_spec.rb +++ b/spec/flatware/rspec/job_builder_spec.rb @@ -16,9 +16,10 @@ let(:persisted_examples) { [] } let(:files_to_run) { [] } + let(:worker_count) { 2 } subject do - described_class.new([], workers: 2).jobs + described_class.new([], workers: worker_count).jobs end context 'when this run includes persisted examples' do @@ -65,5 +66,25 @@ ) end end + + context 'and there are more workers than example-files' do + let(:files_to_run) do + %w[ + fast_1_spec.rb + slow_spec.rb + new_1_spec.rb + ] + end + let(:worker_count) { 5 } + + it "doesn't return empty job-groups" do + expect(subject).to match_array( + [ + have_attributes(id: include('./slow_spec.rb', './new_1_spec.rb')), + have_attributes(id: include('./fast_1_spec.rb')) + ] + ) + end + end end end From 455b61c10b74712be1b90ee8342b260ee9cac812 Mon Sep 17 00:00:00 2001 From: Jonathan del Strother Date: Thu, 11 Dec 2025 16:45:15 +0000 Subject: [PATCH 2/3] Don't spawn more workers than there are jobs --- lib/flatware/cli.rb | 4 ++++ lib/flatware/cucumber/cli.rb | 20 +++++++++----------- lib/flatware/rspec/cli.rb | 6 ++++-- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/lib/flatware/cli.rb b/lib/flatware/cli.rb index 8d3a1b9..066ca3f 100644 --- a/lib/flatware/cli.rb +++ b/lib/flatware/cli.rb @@ -76,6 +76,10 @@ def try_setpgrp def workers options[:workers] end + + def worker_spawn_count(jobs) + [workers, jobs.length].min + end end end diff --git a/lib/flatware/cucumber/cli.rb b/lib/flatware/cucumber/cli.rb index 98092d9..5ed872a 100644 --- a/lib/flatware/cucumber/cli.rb +++ b/lib/flatware/cucumber/cli.rb @@ -16,24 +16,22 @@ class CLI 'parallelizes cucumber with custom arguments' ) def cucumber(*args) - config = Cucumber.configure args + jobs = load_jobs(args) - ensure_jobs(config) + formatter = Flatware::Cucumber::Formatters::Console.new($stdout, $stderr) Flatware.verbose = options[:log] - sink = options['sink-endpoint'] - Worker.spawn(count: workers, runner: Cucumber, sink: sink) - start_sink( - jobs: config.jobs, - workers: workers, - formatter: Flatware::Cucumber::Formatters::Console.new($stdout, $stderr) - ) + + spawn_count = worker_spawn_count(jobs) + Worker.spawn(count: spawn_count, runner: Cucumber, sink: options['sink-endpoint']) + start_sink(jobs: jobs, workers: spawn_count, formatter: formatter) end private - def ensure_jobs(config) - return if config.jobs.any? + def load_jobs(args) + config = Cucumber.configure args + return config.jobs if config.jobs.any? abort( format( diff --git a/lib/flatware/rspec/cli.rb b/lib/flatware/rspec/cli.rb index c0417f7..d912438 100644 --- a/lib/flatware/rspec/cli.rb +++ b/lib/flatware/rspec/cli.rb @@ -23,8 +23,10 @@ def rspec(*rspec_args) ) Flatware.verbose = options[:log] - Worker.spawn count: workers, runner: RSpec, sink: options['sink-endpoint'] - start_sink(jobs: jobs, workers: workers, formatter: formatter) + + spawn_count = worker_spawn_count(jobs) + Worker.spawn(count: spawn_count, runner: RSpec, sink: options['sink-endpoint']) + start_sink(jobs: jobs, workers: spawn_count, formatter: formatter) end end end From 58a316b89a0db7555318c716f52efa733fd1b515 Mon Sep 17 00:00:00 2001 From: Jonathan del Strother Date: Thu, 11 Dec 2025 16:56:35 +0000 Subject: [PATCH 3/3] Improve job-distribution with a mix of timed & untimed files --- lib/flatware/rspec/job_builder.rb | 15 ++++++++++----- spec/flatware/rspec/job_builder_spec.rb | 5 +++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/lib/flatware/rspec/job_builder.rb b/lib/flatware/rspec/job_builder.rb index fdce9f4..42250be 100644 --- a/lib/flatware/rspec/job_builder.rb +++ b/lib/flatware/rspec/job_builder.rb @@ -43,11 +43,16 @@ def jobs private def balance_jobs(bucket_count:, timed_files:, untimed_files:) - balance_by(bucket_count, timed_files, &:last) - .map { |bucket| bucket.map(&:first) } - .zip( - round_robin(bucket_count, untimed_files) - ).map(&:flatten) + timed_groups = balance_by(bucket_count, timed_files, &:last) + .map { |bucket| bucket.map(&:first) } + untimed_groups = round_robin(bucket_count, untimed_files) + + # When the files can't be evenly divided between groups, the first groups + # in each of timed_groups & untimed_groups will have more files. + # By reversing one of them before combining them, we can improve the final distribution. + timed_groups + .zip(untimed_groups.reverse) + .map(&:flatten) .reject(&:empty?) .map { |files| Job.new(files, args) } end diff --git a/spec/flatware/rspec/job_builder_spec.rb b/spec/flatware/rspec/job_builder_spec.rb index 6a8b114..2cfee19 100644 --- a/spec/flatware/rspec/job_builder_spec.rb +++ b/spec/flatware/rspec/job_builder_spec.rb @@ -80,8 +80,9 @@ it "doesn't return empty job-groups" do expect(subject).to match_array( [ - have_attributes(id: include('./slow_spec.rb', './new_1_spec.rb')), - have_attributes(id: include('./fast_1_spec.rb')) + have_attributes(id: include('./fast_1_spec.rb')), + have_attributes(id: include('./slow_spec.rb')), + have_attributes(id: include('./new_1_spec.rb')) ] ) end