From 61eb299cec1283524fd9917fd7269ca69a0fbdc1 Mon Sep 17 00:00:00 2001 From: Marcello Costagliola Date: Sat, 24 Oct 2026 12:41:45 +0200 Subject: [PATCH] Stop Redis after the processes that use it On `docker stop`, bin/boot sent TERM to Redis, Thruster and resque-pool at once. Redis exits straight away, while a request or a job still running needs it: a message posted in that moment was saved, then failed to enqueue its push notifications and answered 500. Puma and resque-pool didn't even get the signal. bin/start-app runs Rails under bash, and the Procfile's environment prefix runs resque-pool under sh; both shells exit on TERM without passing it on, so the two servers were killed outright when bin/boot exited, and a running job with them. Now bin/boot asks the other processes to stop, waits for them to exit, then stops Redis. It asks once, because a second TERM tells resque-pool to stop waiting for its workers, and it starts each process in a group of its own, so a Ctrl-C in a terminal reaches only bin/boot. bin/start-app and the Procfile exec their servers, so the TERM reaches Puma, which finishes its requests, and resque-pool, which with --term-graceful-wait lets its workers finish their current job. Co-Authored-By: Claude Opus 5.5 --- Procfile | 2 +- bin/boot | 26 +++++++++-- bin/start-app | 2 +- test/bin/boot_test.rb | 105 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 7 deletions(-) create mode 100644 test/bin/boot_test.rb diff --git a/Procfile b/Procfile index 20e143048..2b846a507 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,3 @@ web: bundle exec thrust bin/start-app redis: redis-server config/redis.conf -workers: FORK_PER_JOB=false INTERVAL=0.1 bundle exec resque-pool +workers: exec env FORK_PER_JOB=false INTERVAL=0.1 bundle exec resque-pool --term-graceful-wait diff --git a/bin/boot b/bin/boot index 5106115a7..e5ce00ec2 100755 --- a/bin/boot +++ b/bin/boot @@ -6,12 +6,19 @@ require "yaml" class ProcessMonitor SIGNALS = %w[ INT TERM CLD ] + # The other processes use these, so they stop only once the others have exited + SERVICES = %w[ redis ] + def initialize(procfile) @procs = process_list(procfile) + @services, @apps = @procs.partition { |process| SERVICES.include?(process.name) } handle_signals @procs.each &:start - @procs.each &:wait + @apps.each &:wait + + @services.each &:terminate + @services.each &:wait end private @@ -22,20 +29,29 @@ class ProcessMonitor def handle_signals SIGNALS.each do |signal| - Signal.trap(signal) do - @procs.each &:terminate - end + Signal.trap(signal) { stop } + end + end + + # Only once: a second TERM tells resque-pool to stop waiting for its workers + def stop + unless @stopping + @stopping = true + @apps.each &:terminate end end end class MonitoredProcess + attr_reader :name + def initialize(name, cmd) @name, @cmd = name, cmd end + # In a process group of its own, so a Ctrl-C in a terminal reaches only bin/boot, which stops everything in order def start - @pid = Process.spawn(@cmd) + @pid = Process.spawn(@cmd, pgroup: true) end def wait diff --git a/bin/start-app b/bin/start-app index 9a751c2f4..89a0540de 100755 --- a/bin/start-app +++ b/bin/start-app @@ -2,4 +2,4 @@ rm -f tmp/pids/server.pid ./bin/rails db:prepare -./bin/rails server +exec ./bin/rails server diff --git a/test/bin/boot_test.rb b/test/bin/boot_test.rb new file mode 100644 index 000000000..457cae457 --- /dev/null +++ b/test/bin/boot_test.rb @@ -0,0 +1,105 @@ +require "test_helper" + +class BootScriptTest < ActiveSupport::TestCase + # Stands in for a Procfile process: records when it starts, when it's asked to stop, + # and how many TERMs it got by the time it exits. Exits on its own once exit-NAME exists. + FAKE_PROCESS = <<~RUBY + name, stop_delay = ARGV + terms = 0 + trap("TERM") { terms += 1 } + + record = ->(file, line) { File.open(file, "a") { |io| io.puts line } } + record.call "pids", Process.pid + record.call "events", "\#{name} started" + + sleep 0.01 until terms > 0 || File.exist?("exit-\#{name}") + + if terms > 0 + record.call "events", "\#{name} stopping" + sleep stop_delay.to_f + end + record.call "events", "\#{name} stopped" + record.call "events", "\#{name} got \#{terms} TERM" + RUBY + + test "stops redis once the processes that use it have exited" do + events = boot(redis: 0, web: 0.2, workers: 0.4) { |pid| Process.kill "TERM", pid } + + assert_before events, "web stopped", "redis stopping" + assert_before events, "workers stopped", "redis stopping" + end + + test "asks each process to stop only once while the others exit" do + events = boot(redis: 0, web: 0, workers: 0.5) { |pid| Process.kill "TERM", pid } + + assert_includes events, "workers got 1 TERM" + assert_includes events, "redis got 1 TERM" + end + + test "stops the others, redis last, when one process exits on its own" do + events = boot(redis: 0, web: 0, workers: 0.2) { |pid, dir| FileUtils.touch File.join(dir, "exit-web") } + + assert_includes events, "web got 0 TERM" + assert_before events, "workers stopped", "redis stopping" + end + + test "stops in the same order when a terminal's Ctrl-C reaches its whole process group" do + events = boot(redis: 0, web: 0.2, workers: 0.4) { |pid| Process.kill "INT", -pid } + + assert_before events, "web stopped", "redis stopping" + assert_before events, "workers stopped", "redis stopping" + end + + private + def assert_before(events, earlier, later) + assert_includes events, earlier + assert_includes events, later + assert_operator events.index(earlier), :<, events.index(later), "#{earlier.inspect} should come before #{later.inspect} in #{events}" + end + + def boot(stop_delays) + Dir.mktmpdir do |dir| + File.write File.join(dir, "fake_process.rb"), FAKE_PROCESS + File.write File.join(dir, "Procfile"), stop_delays.map { |name, delay| + "#{name}: exec #{RbConfig.ruby} fake_process.rb #{name} #{delay}" + }.join("\n") + + pid = Process.spawn(RbConfig.ruby, Rails.root.join("bin/boot").to_s, chdir: dir, pgroup: true) + + exited = false + begin + assert wait_until { lines_in(dir, "events").count { |event| event.end_with?(" started") } == stop_delays.size }, "processes didn't start" + yield pid, dir + exited = wait_until { Process.wait(pid, Process::WNOHANG) } + assert exited, "bin/boot didn't exit" + + lines_in(dir, "events") + ensure + # Once bin/boot has exited it has reaped every process; otherwise they're left to kill + kill_all [ pid, *lines_in(dir, "pids").map(&:to_i) ] unless exited + end + end + end + + def lines_in(dir, name) + path = File.join(dir, name) + File.exist?(path) ? File.readlines(path, chomp: true) : [] + end + + def wait_until(timeout: 10) + deadline = Time.now + timeout + until (result = yield) || Time.now > deadline + sleep 0.01 + end + result + end + + def kill_all(pids) + pids.each do |pid| + Process.kill "KILL", pid + rescue Errno::ESRCH + end + Process.wait pids.first + rescue Errno::ECHILD + end +end