diff --git a/bin/boot b/bin/boot index 5106115a7..6c910e3fc 100755 --- a/bin/boot +++ b/bin/boot @@ -4,14 +4,14 @@ require "json" require "yaml" class ProcessMonitor - SIGNALS = %w[ INT TERM CLD ] + SIGNALS = %w[ INT TERM ] def initialize(procfile) @procs = process_list(procfile) handle_signals @procs.each &:start - @procs.each &:wait + wait_for_procs end private @@ -26,10 +26,25 @@ class ProcessMonitor @procs.each &:terminate end end + + # A SIGCHLD ignored by whatever started this one would have the kernel reap the children, unseen by the wait + Signal.trap("CLD", "SYSTEM_DEFAULT") + end + + # In a container this is PID 1, which also adopts orphans. Waiting for any child reaps them + # too, and only the exit of one of the processes stops the others. + def wait_for_procs + running = @procs.map(&:pid) + + until running.empty? + @procs.each &:terminate if running.delete(Process.wait) + end end end class MonitoredProcess + attr_reader :pid + def initialize(name, cmd) @name, @cmd = name, cmd end @@ -38,12 +53,6 @@ class MonitoredProcess @pid = Process.spawn(@cmd) end - def wait - Process.wait @pid - rescue Errno::ECHILD - nil - end - def terminate Process.kill "TERM", @pid rescue Errno::ESRCH diff --git a/test/bin/boot_test.rb b/test/bin/boot_test.rb new file mode 100644 index 000000000..0a8312737 --- /dev/null +++ b/test/bin/boot_test.rb @@ -0,0 +1,126 @@ +require "test_helper" + +class BootScriptTest < ActiveSupport::TestCase + # Stands in for a Procfile process: records when it starts and how it ends. Exits on its own + # once exit-NAME exists, and leaves an orphan that exits 0.1 s later once orphan-NAME exists. + FAKE_PROCESS = <<~RUBY + name = ARGV.first + terminated = orphaned = false + trap("TERM") { terminated = true } + + record = ->(file, line) { File.open(file, "a") { |io| io.puts line } } + record.call "events", "\#{name} started" + + until terminated || File.exist?("exit-\#{name}") + if !orphaned && File.exist?("orphan-\#{name}") + orphaned = true + Process.wait fork { record.call "orphans", fork { sleep 0.1 }; exit! } + end + sleep 0.01 + end + record.call "events", terminated ? "\#{name} got TERM" : "\#{name} exited" + RUBY + + # In a container bin/boot is PID 1, which adopts orphans. A subreaper adopts them the same way. + ADOPTING_ORPHANS = <<~RUBY + require "fiddle" + PR_SET_CHILD_SUBREAPER = 36 + prctl = Fiddle::Function.new(Fiddle::Handle::DEFAULT["prctl"], [ Fiddle::TYPE_INT ] + [ Fiddle::TYPE_LONG ] * 4, Fiddle::TYPE_INT) + prctl.call(PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0).zero? or abort "prctl failed" + exec RbConfig.ruby, *ARGV + RUBY + + # As if started by a parent that ignores SIGCHLD, which exec keeps + IGNORING_SIGCHLD = <<~RUBY + trap("CLD", "IGNORE") + exec RbConfig.ruby, *ARGV + RUBY + + test "keeps its processes running when an orphan it adopted exits" do + skip "adopting orphans outside PID 1 needs Linux and fiddle" unless RUBY_PLATFORM.include?("linux") && fiddle? + + events = boot(%w[ web workers ], wrapper: ADOPTING_ORPHANS) do |pid, dir| + FileUtils.touch File.join(dir, "orphan-web") + assert wait_until { lines_in(dir, "orphans").any? }, "no orphan was left" + orphan = lines_in(dir, "orphans").first.to_i + + assert wait_until { [ "Z", nil ].include?(state_of(orphan)) }, "the orphan didn't exit" + sleep 0.2 + assert_empty lines_in(dir, "events").grep(/TERM/), "the orphan's exit stopped the processes" + assert wait_until { state_of(orphan).nil? }, "the orphan was left a zombie" + + Process.kill "TERM", pid + end + + assert_includes events, "web got TERM" + assert_includes events, "workers got TERM" + end + + test "stops the other processes when one of them exits" do + events = boot(%w[ web workers ]) { |pid, dir| FileUtils.touch File.join(dir, "exit-workers") } + + assert_includes events, "workers exited" + assert_includes events, "web got TERM" + end + + test "stops the other processes when one of them exits, though started with SIGCHLD ignored" do + events = boot(%w[ web workers ], wrapper: IGNORING_SIGCHLD) { |pid, dir| FileUtils.touch File.join(dir, "exit-workers") } + + assert_includes events, "workers exited" + assert_includes events, "web got TERM" + end + + private + def boot(names, wrapper: nil) + Dir.mktmpdir do |dir| + File.write File.join(dir, "fake_process.rb"), FAKE_PROCESS + File.write File.join(dir, "Procfile"), names.map { |name| "#{name}: exec #{RbConfig.ruby} fake_process.rb #{name}" }.join("\n") + + command = [ RbConfig.ruby, *([ "-e", wrapper ] if wrapper), Rails.root.join("bin/boot").to_s ] + pid = Bundler.with_unbundled_env { Process.spawn(*command, chdir: dir, pgroup: true) } + + exited = false + begin + assert wait_until { lines_in(dir, "events").count { |event| event.end_with?(" started") } == names.size }, "processes didn't start" + yield pid, dir + exited = wait_until { Process.wait(pid, Process::WNOHANG) } + assert exited, "bin/boot didn't exit" + + lines_in(dir, "events") + ensure + # Once bin/boot has exited it has reaped every process; otherwise its process group goes, orphans included + kill_group pid unless exited + end + end + end + + def fiddle? + Bundler.with_unbundled_env { system(RbConfig.ruby, "-e", "require 'fiddle'", err: File::NULL) } + end + + def lines_in(dir, name) + path = File.join(dir, name) + File.exist?(path) ? File.readlines(path, chomp: true) : [] + end + + # The process's state letter from /proc ("Z" for a zombie), or nil once it's gone + def state_of(pid) + File.read("/proc/#{pid}/stat")[/\) (\w)/, 1] + rescue Errno::ENOENT + nil + end + + def wait_until(timeout: 10) + deadline = Time.now + timeout + until (result = yield) || Time.now > deadline + sleep 0.01 + end + result + end + + def kill_group(pid) + Process.kill "KILL", -pid + Process.wait pid + rescue Errno::ESRCH, Errno::ECHILD + end +end