From 6370d28b75be2b5f658a3d0cec83b0c61b267e15 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 29 Jul 2026 13:12:24 +1200 Subject: [PATCH 1/9] Add worker diagnostic Bake tasks Assisted-By: devx/d15f74b3-893f-4846-bbae-4dcdd10e8b8a --- bake/async/service/supervisor.rb | 20 ++++++++++++++++++++ examples/simple/simple.rb | 1 + 2 files changed, 21 insertions(+) diff --git a/bake/async/service/supervisor.rb b/bake/async/service/supervisor.rb index 140ebb6..7ed2d7a 100644 --- a/bake/async/service/supervisor.rb +++ b/bake/async/service/supervisor.rb @@ -32,6 +32,26 @@ def status end end +# List the connection IDs of all registered workers. +def workers + client do |connection| + supervisor = connection[:supervisor] + supervisor.keys + end +end + +# Dump the object space of a worker to a file on the worker's filesystem. +# +# @parameter connection_id [Integer] The connection ID of the worker to target. +# @parameter path [String] The file path where the worker should write the dump. +def memory_dump(connection_id:, path:) + client do |connection| + supervisor = connection[:supervisor] + worker = supervisor[connection_id] + worker.memory_dump(path: path) + end +end + private def endpoint diff --git a/examples/simple/simple.rb b/examples/simple/simple.rb index 9e45467..561f846 100755 --- a/examples/simple/simple.rb +++ b/examples/simple/simple.rb @@ -39,6 +39,7 @@ def setup(container) service "sleep" do service_class SleepService + include Async::Service::Managed::Environment include Async::Service::Supervisor::Supervised end From 0a3f49f173cd9e5a01abb1980da2c30ea98949ed Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 29 Jul 2026 13:15:08 +1200 Subject: [PATCH 2/9] Expose remaining worker diagnostics through Bake Assisted-By: devx/d15f74b3-893f-4846-bbae-4dcdd10e8b8a --- bake/async/service/supervisor.rb | 51 +++++++++++++- test/bake/async/service/supervisor.rb | 95 +++++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 3 deletions(-) create mode 100644 test/bake/async/service/supervisor.rb diff --git a/bake/async/service/supervisor.rb b/bake/async/service/supervisor.rb index 7ed2d7a..a204b8a 100644 --- a/bake/async/service/supervisor.rb +++ b/bake/async/service/supervisor.rb @@ -45,19 +45,64 @@ def workers # @parameter connection_id [Integer] The connection ID of the worker to target. # @parameter path [String] The file path where the worker should write the dump. def memory_dump(connection_id:, path:) - client do |connection| - supervisor = connection[:supervisor] - worker = supervisor[connection_id] + with_worker(connection_id) do |worker| worker.memory_dump(path: path) end end +# Dump the fiber scheduler hierarchy of a worker. +# +# @parameter connection_id [Integer] The connection ID of the worker to target. +# @parameter path [String | Nil] An optional file path on the worker's filesystem. +# @parameter log [String | Nil] An optional message to log with the dump. +def scheduler_dump(connection_id:, path: nil, log: nil) + with_worker(connection_id) do |worker| + worker.scheduler_dump(path: path, log: log) + end +end + +# Dump information about all threads in a worker. +# +# @parameter connection_id [Integer] The connection ID of the worker to target. +# @parameter path [String | Nil] An optional file path on the worker's filesystem. +def thread_dump(connection_id:, path: nil) + with_worker(connection_id) do |worker| + worker.thread_dump(path: path) + end +end + +# Start garbage collection profiling in a worker. +# +# @parameter connection_id [Integer] The connection ID of the worker to target. +def garbage_profile_start(connection_id:) + with_worker(connection_id) do |worker| + worker.garbage_profile_start + end +end + +# Stop garbage collection profiling in a worker and return or save the results. +# +# @parameter connection_id [Integer] The connection ID of the worker to target. +# @parameter path [String | Nil] An optional file path on the worker's filesystem. +def garbage_profile_stop(connection_id:, path: nil) + with_worker(connection_id) do |worker| + worker.garbage_profile_stop(path: path) + end +end + private def endpoint Async::Service::Supervisor.endpoint end +def with_worker(connection_id) + client do |connection| + supervisor = connection[:supervisor] + yield supervisor[connection_id] + end +end + def client(&block) Sync do Async::Service::Supervisor::Client.new(endpoint: self.endpoint).connect(&block) diff --git a/test/bake/async/service/supervisor.rb b/test/bake/async/service/supervisor.rb new file mode 100644 index 0000000..df4224e --- /dev/null +++ b/test/bake/async/service/supervisor.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2026, by Samuel Williams. + +require "bake/context" + +describe "async:service:supervisor" do + let(:context) {@@context ||= Bake::Context.load} + let(:worker) {Object.new} + let(:result) {Object.new} + + def invoke(name, connection_id: 7, **options) + recipe = context.lookup("async:service:supervisor:#{name}") + + mock(recipe.instance) do |mock| + mock.replace(:with_worker) do |id, &block| + expect(id).to be == connection_id + block.call(worker) + end + + return recipe.call(connection_id: connection_id, **options) + end + end + + it "lists the workers" do + recipe = context.lookup("async:service:supervisor:workers") + supervisor = Object.new + connection = Object.new + supervisor.define_singleton_method(:keys) {[1, 2, 3]} + connection.define_singleton_method(:[]) do |name| + raise ArgumentError, "Unexpected controller: #{name.inspect}" unless name == :supervisor + supervisor + end + + mock(recipe.instance) do |mock| + mock.replace(:client) {|&block| block.call(connection)} + + expect(recipe.call).to be == [1, 2, 3] + end + end + + it "dumps memory" do + mock(worker) do |mock| + mock.replace(:memory_dump) do |path:| + expect(path).to be == "/tmp/memory.json" + result + end + + expect(invoke("memory_dump", path: "/tmp/memory.json")).to be == result + end + end + + it "dumps the scheduler" do + mock(worker) do |mock| + mock.replace(:scheduler_dump) do |path:, log:| + expect(path).to be == "/tmp/scheduler.txt" + expect(log).to be == "Scheduler dump" + result + end + + expect(invoke("scheduler_dump", path: "/tmp/scheduler.txt", log: "Scheduler dump")).to be == result + end + end + + it "dumps the threads" do + mock(worker) do |mock| + mock.replace(:thread_dump) do |path:| + expect(path).to be == "/tmp/threads.txt" + result + end + + expect(invoke("thread_dump", path: "/tmp/threads.txt")).to be == result + end + end + + it "starts garbage collection profiling" do + mock(worker) do |mock| + mock.replace(:garbage_profile_start) {result} + + expect(invoke("garbage_profile_start")).to be == result + end + end + + it "stops garbage collection profiling" do + mock(worker) do |mock| + mock.replace(:garbage_profile_stop) do |path:| + expect(path).to be == "/tmp/gc.txt" + result + end + + expect(invoke("garbage_profile_stop", path: "/tmp/gc.txt")).to be == result + end + end +end From b03b14a9aaf3d634693357d31510c30df91e3482 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 29 Jul 2026 13:18:37 +1200 Subject: [PATCH 3/9] Refine worker diagnostic Bake tests Assisted-By: devx/d15f74b3-893f-4846-bbae-4dcdd10e8b8a --- test/bake/async/service/supervisor.rb | 56 ++++++++++++++++----------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/test/bake/async/service/supervisor.rb b/test/bake/async/service/supervisor.rb index df4224e..636836f 100644 --- a/test/bake/async/service/supervisor.rb +++ b/test/bake/async/service/supervisor.rb @@ -7,36 +7,48 @@ describe "async:service:supervisor" do let(:context) {@@context ||= Bake::Context.load} + let(:supervisor) {Object.new} let(:worker) {Object.new} let(:result) {Object.new} + let(:connection) do + {supervisor: supervisor} + end - def invoke(name, connection_id: 7, **options) + def invoke(name, **options) recipe = context.lookup("async:service:supervisor:#{name}") + result = nil mock(recipe.instance) do |mock| - mock.replace(:with_worker) do |id, &block| - expect(id).to be == connection_id - block.call(worker) + mock.replace(:client) do |&block| + block.call(connection) end - return recipe.call(connection_id: connection_id, **options) + result = recipe.call(**options) end + + result end - it "lists the workers" do - recipe = context.lookup("async:service:supervisor:workers") - supervisor = Object.new - connection = Object.new - supervisor.define_singleton_method(:keys) {[1, 2, 3]} - connection.define_singleton_method(:[]) do |name| - raise ArgumentError, "Unexpected controller: #{name.inspect}" unless name == :supervisor - supervisor + def invoke_worker(name, connection_id: 7, **options) + result = nil + + mock(supervisor) do |mock| + mock.replace(:[]) do |id| + expect(id).to be == connection_id + worker + end + + result = invoke(name, connection_id: connection_id, **options) end - mock(recipe.instance) do |mock| - mock.replace(:client) {|&block| block.call(connection)} + result + end + + it "lists the workers" do + mock(supervisor) do |mock| + mock.replace(:keys){[1, 2, 3]} - expect(recipe.call).to be == [1, 2, 3] + expect(invoke("workers")).to be == [1, 2, 3] end end @@ -47,7 +59,7 @@ def invoke(name, connection_id: 7, **options) result end - expect(invoke("memory_dump", path: "/tmp/memory.json")).to be == result + expect(invoke_worker("memory_dump", path: "/tmp/memory.json")).to be == result end end @@ -59,7 +71,7 @@ def invoke(name, connection_id: 7, **options) result end - expect(invoke("scheduler_dump", path: "/tmp/scheduler.txt", log: "Scheduler dump")).to be == result + expect(invoke_worker("scheduler_dump", path: "/tmp/scheduler.txt", log: "Scheduler dump")).to be == result end end @@ -70,15 +82,15 @@ def invoke(name, connection_id: 7, **options) result end - expect(invoke("thread_dump", path: "/tmp/threads.txt")).to be == result + expect(invoke_worker("thread_dump", path: "/tmp/threads.txt")).to be == result end end it "starts garbage collection profiling" do mock(worker) do |mock| - mock.replace(:garbage_profile_start) {result} + mock.replace(:garbage_profile_start){result} - expect(invoke("garbage_profile_start")).to be == result + expect(invoke_worker("garbage_profile_start")).to be == result end end @@ -89,7 +101,7 @@ def invoke(name, connection_id: 7, **options) result end - expect(invoke("garbage_profile_stop", path: "/tmp/gc.txt")).to be == result + expect(invoke_worker("garbage_profile_stop", path: "/tmp/gc.txt")).to be == result end end end From cc1eabd799ee3b1f8dd8634a0c736975195cf08c Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 29 Jul 2026 13:19:58 +1200 Subject: [PATCH 4/9] Use temporary paths in Bake tests Assisted-By: devx/d15f74b3-893f-4846-bbae-4dcdd10e8b8a --- test/bake/async/service/supervisor.rb | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/test/bake/async/service/supervisor.rb b/test/bake/async/service/supervisor.rb index 636836f..b01a0a1 100644 --- a/test/bake/async/service/supervisor.rb +++ b/test/bake/async/service/supervisor.rb @@ -4,8 +4,11 @@ # Copyright, 2026, by Samuel Williams. require "bake/context" +require "sus/fixtures/temporary_directory_context" describe "async:service:supervisor" do + include Sus::Fixtures::TemporaryDirectoryContext + let(:context) {@@context ||= Bake::Context.load} let(:supervisor) {Object.new} let(:worker) {Object.new} @@ -53,36 +56,42 @@ def invoke_worker(name, connection_id: 7, **options) end it "dumps memory" do + expected_path = File.join(root, "memory.json") + mock(worker) do |mock| mock.replace(:memory_dump) do |path:| - expect(path).to be == "/tmp/memory.json" + expect(path).to be == expected_path result end - expect(invoke_worker("memory_dump", path: "/tmp/memory.json")).to be == result + expect(invoke_worker("memory_dump", path: expected_path)).to be == result end end it "dumps the scheduler" do + expected_path = File.join(root, "scheduler.txt") + mock(worker) do |mock| mock.replace(:scheduler_dump) do |path:, log:| - expect(path).to be == "/tmp/scheduler.txt" + expect(path).to be == expected_path expect(log).to be == "Scheduler dump" result end - expect(invoke_worker("scheduler_dump", path: "/tmp/scheduler.txt", log: "Scheduler dump")).to be == result + expect(invoke_worker("scheduler_dump", path: expected_path, log: "Scheduler dump")).to be == result end end it "dumps the threads" do + expected_path = File.join(root, "threads.txt") + mock(worker) do |mock| mock.replace(:thread_dump) do |path:| - expect(path).to be == "/tmp/threads.txt" + expect(path).to be == expected_path result end - expect(invoke_worker("thread_dump", path: "/tmp/threads.txt")).to be == result + expect(invoke_worker("thread_dump", path: expected_path)).to be == result end end @@ -95,13 +104,15 @@ def invoke_worker(name, connection_id: 7, **options) end it "stops garbage collection profiling" do + expected_path = File.join(root, "gc.txt") + mock(worker) do |mock| mock.replace(:garbage_profile_stop) do |path:| - expect(path).to be == "/tmp/gc.txt" + expect(path).to be == expected_path result end - expect(invoke_worker("garbage_profile_stop", path: "/tmp/gc.txt")).to be == result + expect(invoke_worker("garbage_profile_stop", path: expected_path)).to be == result end end end From 336bef7f9f1f359b1ffc04db2b3e73e81738c2a6 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 29 Jul 2026 13:25:15 +1200 Subject: [PATCH 5/9] Exercise Bake tasks with a real supervisor Assisted-By: devx/d15f74b3-893f-4846-bbae-4dcdd10e8b8a --- fixtures/async/service/supervisor/a_server.rb | 3 +- .../service/supervisor/a_simple_service.rb | 40 +++++++ test/bake/async/service/supervisor.rb | 101 +++++------------- 3 files changed, 68 insertions(+), 76 deletions(-) create mode 100644 fixtures/async/service/supervisor/a_simple_service.rb diff --git a/fixtures/async/service/supervisor/a_server.rb b/fixtures/async/service/supervisor/a_server.rb index 265a9fc..a727fa5 100644 --- a/fixtures/async/service/supervisor/a_server.rb +++ b/fixtures/async/service/supervisor/a_server.rb @@ -21,7 +21,7 @@ def initialize attr :registrations - def run + def run(parent: nil) end def status @@ -87,4 +87,3 @@ def restart_supervisor end end end - diff --git a/fixtures/async/service/supervisor/a_simple_service.rb b/fixtures/async/service/supervisor/a_simple_service.rb new file mode 100644 index 0000000..23babe5 --- /dev/null +++ b/fixtures/async/service/supervisor/a_simple_service.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2026, by Samuel Williams. + +require "async/service/supervisor/a_server" + +module Async + module Service + module Supervisor + ASimpleService = Sus::Shared("a simple service") do + include_context AServer + + let(:worker) do + Worker.new( + process_id: Process.pid, + endpoint: endpoint, + state: {name: "simple"}, + ) + end + + let(:connection_id) {@registration.supervisor_controller.id} + + before do + @worker_task = worker.run + @registration = registration_monitor.pop(timeout: 5) + end + + after do + GC::Profiler.disable + + if worker_task = @worker_task + @worker_task = nil + worker_task.stop + end + end + end + end + end +end diff --git a/test/bake/async/service/supervisor.rb b/test/bake/async/service/supervisor.rb index b01a0a1..afd4718 100644 --- a/test/bake/async/service/supervisor.rb +++ b/test/bake/async/service/supervisor.rb @@ -4,115 +4,68 @@ # Copyright, 2026, by Samuel Williams. require "bake/context" -require "sus/fixtures/temporary_directory_context" +require "async/service/supervisor/a_simple_service" describe "async:service:supervisor" do - include Sus::Fixtures::TemporaryDirectoryContext + include_context Async::Service::Supervisor::ASimpleService let(:context) {@@context ||= Bake::Context.load} - let(:supervisor) {Object.new} - let(:worker) {Object.new} - let(:result) {Object.new} - let(:connection) do - {supervisor: supervisor} - end def invoke(name, **options) recipe = context.lookup("async:service:supervisor:#{name}") result = nil mock(recipe.instance) do |mock| - mock.replace(:client) do |&block| - block.call(connection) - end - + mock.replace(:endpoint){endpoint} result = recipe.call(**options) end result end - def invoke_worker(name, connection_id: 7, **options) - result = nil - - mock(supervisor) do |mock| - mock.replace(:[]) do |id| - expect(id).to be == connection_id - worker - end - - result = invoke(name, connection_id: connection_id, **options) - end - - result - end - it "lists the workers" do - mock(supervisor) do |mock| - mock.replace(:keys){[1, 2, 3]} - - expect(invoke("workers")).to be == [1, 2, 3] - end + expect(invoke("workers")).to be == [connection_id] end it "dumps memory" do - expected_path = File.join(root, "memory.json") + path = File.join(@root, "memory.json") + result = invoke("memory_dump", connection_id: connection_id, path: path) - mock(worker) do |mock| - mock.replace(:memory_dump) do |path:| - expect(path).to be == expected_path - result - end - - expect(invoke_worker("memory_dump", path: expected_path)).to be == result - end + expect(result).to be == {path: path} + expect(File.size(path)).to be > 0 end it "dumps the scheduler" do - expected_path = File.join(root, "scheduler.txt") + path = File.join(@root, "scheduler.txt") + result = invoke("scheduler_dump", connection_id: connection_id, path: path) - mock(worker) do |mock| - mock.replace(:scheduler_dump) do |path:, log:| - expect(path).to be == expected_path - expect(log).to be == "Scheduler dump" - result - end - - expect(invoke_worker("scheduler_dump", path: expected_path, log: "Scheduler dump")).to be == result - end + expect(result).to be == {path: path} + expect(File.size(path)).to be > 0 end it "dumps the threads" do - expected_path = File.join(root, "threads.txt") + path = File.join(@root, "threads.txt") + result = invoke("thread_dump", connection_id: connection_id, path: path) - mock(worker) do |mock| - mock.replace(:thread_dump) do |path:| - expect(path).to be == expected_path - result - end - - expect(invoke_worker("thread_dump", path: expected_path)).to be == result - end + expect(result).to be == {path: path} + expect(File.size(path)).to be > 0 end it "starts garbage collection profiling" do - mock(worker) do |mock| - mock.replace(:garbage_profile_start){result} - - expect(invoke_worker("garbage_profile_start")).to be == result - end + result = invoke("garbage_profile_start", connection_id: connection_id) + + expect(result).to be == {started: true} + expect(GC::Profiler.enabled?).to be == true end it "stops garbage collection profiling" do - expected_path = File.join(root, "gc.txt") + path = File.join(@root, "gc.txt") - mock(worker) do |mock| - mock.replace(:garbage_profile_stop) do |path:| - expect(path).to be == expected_path - result - end - - expect(invoke_worker("garbage_profile_stop", path: expected_path)).to be == result - end + invoke("garbage_profile_start", connection_id: connection_id) + result = invoke("garbage_profile_stop", connection_id: connection_id, path: path) + + expect(result).to be == {path: path} + expect(File).to be(:exist?, path) + expect(GC::Profiler.enabled?).to be == false end end From f678150b39b67d5265b626edf07501cbff87f7cf Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 29 Jul 2026 13:30:44 +1200 Subject: [PATCH 6/9] Document worker memory diagnostics Assisted-By: devx/d15f74b3-893f-4846-bbae-4dcdd10e8b8a --- context/getting-started.md | 36 ++---- context/index.yaml | 4 + context/memory-diagnostics.md | 191 ++++++++++++++++++++++++++++ context/memory-monitor.md | 55 ++------ context/migration.md | 2 +- guides/getting-started/readme.md | 36 ++---- guides/links.yaml | 7 +- guides/memory-diagnostics/readme.md | 191 ++++++++++++++++++++++++++++ guides/memory-monitor/readme.md | 55 ++------ guides/migration/readme.md | 2 +- readme.md | 2 + 11 files changed, 439 insertions(+), 142 deletions(-) create mode 100644 context/memory-diagnostics.md create mode 100644 guides/memory-diagnostics/readme.md diff --git a/context/getting-started.md b/context/getting-started.md index 6e848ca..87d7810 100644 --- a/context/getting-started.md +++ b/context/getting-started.md @@ -149,44 +149,28 @@ end The supervisor can collect various diagnostics from workers on demand: - **Memory dumps**: Full heap dumps for memory analysis via `ObjectSpace.dump_all`. -- **Memory samples**: Lightweight sampling to identify memory leaks. - **Thread dumps**: Stack traces of all threads. -- **Scheduler dumps**: Async fiber hierarchy -- **Garbage collection profiles**: GC performance data +- **Scheduler dumps**: Async fiber hierarchy. +- **Garbage collection profiles**: GC performance data. -These can be triggered programmatically or via command-line tools (when available). +These can be triggered programmatically or with Bake tasks. #### Memory Leak Diagnosis -To identify memory leaks, you can use the memory sampling feature which is much lighter weight than a full memory dump. It tracks allocations over a time period and focuses on retained objects. - -**Using the bake task:** +Start by listing the workers registered with the supervisor: ```bash -# Sample for 30 seconds and print report to console -$ bake async:container:supervisor:memory_sample duration=30 +$ bake async:service:supervisor:workers ``` -**Programmatically:** +Then capture heap dumps from the same worker before and after the suspected growth period: -```ruby -# Assuming you have a connection to a worker: -result = connection.call(do: :memory_sample, duration: 30) -puts result[:data] +```bash +$ bake async:service:supervisor:memory_dump connection_id=1 path=/var/tmp/worker-before.json +$ bake async:service:supervisor:memory_dump connection_id=1 path=/var/tmp/worker-after.json ``` -This will sample memory allocations for the specified duration, then force a garbage collection and return a JSON report showing what objects were allocated during that period and retained after GC. Late-lifecycle allocations that are retained are likely memory leaks. - -The JSON report includes: -- `total_allocated`: Total allocated memory and count -- `total_retained`: Total retained memory and count -- `by_gem`: Breakdown by gem/library -- `by_file`: Breakdown by source file -- `by_location`: Breakdown by specific file:line locations -- `by_class`: Breakdown by object class -- `strings`: String allocation analysis - -This is much more efficient than `do: :memory_dump` which uses `ObjectSpace.dump_all` and can be slow and blocking on large heaps. The JSON format also makes it easy to integrate with monitoring and analysis tools. +Heap dumps are heavyweight and may contain sensitive application data. See the [Memory Diagnostics](../memory-diagnostics/index) guide for the complete capture, comparison, and GC profiling workflow. ## Advanced Usage diff --git a/context/index.yaml b/context/index.yaml index 23e81af..e715bce 100644 --- a/context/index.yaml +++ b/context/index.yaml @@ -19,6 +19,10 @@ files: title: Memory Monitor description: This guide explains how to use the Async::Service::Supervisor::MemoryMonitor to detect and restart workers that exceed memory limits or develop memory leaks. +- path: memory-diagnostics.md + title: Memory Diagnostics + description: This guide explains how to capture Ruby heap dumps and garbage collection + profiles from supervised workers, then use them to investigate memory growth. - path: process-monitor.md title: Process Monitor description: This guide explains how to use the Async::Service::Supervisor::ProcessMonitor diff --git a/context/memory-diagnostics.md b/context/memory-diagnostics.md new file mode 100644 index 0000000..d482c23 --- /dev/null +++ b/context/memory-diagnostics.md @@ -0,0 +1,191 @@ +# Memory Diagnostics + +This guide explains how to capture Ruby heap dumps and garbage collection profiles from supervised workers, then use them to investigate memory growth. + +## Overview + +Use the diagnostic Bake tasks when a worker's memory grows unexpectedly and you need to understand what it is retaining. A typical investigation combines: + +- {ruby Async::Service::Supervisor::MemoryMonitor} or process metrics to identify sustained growth. +- Heap dumps to compare the objects present before and after representative load. +- Garbage collection profiles to measure time spent collecting objects. +- Scheduler and thread dumps to correlate memory growth with stuck or unusually long-running work. + +These operations run inside the selected worker. They are intended for diagnosis rather than continuous monitoring. + +## Finding a Worker + +Run Bake from the service root where `supervisor.ipc` is located, then list the registered workers: + +```bash +$ bake async:service:supervisor:workers +``` + +The result contains supervisor-assigned connection IDs. These IDs are not process IDs and may change whenever a worker or supervisor restarts. List the workers again if an operation reports that the connection no longer exists. + +When workers are interchangeable, choose one ID and use that same worker throughout an investigation. If a worker restarts between snapshots, its replacement has a new heap and a new connection ID, so the snapshots are not directly comparable. + +## Capturing a Heap Dump + +Choose a path that is writable from the worker and has enough free space: + +```bash +$ bake async:service:supervisor:memory_dump \ + connection_id=1 \ + path=/var/tmp/worker-1-before.json +``` + +The returned path identifies the file written by the worker: + +```text +{:path=>"/var/tmp/worker-1-before.json"} +``` + +The path is resolved in the worker's filesystem namespace, not the shell running Bake. For a containerized or remote worker, retrieve the file from that worker or write it to a shared volume. + +Ruby heap dumps use newline-delimited JSON: each line describes an object, root, or heap record. They are not a single JSON array. + +### Operational Safety + +A full heap dump calls `ObjectSpace.dump_all`. Before running it in production, consider the following: + +- Dumping a large heap is slow and blocks the selected worker while Ruby walks its object space. +- The output can be much larger than the worker's resident memory. Check available disk space first. +- Heap records may contain application strings and other sensitive data. Store and transfer dumps accordingly. +- Avoid capturing every worker at once. Start with one representative worker during a low-risk period. +- The dump includes objects present at that instant, including garbage that Ruby has not collected yet. For retained-growth comparisons, capture snapshots shortly after comparable GC activity when possible. + +## Comparing Snapshots + +Capture a baseline, exercise the suspected workload, then capture the same worker again: + +```bash +$ bake async:service:supervisor:memory_dump \ + connection_id=1 \ + path=/var/tmp/worker-1-before.json + +# Run representative traffic or wait through the suspected growth period. + +$ bake async:service:supervisor:memory_dump \ + connection_id=1 \ + path=/var/tmp/worker-1-after.json +``` + +Start by summarizing object counts and shallow memory size by Ruby heap type. The following shell function processes the dump as a stream, so `jq` does not need to load the entire heap into memory: + +```bash +summarize_heap() { + jq -r 'select(.type != null) | [.type, (.memsize // 0)] | @tsv' "$1" | + awk -F '\t' ' + {count[$1] += 1; bytes[$1] += $2} + END { + for (type in count) + printf "%s\t%d\t%d\n", type, count[type], bytes[type] + } + ' | + sort +} + +summarize_heap /var/tmp/worker-1-before.json > before.types +summarize_heap /var/tmp/worker-1-after.json > after.types +diff -u before.types after.types +``` + +The columns are heap type, object count, and total shallow `memsize`. Large increases in types such as `STRING`, `ARRAY`, `HASH`, `OBJECT`, or `DATA` provide a direction for deeper analysis. The `class` field is an address; heap-analysis tools resolve it through the corresponding `CLASS` record to recover the Ruby class name. Shallow size does not include all objects referenced by a container, so counts and reference graphs matter as much as byte totals. + +Heap records contain addresses and `references` arrays that heap-analysis tools can use to build an object graph. When investigating retention, look for: + +- Collections whose referenced object count continually grows. +- Repeated strings or payloads that should have expired. +- Objects reachable from long-lived roots, class variables, registries, caches, or queues. +- Growth that remains across multiple snapshots captured after comparable GC activity. + +A two-snapshot difference is evidence of growth, not necessarily a leak. Confirm that the same classes continue growing across several workload and GC cycles. + +## Recording Allocation Locations + +By default, a heap dump may not contain allocation source locations. To record them, enable ObjectSpace allocation tracing in the worker before the workload you want to investigate: + +```ruby +require "objspace" +ObjectSpace.trace_object_allocations_start +``` + +Allocation tracing adds runtime and memory overhead, so enable it selectively and measure its impact before using it in production. When tracing is enabled, heap records can include `file`, `line`, `method`, and allocation generation fields. + +Summarize traced allocation locations with: + +```bash +$ jq -r \ + 'select(.file != null) | [.file, (.line // 0), (.memsize // 0)] | @tsv' \ + /var/tmp/worker-1-after.json | + awk -F '\t' ' + {location = $1 ":" $2; count[location] += 1; bytes[location] += $3} + END { + for (location in count) + printf "%12d %12d %s\n", count[location], bytes[location], location + } + ' | + sort -nr | + head -50 +``` + +High allocation counts identify hot allocation sites. Compare snapshots and focus on locations whose objects remain present and continue accumulating, rather than sites that merely allocate many short-lived objects. + +## Profiling Garbage Collection + +Start the Ruby GC profiler on the same worker you are investigating: + +```bash +$ bake async:service:supervisor:garbage_profile_start connection_id=1 +``` + +Exercise the workload, then stop profiling and write the report: + +```bash +$ bake async:service:supervisor:garbage_profile_stop \ + connection_id=1 \ + path=/var/tmp/worker-1-gc.txt +``` + +The start and stop commands must target the same live connection ID. The report is written in the worker's filesystem namespace. + +GC profiling helps distinguish several patterns: + +- Growing heap size and increasing GC time can indicate retained objects or a cache without a bound. +- High GC time without retained heap growth usually indicates allocation churn rather than a leak. +- Heap growth with little GC activity may mean the workload has not yet forced collection; compare snapshots after similar GC activity. + +GC profiling records collection timing and heap statistics. It does not identify which objects retain memory, so use it alongside heap dumps. + +## Capturing Scheduler and Thread State + +Memory growth can be caused by queued work, blocked requests, or fibers retaining large request graphs. Capture scheduler and thread state from the same worker: + +```bash +$ bake async:service:supervisor:scheduler_dump \ + connection_id=1 \ + path=/var/tmp/worker-1-scheduler.txt + +$ bake async:service:supervisor:thread_dump \ + connection_id=1 \ + path=/var/tmp/worker-1-threads.txt +``` + +Without `path`, scheduler and thread dumps are returned to Bake instead of being written by the worker. A path is usually more convenient for large diagnostic output. + +Look for fibers or threads that remain in every snapshot, queues that never drain, and repeated backtraces corresponding to the workload that causes memory growth. + +## Suggested Investigation Workflow + +1. Confirm sustained growth with the `MemoryMonitor`, `ProcessMonitor`, or external process metrics. +2. List worker connection IDs and select one representative worker. +3. Capture a baseline heap dump after normal warm-up. +4. Start GC profiling. +5. Run representative traffic or wait through the suspected leak interval. +6. Capture scheduler and thread dumps if work appears stuck or backlogged. +7. Stop GC profiling and capture a second heap dump from the same worker. +8. Compare heap types, allocation locations when tracing is enabled, and retained reference paths. +9. Repeat across another workload interval to distinguish sustained retention from normal cache warm-up. + +Use the {ruby Async::Service::Supervisor::MemoryMonitor} to protect production from unbounded growth, but complete diagnostics before its configured limit restarts the worker. A restarted worker receives a new connection ID and loses the heap state you were observing. diff --git a/context/memory-monitor.md b/context/memory-monitor.md index f33d232..70e4a58 100644 --- a/context/memory-monitor.md +++ b/context/memory-monitor.md @@ -11,7 +11,7 @@ Use the `MemoryMonitor` when you need: - **Memory leak protection**: Automatically restart workers that continuously accumulate memory. - **Resource limits**: Enforce maximum memory usage per worker. - **System stability**: Prevent runaway processes from exhausting system memory. -- **Leak diagnosis**: Capture memory samples when leaks are detected for debugging. +- **Leak diagnosis**: Identify workers that should be investigated with heap diagnostics. The monitor uses the `memory-leak` gem to track process memory usage over time, detecting abnormal growth patterns that indicate leaks. @@ -39,9 +39,8 @@ end When a worker exceeds the limit: 1. The monitor logs the leak detection. -2. Optionally captures a memory sample for debugging. -3. Sends `SIGINT` to gracefully shut down the worker. -4. The container automatically spawns a replacement worker. +2. Sends `SIGINT` to gracefully shut down the worker. +3. The container automatically spawns a replacement worker. ## Configuration Options @@ -79,51 +78,21 @@ Async::Service::Supervisor::MemoryMonitor.new( ) ``` -### `memory_sample` - -Options for capturing memory samples when a leak is detected. If `nil`, memory sampling is disabled. - -Default: `{duration: 30, timeout: 120}` - -```ruby -# Customize memory sampling: -Async::Service::Supervisor::MemoryMonitor.new( - memory_sample: { - duration: 60, # Sample for 60 seconds - timeout: 180 # Timeout after 180 seconds - } -) - -# Disable memory sampling: -Async::Service::Supervisor::MemoryMonitor.new( - memory_sample: nil -) -``` - ## Memory Leak Detection When a memory leak is detected, the monitor will: 1. Log the leak detection with process details. -2. If `memory_sample` is configured, capture a memory sample from the worker. -3. Send a `SIGINT` signal to gracefully restart the worker. -4. The container will automatically restart the worker process. - -### Memory Sampling +2. Send a `SIGINT` signal to gracefully restart the worker. +3. The container will automatically restart the worker process. -When a memory leak is detected and `memory_sample` is configured, the monitor requests a lightweight memory sample from the worker. This sample: +### Heap Diagnostics -- Tracks allocations during the sampling period. -- Forces a garbage collection. -- Returns a JSON report showing retained objects. +The monitor does not automatically capture heap data before restarting a worker. When investigating growth, use the worker diagnostic Bake tasks to: -The report includes: -- `total_allocated`: Total allocated memory and object count. -- `total_retained`: Total retained memory and count after GC. -- `by_gem`: Breakdown by gem/library. -- `by_file`: Breakdown by source file. -- `by_location`: Breakdown by specific file:line locations. -- `by_class`: Breakdown by object class. -- `strings`: String allocation analysis. +- List live worker connection IDs. +- Capture full `ObjectSpace` heap dumps before and after representative load. +- Record garbage collection profiles. +- Capture scheduler and thread state. -This is much more efficient than a full heap dump using `ObjectSpace.dump_all`. +See the [Memory Diagnostics](../memory-diagnostics/index) guide for a safe capture and comparison workflow. Complete the diagnostic capture before the configured limit restarts the worker. diff --git a/context/migration.md b/context/migration.md index b020335..74ff6c6 100644 --- a/context/migration.md +++ b/context/migration.md @@ -223,7 +223,7 @@ $ bake async:service:supervisor:reload $ bake async:service:supervisor:status ``` -**Note:** The `memory_sample` bake task has been removed in `async-service-supervisor`. This functionality was removed as it wasn't very useful and added complexity. If you were using memory sampling, you'll need to find alternative approaches for memory leak detection, such as using the `MemoryMonitor` with appropriate limits or external memory profiling tools. +**Note:** The `memory_sample` Bake task has been removed in `async-service-supervisor`. Use the `MemoryMonitor` to detect sustained growth and the worker diagnostic Bake tasks to capture heap dumps and GC profiles. See the [Memory Diagnostics](../memory-diagnostics/index) guide for the recommended workflow. ### 8. Update Programmatic Client Usage diff --git a/guides/getting-started/readme.md b/guides/getting-started/readme.md index 6e848ca..87d7810 100644 --- a/guides/getting-started/readme.md +++ b/guides/getting-started/readme.md @@ -149,44 +149,28 @@ end The supervisor can collect various diagnostics from workers on demand: - **Memory dumps**: Full heap dumps for memory analysis via `ObjectSpace.dump_all`. -- **Memory samples**: Lightweight sampling to identify memory leaks. - **Thread dumps**: Stack traces of all threads. -- **Scheduler dumps**: Async fiber hierarchy -- **Garbage collection profiles**: GC performance data +- **Scheduler dumps**: Async fiber hierarchy. +- **Garbage collection profiles**: GC performance data. -These can be triggered programmatically or via command-line tools (when available). +These can be triggered programmatically or with Bake tasks. #### Memory Leak Diagnosis -To identify memory leaks, you can use the memory sampling feature which is much lighter weight than a full memory dump. It tracks allocations over a time period and focuses on retained objects. - -**Using the bake task:** +Start by listing the workers registered with the supervisor: ```bash -# Sample for 30 seconds and print report to console -$ bake async:container:supervisor:memory_sample duration=30 +$ bake async:service:supervisor:workers ``` -**Programmatically:** +Then capture heap dumps from the same worker before and after the suspected growth period: -```ruby -# Assuming you have a connection to a worker: -result = connection.call(do: :memory_sample, duration: 30) -puts result[:data] +```bash +$ bake async:service:supervisor:memory_dump connection_id=1 path=/var/tmp/worker-before.json +$ bake async:service:supervisor:memory_dump connection_id=1 path=/var/tmp/worker-after.json ``` -This will sample memory allocations for the specified duration, then force a garbage collection and return a JSON report showing what objects were allocated during that period and retained after GC. Late-lifecycle allocations that are retained are likely memory leaks. - -The JSON report includes: -- `total_allocated`: Total allocated memory and count -- `total_retained`: Total retained memory and count -- `by_gem`: Breakdown by gem/library -- `by_file`: Breakdown by source file -- `by_location`: Breakdown by specific file:line locations -- `by_class`: Breakdown by object class -- `strings`: String allocation analysis - -This is much more efficient than `do: :memory_dump` which uses `ObjectSpace.dump_all` and can be slow and blocking on large heaps. The JSON format also makes it easy to integrate with monitoring and analysis tools. +Heap dumps are heavyweight and may contain sensitive application data. See the [Memory Diagnostics](../memory-diagnostics/index) guide for the complete capture, comparison, and GC profiling workflow. ## Advanced Usage diff --git a/guides/links.yaml b/guides/links.yaml index 7d08498..80b41c2 100644 --- a/guides/links.yaml +++ b/guides/links.yaml @@ -7,8 +7,11 @@ migration: memory-monitor: order: 3 -process-monitor: +memory-diagnostics: order: 4 -utilization-monitor: +process-monitor: order: 5 + +utilization-monitor: + order: 6 diff --git a/guides/memory-diagnostics/readme.md b/guides/memory-diagnostics/readme.md new file mode 100644 index 0000000..d482c23 --- /dev/null +++ b/guides/memory-diagnostics/readme.md @@ -0,0 +1,191 @@ +# Memory Diagnostics + +This guide explains how to capture Ruby heap dumps and garbage collection profiles from supervised workers, then use them to investigate memory growth. + +## Overview + +Use the diagnostic Bake tasks when a worker's memory grows unexpectedly and you need to understand what it is retaining. A typical investigation combines: + +- {ruby Async::Service::Supervisor::MemoryMonitor} or process metrics to identify sustained growth. +- Heap dumps to compare the objects present before and after representative load. +- Garbage collection profiles to measure time spent collecting objects. +- Scheduler and thread dumps to correlate memory growth with stuck or unusually long-running work. + +These operations run inside the selected worker. They are intended for diagnosis rather than continuous monitoring. + +## Finding a Worker + +Run Bake from the service root where `supervisor.ipc` is located, then list the registered workers: + +```bash +$ bake async:service:supervisor:workers +``` + +The result contains supervisor-assigned connection IDs. These IDs are not process IDs and may change whenever a worker or supervisor restarts. List the workers again if an operation reports that the connection no longer exists. + +When workers are interchangeable, choose one ID and use that same worker throughout an investigation. If a worker restarts between snapshots, its replacement has a new heap and a new connection ID, so the snapshots are not directly comparable. + +## Capturing a Heap Dump + +Choose a path that is writable from the worker and has enough free space: + +```bash +$ bake async:service:supervisor:memory_dump \ + connection_id=1 \ + path=/var/tmp/worker-1-before.json +``` + +The returned path identifies the file written by the worker: + +```text +{:path=>"/var/tmp/worker-1-before.json"} +``` + +The path is resolved in the worker's filesystem namespace, not the shell running Bake. For a containerized or remote worker, retrieve the file from that worker or write it to a shared volume. + +Ruby heap dumps use newline-delimited JSON: each line describes an object, root, or heap record. They are not a single JSON array. + +### Operational Safety + +A full heap dump calls `ObjectSpace.dump_all`. Before running it in production, consider the following: + +- Dumping a large heap is slow and blocks the selected worker while Ruby walks its object space. +- The output can be much larger than the worker's resident memory. Check available disk space first. +- Heap records may contain application strings and other sensitive data. Store and transfer dumps accordingly. +- Avoid capturing every worker at once. Start with one representative worker during a low-risk period. +- The dump includes objects present at that instant, including garbage that Ruby has not collected yet. For retained-growth comparisons, capture snapshots shortly after comparable GC activity when possible. + +## Comparing Snapshots + +Capture a baseline, exercise the suspected workload, then capture the same worker again: + +```bash +$ bake async:service:supervisor:memory_dump \ + connection_id=1 \ + path=/var/tmp/worker-1-before.json + +# Run representative traffic or wait through the suspected growth period. + +$ bake async:service:supervisor:memory_dump \ + connection_id=1 \ + path=/var/tmp/worker-1-after.json +``` + +Start by summarizing object counts and shallow memory size by Ruby heap type. The following shell function processes the dump as a stream, so `jq` does not need to load the entire heap into memory: + +```bash +summarize_heap() { + jq -r 'select(.type != null) | [.type, (.memsize // 0)] | @tsv' "$1" | + awk -F '\t' ' + {count[$1] += 1; bytes[$1] += $2} + END { + for (type in count) + printf "%s\t%d\t%d\n", type, count[type], bytes[type] + } + ' | + sort +} + +summarize_heap /var/tmp/worker-1-before.json > before.types +summarize_heap /var/tmp/worker-1-after.json > after.types +diff -u before.types after.types +``` + +The columns are heap type, object count, and total shallow `memsize`. Large increases in types such as `STRING`, `ARRAY`, `HASH`, `OBJECT`, or `DATA` provide a direction for deeper analysis. The `class` field is an address; heap-analysis tools resolve it through the corresponding `CLASS` record to recover the Ruby class name. Shallow size does not include all objects referenced by a container, so counts and reference graphs matter as much as byte totals. + +Heap records contain addresses and `references` arrays that heap-analysis tools can use to build an object graph. When investigating retention, look for: + +- Collections whose referenced object count continually grows. +- Repeated strings or payloads that should have expired. +- Objects reachable from long-lived roots, class variables, registries, caches, or queues. +- Growth that remains across multiple snapshots captured after comparable GC activity. + +A two-snapshot difference is evidence of growth, not necessarily a leak. Confirm that the same classes continue growing across several workload and GC cycles. + +## Recording Allocation Locations + +By default, a heap dump may not contain allocation source locations. To record them, enable ObjectSpace allocation tracing in the worker before the workload you want to investigate: + +```ruby +require "objspace" +ObjectSpace.trace_object_allocations_start +``` + +Allocation tracing adds runtime and memory overhead, so enable it selectively and measure its impact before using it in production. When tracing is enabled, heap records can include `file`, `line`, `method`, and allocation generation fields. + +Summarize traced allocation locations with: + +```bash +$ jq -r \ + 'select(.file != null) | [.file, (.line // 0), (.memsize // 0)] | @tsv' \ + /var/tmp/worker-1-after.json | + awk -F '\t' ' + {location = $1 ":" $2; count[location] += 1; bytes[location] += $3} + END { + for (location in count) + printf "%12d %12d %s\n", count[location], bytes[location], location + } + ' | + sort -nr | + head -50 +``` + +High allocation counts identify hot allocation sites. Compare snapshots and focus on locations whose objects remain present and continue accumulating, rather than sites that merely allocate many short-lived objects. + +## Profiling Garbage Collection + +Start the Ruby GC profiler on the same worker you are investigating: + +```bash +$ bake async:service:supervisor:garbage_profile_start connection_id=1 +``` + +Exercise the workload, then stop profiling and write the report: + +```bash +$ bake async:service:supervisor:garbage_profile_stop \ + connection_id=1 \ + path=/var/tmp/worker-1-gc.txt +``` + +The start and stop commands must target the same live connection ID. The report is written in the worker's filesystem namespace. + +GC profiling helps distinguish several patterns: + +- Growing heap size and increasing GC time can indicate retained objects or a cache without a bound. +- High GC time without retained heap growth usually indicates allocation churn rather than a leak. +- Heap growth with little GC activity may mean the workload has not yet forced collection; compare snapshots after similar GC activity. + +GC profiling records collection timing and heap statistics. It does not identify which objects retain memory, so use it alongside heap dumps. + +## Capturing Scheduler and Thread State + +Memory growth can be caused by queued work, blocked requests, or fibers retaining large request graphs. Capture scheduler and thread state from the same worker: + +```bash +$ bake async:service:supervisor:scheduler_dump \ + connection_id=1 \ + path=/var/tmp/worker-1-scheduler.txt + +$ bake async:service:supervisor:thread_dump \ + connection_id=1 \ + path=/var/tmp/worker-1-threads.txt +``` + +Without `path`, scheduler and thread dumps are returned to Bake instead of being written by the worker. A path is usually more convenient for large diagnostic output. + +Look for fibers or threads that remain in every snapshot, queues that never drain, and repeated backtraces corresponding to the workload that causes memory growth. + +## Suggested Investigation Workflow + +1. Confirm sustained growth with the `MemoryMonitor`, `ProcessMonitor`, or external process metrics. +2. List worker connection IDs and select one representative worker. +3. Capture a baseline heap dump after normal warm-up. +4. Start GC profiling. +5. Run representative traffic or wait through the suspected leak interval. +6. Capture scheduler and thread dumps if work appears stuck or backlogged. +7. Stop GC profiling and capture a second heap dump from the same worker. +8. Compare heap types, allocation locations when tracing is enabled, and retained reference paths. +9. Repeat across another workload interval to distinguish sustained retention from normal cache warm-up. + +Use the {ruby Async::Service::Supervisor::MemoryMonitor} to protect production from unbounded growth, but complete diagnostics before its configured limit restarts the worker. A restarted worker receives a new connection ID and loses the heap state you were observing. diff --git a/guides/memory-monitor/readme.md b/guides/memory-monitor/readme.md index f33d232..70e4a58 100644 --- a/guides/memory-monitor/readme.md +++ b/guides/memory-monitor/readme.md @@ -11,7 +11,7 @@ Use the `MemoryMonitor` when you need: - **Memory leak protection**: Automatically restart workers that continuously accumulate memory. - **Resource limits**: Enforce maximum memory usage per worker. - **System stability**: Prevent runaway processes from exhausting system memory. -- **Leak diagnosis**: Capture memory samples when leaks are detected for debugging. +- **Leak diagnosis**: Identify workers that should be investigated with heap diagnostics. The monitor uses the `memory-leak` gem to track process memory usage over time, detecting abnormal growth patterns that indicate leaks. @@ -39,9 +39,8 @@ end When a worker exceeds the limit: 1. The monitor logs the leak detection. -2. Optionally captures a memory sample for debugging. -3. Sends `SIGINT` to gracefully shut down the worker. -4. The container automatically spawns a replacement worker. +2. Sends `SIGINT` to gracefully shut down the worker. +3. The container automatically spawns a replacement worker. ## Configuration Options @@ -79,51 +78,21 @@ Async::Service::Supervisor::MemoryMonitor.new( ) ``` -### `memory_sample` - -Options for capturing memory samples when a leak is detected. If `nil`, memory sampling is disabled. - -Default: `{duration: 30, timeout: 120}` - -```ruby -# Customize memory sampling: -Async::Service::Supervisor::MemoryMonitor.new( - memory_sample: { - duration: 60, # Sample for 60 seconds - timeout: 180 # Timeout after 180 seconds - } -) - -# Disable memory sampling: -Async::Service::Supervisor::MemoryMonitor.new( - memory_sample: nil -) -``` - ## Memory Leak Detection When a memory leak is detected, the monitor will: 1. Log the leak detection with process details. -2. If `memory_sample` is configured, capture a memory sample from the worker. -3. Send a `SIGINT` signal to gracefully restart the worker. -4. The container will automatically restart the worker process. - -### Memory Sampling +2. Send a `SIGINT` signal to gracefully restart the worker. +3. The container will automatically restart the worker process. -When a memory leak is detected and `memory_sample` is configured, the monitor requests a lightweight memory sample from the worker. This sample: +### Heap Diagnostics -- Tracks allocations during the sampling period. -- Forces a garbage collection. -- Returns a JSON report showing retained objects. +The monitor does not automatically capture heap data before restarting a worker. When investigating growth, use the worker diagnostic Bake tasks to: -The report includes: -- `total_allocated`: Total allocated memory and object count. -- `total_retained`: Total retained memory and count after GC. -- `by_gem`: Breakdown by gem/library. -- `by_file`: Breakdown by source file. -- `by_location`: Breakdown by specific file:line locations. -- `by_class`: Breakdown by object class. -- `strings`: String allocation analysis. +- List live worker connection IDs. +- Capture full `ObjectSpace` heap dumps before and after representative load. +- Record garbage collection profiles. +- Capture scheduler and thread state. -This is much more efficient than a full heap dump using `ObjectSpace.dump_all`. +See the [Memory Diagnostics](../memory-diagnostics/index) guide for a safe capture and comparison workflow. Complete the diagnostic capture before the configured limit restarts the worker. diff --git a/guides/migration/readme.md b/guides/migration/readme.md index b020335..74ff6c6 100644 --- a/guides/migration/readme.md +++ b/guides/migration/readme.md @@ -223,7 +223,7 @@ $ bake async:service:supervisor:reload $ bake async:service:supervisor:status ``` -**Note:** The `memory_sample` bake task has been removed in `async-service-supervisor`. This functionality was removed as it wasn't very useful and added complexity. If you were using memory sampling, you'll need to find alternative approaches for memory leak detection, such as using the `MemoryMonitor` with appropriate limits or external memory profiling tools. +**Note:** The `memory_sample` Bake task has been removed in `async-service-supervisor`. Use the `MemoryMonitor` to detect sustained growth and the worker diagnostic Bake tasks to capture heap dumps and GC profiles. See the [Memory Diagnostics](../memory-diagnostics/index) guide for the recommended workflow. ### 8. Update Programmatic Client Usage diff --git a/readme.md b/readme.md index ba798dc..ad9cc02 100644 --- a/readme.md +++ b/readme.md @@ -20,6 +20,8 @@ Please see the [project documentation](https://socketry.github.io/async-service- - [Memory Monitor](https://socketry.github.io/async-service-supervisor/guides/memory-monitor/index) - This guide explains how to use the Async::Service::Supervisor::MemoryMonitor to detect and restart workers that exceed memory limits or develop memory leaks. + - [Memory Diagnostics](https://socketry.github.io/async-service-supervisor/guides/memory-diagnostics/index) - This guide explains how to capture Ruby heap dumps and garbage collection profiles from supervised workers, then use them to investigate memory growth. + - [Process Monitor](https://socketry.github.io/async-service-supervisor/guides/process-monitor/index) - This guide explains how to use the Async::Service::Supervisor::ProcessMonitor to log CPU and memory metrics for your worker processes. - [Utilization Monitor](https://socketry.github.io/async-service-supervisor/guides/utilization-monitor/index) - This guide explains how to use the Async::Service::Supervisor::UtilizationMonitor to collect and aggregate application-level utilization metrics from your worker processes. From 258989eb69673c467f73352c74f2c6ed1b0858fd Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 29 Jul 2026 13:37:48 +1200 Subject: [PATCH 7/9] Recommend heap-profiler for heap analysis Assisted-By: devx/d15f74b3-893f-4846-bbae-4dcdd10e8b8a --- context/memory-diagnostics.md | 57 ++++++++--------------------- guides/memory-diagnostics/readme.md | 57 ++++++++--------------------- 2 files changed, 32 insertions(+), 82 deletions(-) diff --git a/context/memory-diagnostics.md b/context/memory-diagnostics.md index d482c23..25d1bbc 100644 --- a/context/memory-diagnostics.md +++ b/context/memory-diagnostics.md @@ -55,7 +55,7 @@ A full heap dump calls `ObjectSpace.dump_all`. Before running it in production, - Avoid capturing every worker at once. Start with one representative worker during a low-risk period. - The dump includes objects present at that instant, including garbage that Ruby has not collected yet. For retained-growth comparisons, capture snapshots shortly after comparable GC activity when possible. -## Comparing Snapshots +## Analyzing Heap Dumps Capture a baseline, exercise the suspected workload, then capture the same worker again: @@ -71,36 +71,28 @@ $ bake async:service:supervisor:memory_dump \ path=/var/tmp/worker-1-after.json ``` -Start by summarizing object counts and shallow memory size by Ruby heap type. The following shell function processes the dump as a stream, so `jq` does not need to load the entire heap into memory: +Use [Shopify's `heap-profiler`](https://github.com/Shopify/heap-profiler) to turn each heap dump into a report. It reads `ObjectSpace.dump_all` output directly and summarizes memory and object counts by class, gem, file, and allocation location. Install it on the system where you will analyze the dumps; it does not need to be installed in the worker: ```bash -summarize_heap() { - jq -r 'select(.type != null) | [.type, (.memsize // 0)] | @tsv' "$1" | - awk -F '\t' ' - {count[$1] += 1; bytes[$1] += $2} - END { - for (type in count) - printf "%s\t%d\t%d\n", type, count[type], bytes[type] - } - ' | - sort -} - -summarize_heap /var/tmp/worker-1-before.json > before.types -summarize_heap /var/tmp/worker-1-after.json > after.types -diff -u before.types after.types +$ gem install heap-profiler +$ heap-profiler /var/tmp/worker-1-before.json +$ heap-profiler /var/tmp/worker-1-after.json ``` -The columns are heap type, object count, and total shallow `memsize`. Large increases in types such as `STRING`, `ARRAY`, `HASH`, `OBJECT`, or `DATA` provide a direction for deeper analysis. The `class` field is an address; heap-analysis tools resolve it through the corresponding `CLASS` record to recover the Ruby class name. Shallow size does not include all objects referenced by a container, so counts and reference graphs matter as much as byte totals. +Use `--max` to show more entries when the default report is too short: -Heap records contain addresses and `references` arrays that heap-analysis tools can use to build an object graph. When investigating retention, look for: +```bash +$ heap-profiler --max=100 /var/tmp/worker-1-after.json +``` -- Collections whose referenced object count continually grows. +Compare the reports for classes, locations, or repeated strings whose object count and memory continue to increase. In particular, look for: + +- Collection classes whose object count or shallow memory continually grows. - Repeated strings or payloads that should have expired. -- Objects reachable from long-lived roots, class variables, registries, caches, or queues. +- Application classes associated with registries, caches, queues, or other long-lived state. - Growth that remains across multiple snapshots captured after comparable GC activity. -A two-snapshot difference is evidence of growth, not necessarily a leak. Confirm that the same classes continue growing across several workload and GC cycles. +The reported memory is shallow size and does not include every object referenced by a container. Treat a two-snapshot increase as evidence of growth, not necessarily a leak. Capture a third snapshot after another comparable workload and GC cycle to confirm that the same classes or allocation locations continue growing. ## Recording Allocation Locations @@ -111,26 +103,9 @@ require "objspace" ObjectSpace.trace_object_allocations_start ``` -Allocation tracing adds runtime and memory overhead, so enable it selectively and measure its impact before using it in production. When tracing is enabled, heap records can include `file`, `line`, `method`, and allocation generation fields. - -Summarize traced allocation locations with: - -```bash -$ jq -r \ - 'select(.file != null) | [.file, (.line // 0), (.memsize // 0)] | @tsv' \ - /var/tmp/worker-1-after.json | - awk -F '\t' ' - {location = $1 ":" $2; count[location] += 1; bytes[location] += $3} - END { - for (location in count) - printf "%12d %12d %s\n", count[location], bytes[location], location - } - ' | - sort -nr | - head -50 -``` +Allocation tracing adds runtime and memory overhead, so enable it selectively and measure its impact before using it in production. When tracing is enabled, heap records can include `file`, `line`, `method`, and allocation generation fields. `heap-profiler` automatically includes breakdowns by gem, file, and location when this information is present. -High allocation counts identify hot allocation sites. Compare snapshots and focus on locations whose objects remain present and continue accumulating, rather than sites that merely allocate many short-lived objects. +High allocation counts identify hot allocation sites. Compare the `heap-profiler` reports and focus on locations whose objects remain present and continue accumulating, rather than sites that merely allocate many short-lived objects. ## Profiling Garbage Collection diff --git a/guides/memory-diagnostics/readme.md b/guides/memory-diagnostics/readme.md index d482c23..25d1bbc 100644 --- a/guides/memory-diagnostics/readme.md +++ b/guides/memory-diagnostics/readme.md @@ -55,7 +55,7 @@ A full heap dump calls `ObjectSpace.dump_all`. Before running it in production, - Avoid capturing every worker at once. Start with one representative worker during a low-risk period. - The dump includes objects present at that instant, including garbage that Ruby has not collected yet. For retained-growth comparisons, capture snapshots shortly after comparable GC activity when possible. -## Comparing Snapshots +## Analyzing Heap Dumps Capture a baseline, exercise the suspected workload, then capture the same worker again: @@ -71,36 +71,28 @@ $ bake async:service:supervisor:memory_dump \ path=/var/tmp/worker-1-after.json ``` -Start by summarizing object counts and shallow memory size by Ruby heap type. The following shell function processes the dump as a stream, so `jq` does not need to load the entire heap into memory: +Use [Shopify's `heap-profiler`](https://github.com/Shopify/heap-profiler) to turn each heap dump into a report. It reads `ObjectSpace.dump_all` output directly and summarizes memory and object counts by class, gem, file, and allocation location. Install it on the system where you will analyze the dumps; it does not need to be installed in the worker: ```bash -summarize_heap() { - jq -r 'select(.type != null) | [.type, (.memsize // 0)] | @tsv' "$1" | - awk -F '\t' ' - {count[$1] += 1; bytes[$1] += $2} - END { - for (type in count) - printf "%s\t%d\t%d\n", type, count[type], bytes[type] - } - ' | - sort -} - -summarize_heap /var/tmp/worker-1-before.json > before.types -summarize_heap /var/tmp/worker-1-after.json > after.types -diff -u before.types after.types +$ gem install heap-profiler +$ heap-profiler /var/tmp/worker-1-before.json +$ heap-profiler /var/tmp/worker-1-after.json ``` -The columns are heap type, object count, and total shallow `memsize`. Large increases in types such as `STRING`, `ARRAY`, `HASH`, `OBJECT`, or `DATA` provide a direction for deeper analysis. The `class` field is an address; heap-analysis tools resolve it through the corresponding `CLASS` record to recover the Ruby class name. Shallow size does not include all objects referenced by a container, so counts and reference graphs matter as much as byte totals. +Use `--max` to show more entries when the default report is too short: -Heap records contain addresses and `references` arrays that heap-analysis tools can use to build an object graph. When investigating retention, look for: +```bash +$ heap-profiler --max=100 /var/tmp/worker-1-after.json +``` -- Collections whose referenced object count continually grows. +Compare the reports for classes, locations, or repeated strings whose object count and memory continue to increase. In particular, look for: + +- Collection classes whose object count or shallow memory continually grows. - Repeated strings or payloads that should have expired. -- Objects reachable from long-lived roots, class variables, registries, caches, or queues. +- Application classes associated with registries, caches, queues, or other long-lived state. - Growth that remains across multiple snapshots captured after comparable GC activity. -A two-snapshot difference is evidence of growth, not necessarily a leak. Confirm that the same classes continue growing across several workload and GC cycles. +The reported memory is shallow size and does not include every object referenced by a container. Treat a two-snapshot increase as evidence of growth, not necessarily a leak. Capture a third snapshot after another comparable workload and GC cycle to confirm that the same classes or allocation locations continue growing. ## Recording Allocation Locations @@ -111,26 +103,9 @@ require "objspace" ObjectSpace.trace_object_allocations_start ``` -Allocation tracing adds runtime and memory overhead, so enable it selectively and measure its impact before using it in production. When tracing is enabled, heap records can include `file`, `line`, `method`, and allocation generation fields. - -Summarize traced allocation locations with: - -```bash -$ jq -r \ - 'select(.file != null) | [.file, (.line // 0), (.memsize // 0)] | @tsv' \ - /var/tmp/worker-1-after.json | - awk -F '\t' ' - {location = $1 ":" $2; count[location] += 1; bytes[location] += $3} - END { - for (location in count) - printf "%12d %12d %s\n", count[location], bytes[location], location - } - ' | - sort -nr | - head -50 -``` +Allocation tracing adds runtime and memory overhead, so enable it selectively and measure its impact before using it in production. When tracing is enabled, heap records can include `file`, `line`, `method`, and allocation generation fields. `heap-profiler` automatically includes breakdowns by gem, file, and location when this information is present. -High allocation counts identify hot allocation sites. Compare snapshots and focus on locations whose objects remain present and continue accumulating, rather than sites that merely allocate many short-lived objects. +High allocation counts identify hot allocation sites. Compare the `heap-profiler` reports and focus on locations whose objects remain present and continue accumulating, rather than sites that merely allocate many short-lived objects. ## Profiling Garbage Collection From 43ec3b3b37cb0332ec99cef7221486e28b14145c Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 29 Jul 2026 13:50:39 +1200 Subject: [PATCH 8/9] Document heap analysis tools Assisted-By: devx/d15f74b3-893f-4846-bbae-4dcdd10e8b8a --- bake/async/service/supervisor.rb | 5 +- context/memory-diagnostics.md | 118 ++++++++++++++---- guides/memory-diagnostics/readme.md | 118 ++++++++++++++---- .../service/supervisor/worker_controller.rb | 6 +- test/bake/async/service/supervisor.rb | 3 +- 5 files changed, 200 insertions(+), 50 deletions(-) diff --git a/bake/async/service/supervisor.rb b/bake/async/service/supervisor.rb index a204b8a..1719ad6 100644 --- a/bake/async/service/supervisor.rb +++ b/bake/async/service/supervisor.rb @@ -44,9 +44,10 @@ def workers # # @parameter connection_id [Integer] The connection ID of the worker to target. # @parameter path [String] The file path where the worker should write the dump. -def memory_dump(connection_id:, path:) +# @parameter shapes [Boolean] Whether to include Ruby shape-tree records. +def memory_dump(connection_id:, path:, shapes: true) with_worker(connection_id) do |worker| - worker.memory_dump(path: path) + worker.memory_dump(path: path, shapes: shapes) end end diff --git a/context/memory-diagnostics.md b/context/memory-diagnostics.md index 25d1bbc..15eef2b 100644 --- a/context/memory-diagnostics.md +++ b/context/memory-diagnostics.md @@ -57,42 +57,115 @@ A full heap dump calls `ObjectSpace.dump_all`. Before running it in production, ## Analyzing Heap Dumps -Capture a baseline, exercise the suspected workload, then capture the same worker again: +Different tools answer different questions about a heap dump: + +| Tool | Best for | Snapshots | +| --- | --- | --- | +| [`heap-profiler`](https://github.com/Shopify/heap-profiler) | Aggregate memory and object counts by class, gem, file, and location | One | +| [`sheap`](https://github.com/jhawthorn/sheap) | Finding objects retained across snapshots and tracing paths back to roots | Two or three | +| [Reap](https://github.com/oxidize-rb/reap) | Finding objects that dominate and retain large portions of a heap | One | + +Install and run these tools on a trusted analysis system after retrieving the dumps from the worker. The dumps can contain sensitive application data. + +### Aggregate Reports with `heap-profiler` + +Capture a heap dump and pass it directly to `heap-profiler`: ```bash $ bake async:service:supervisor:memory_dump \ connection_id=1 \ - path=/var/tmp/worker-1-before.json + path=/var/tmp/worker-1.json -# Run representative traffic or wait through the suspected growth period. +$ gem install heap-profiler +$ heap-profiler /var/tmp/worker-1.json +``` -$ bake async:service:supervisor:memory_dump \ - connection_id=1 \ - path=/var/tmp/worker-1-after.json +Use `--max` to show more entries when the default report is too short: + +```bash +$ heap-profiler --max=100 /var/tmp/worker-1.json ``` -Use [Shopify's `heap-profiler`](https://github.com/Shopify/heap-profiler) to turn each heap dump into a report. It reads `ObjectSpace.dump_all` output directly and summarizes memory and object counts by class, gem, file, and allocation location. Install it on the system where you will analyze the dumps; it does not need to be installed in the worker: +The report is a useful overview of the largest classes, strings, and allocation locations in one snapshot. It does not calculate a delta between arbitrary Bake snapshots. Capture reports at comparable points in the workload if you want to compare their aggregate counts manually. + +### Retention Diffs with `sheap` + +Capture a baseline and two later snapshots from the same worker. The third snapshot distinguishes objects that remain retained from temporary allocations present only in the second dump: ```bash -$ gem install heap-profiler -$ heap-profiler /var/tmp/worker-1-before.json -$ heap-profiler /var/tmp/worker-1-after.json +$ bake async:service:supervisor:memory_dump connection_id=1 path=/var/tmp/worker-1-before.json + +# Run representative traffic or wait through the suspected growth period. +$ bake async:service:supervisor:memory_dump connection_id=1 path=/var/tmp/worker-1-after.json + +# Run another comparable workload and GC cycle. +$ bake async:service:supervisor:memory_dump connection_id=1 path=/var/tmp/worker-1-later.json ``` -Use `--max` to show more entries when the default report is too short: +Install `sheap` and open an interactive two-snapshot diff: ```bash -$ heap-profiler --max=100 /var/tmp/worker-1-after.json +$ gem install sheap +$ sheap /var/tmp/worker-1-before.json /var/tmp/worker-1-after.json +``` + +The command opens IRB with `$before`, `$after`, and `$diff` available. For example: + +```ruby +# Count newly retained objects by Ruby heap type: +$diff.retained.map(&:type_str).tally.sort_by(&:last).last(20) + +# Inspect a large retained collection and find its path from a heap root: +large_array = $diff.retained.arrays.max_by(&:length) +$after.find_path(large_array) ``` -Compare the reports for classes, locations, or repeated strings whose object count and memory continue to increase. In particular, look for: +Use the library API for a three-snapshot diff: + +```ruby +three_way = Sheap::Diff.new( + "/var/tmp/worker-1-before.json", + "/var/tmp/worker-1-after.json", + "/var/tmp/worker-1-later.json" +) + +three_way.retained.map(&:type_str).tally.sort_by(&:last).last(20) +``` + +`sheap` identifies objects by heap address and type. Heap compaction can move objects, while freed addresses can be reused, so disable automatic compaction during the investigation and prefer a three-snapshot diff. Always compare dumps from the same live worker. + +Look for: - Collection classes whose object count or shallow memory continually grows. - Repeated strings or payloads that should have expired. -- Application classes associated with registries, caches, queues, or other long-lived state. -- Growth that remains across multiple snapshots captured after comparable GC activity. +- Paths from roots through registries, caches, queues, or other long-lived state. +- Objects that remain in the three-snapshot diff after comparable GC activity. + +### Dominator Analysis with Reap + +Reap builds a dominator tree from a single heap's reference graph. An object dominates another object when every path from a heap root to the second object passes through the first. This makes Reap useful for finding a small cache, queue, thread, or registry that keeps a much larger object graph alive. + +Reap does not currently accept Ruby's address-less `SHAPE` records. Disable them when capturing a dump for Reap: + +```bash +$ bake async:service:supervisor:memory_dump \ + connection_id=1 \ + path=/var/tmp/worker-1-reap.json \ + shapes=false +``` + +Install Reap with Cargo, then print the largest dominators and optionally generate an inverted flame graph: + +```bash +$ cargo install reap +$ reap /var/tmp/worker-1-reap.json \ + --count 20 \ + --flamegraph /var/tmp/worker-1-retained.svg +``` + +The report separates an object's shallow size from the total memory it dominates. Use `--root ADDRESS` to repeat the analysis for a suspicious subtree, or `--dot FILE` to write its dominator graph. Dumps captured with `shapes=false` remain compatible with `heap-profiler` and `sheap`. -The reported memory is shallow size and does not include every object referenced by a container. Treat a two-snapshot increase as evidence of growth, not necessarily a leak. Capture a third snapshot after another comparable workload and GC cycle to confirm that the same classes or allocation locations continue growing. +Reap does not compare snapshots or use allocation generations. Combine it with a `sheap` diff when you need both evidence of continued retention and the aggregate size of the retained graph. ## Recording Allocation Locations @@ -155,12 +228,13 @@ Look for fibers or threads that remain in every snapshot, queues that never drai 1. Confirm sustained growth with the `MemoryMonitor`, `ProcessMonitor`, or external process metrics. 2. List worker connection IDs and select one representative worker. -3. Capture a baseline heap dump after normal warm-up. -4. Start GC profiling. -5. Run representative traffic or wait through the suspected leak interval. +3. Enable allocation tracing before the workload if its overhead is acceptable. +4. Capture a baseline heap dump after normal warm-up, using `shapes=false` if you plan to use Reap. +5. Start GC profiling and run representative traffic or wait through the suspected leak interval. 6. Capture scheduler and thread dumps if work appears stuck or backlogged. -7. Stop GC profiling and capture a second heap dump from the same worker. -8. Compare heap types, allocation locations when tracing is enabled, and retained reference paths. -9. Repeat across another workload interval to distinguish sustained retention from normal cache warm-up. +7. Stop GC profiling and capture the second heap dump from the same worker. +8. Repeat the workload and capture a third dump for a stronger `sheap` retention signal. +9. Use `heap-profiler` for aggregate counts, `sheap` for retained objects and root paths, and Reap for dominator sizes. +10. Repeat the experiment if necessary to distinguish sustained retention from normal cache warm-up. Use the {ruby Async::Service::Supervisor::MemoryMonitor} to protect production from unbounded growth, but complete diagnostics before its configured limit restarts the worker. A restarted worker receives a new connection ID and loses the heap state you were observing. diff --git a/guides/memory-diagnostics/readme.md b/guides/memory-diagnostics/readme.md index 25d1bbc..15eef2b 100644 --- a/guides/memory-diagnostics/readme.md +++ b/guides/memory-diagnostics/readme.md @@ -57,42 +57,115 @@ A full heap dump calls `ObjectSpace.dump_all`. Before running it in production, ## Analyzing Heap Dumps -Capture a baseline, exercise the suspected workload, then capture the same worker again: +Different tools answer different questions about a heap dump: + +| Tool | Best for | Snapshots | +| --- | --- | --- | +| [`heap-profiler`](https://github.com/Shopify/heap-profiler) | Aggregate memory and object counts by class, gem, file, and location | One | +| [`sheap`](https://github.com/jhawthorn/sheap) | Finding objects retained across snapshots and tracing paths back to roots | Two or three | +| [Reap](https://github.com/oxidize-rb/reap) | Finding objects that dominate and retain large portions of a heap | One | + +Install and run these tools on a trusted analysis system after retrieving the dumps from the worker. The dumps can contain sensitive application data. + +### Aggregate Reports with `heap-profiler` + +Capture a heap dump and pass it directly to `heap-profiler`: ```bash $ bake async:service:supervisor:memory_dump \ connection_id=1 \ - path=/var/tmp/worker-1-before.json + path=/var/tmp/worker-1.json -# Run representative traffic or wait through the suspected growth period. +$ gem install heap-profiler +$ heap-profiler /var/tmp/worker-1.json +``` -$ bake async:service:supervisor:memory_dump \ - connection_id=1 \ - path=/var/tmp/worker-1-after.json +Use `--max` to show more entries when the default report is too short: + +```bash +$ heap-profiler --max=100 /var/tmp/worker-1.json ``` -Use [Shopify's `heap-profiler`](https://github.com/Shopify/heap-profiler) to turn each heap dump into a report. It reads `ObjectSpace.dump_all` output directly and summarizes memory and object counts by class, gem, file, and allocation location. Install it on the system where you will analyze the dumps; it does not need to be installed in the worker: +The report is a useful overview of the largest classes, strings, and allocation locations in one snapshot. It does not calculate a delta between arbitrary Bake snapshots. Capture reports at comparable points in the workload if you want to compare their aggregate counts manually. + +### Retention Diffs with `sheap` + +Capture a baseline and two later snapshots from the same worker. The third snapshot distinguishes objects that remain retained from temporary allocations present only in the second dump: ```bash -$ gem install heap-profiler -$ heap-profiler /var/tmp/worker-1-before.json -$ heap-profiler /var/tmp/worker-1-after.json +$ bake async:service:supervisor:memory_dump connection_id=1 path=/var/tmp/worker-1-before.json + +# Run representative traffic or wait through the suspected growth period. +$ bake async:service:supervisor:memory_dump connection_id=1 path=/var/tmp/worker-1-after.json + +# Run another comparable workload and GC cycle. +$ bake async:service:supervisor:memory_dump connection_id=1 path=/var/tmp/worker-1-later.json ``` -Use `--max` to show more entries when the default report is too short: +Install `sheap` and open an interactive two-snapshot diff: ```bash -$ heap-profiler --max=100 /var/tmp/worker-1-after.json +$ gem install sheap +$ sheap /var/tmp/worker-1-before.json /var/tmp/worker-1-after.json +``` + +The command opens IRB with `$before`, `$after`, and `$diff` available. For example: + +```ruby +# Count newly retained objects by Ruby heap type: +$diff.retained.map(&:type_str).tally.sort_by(&:last).last(20) + +# Inspect a large retained collection and find its path from a heap root: +large_array = $diff.retained.arrays.max_by(&:length) +$after.find_path(large_array) ``` -Compare the reports for classes, locations, or repeated strings whose object count and memory continue to increase. In particular, look for: +Use the library API for a three-snapshot diff: + +```ruby +three_way = Sheap::Diff.new( + "/var/tmp/worker-1-before.json", + "/var/tmp/worker-1-after.json", + "/var/tmp/worker-1-later.json" +) + +three_way.retained.map(&:type_str).tally.sort_by(&:last).last(20) +``` + +`sheap` identifies objects by heap address and type. Heap compaction can move objects, while freed addresses can be reused, so disable automatic compaction during the investigation and prefer a three-snapshot diff. Always compare dumps from the same live worker. + +Look for: - Collection classes whose object count or shallow memory continually grows. - Repeated strings or payloads that should have expired. -- Application classes associated with registries, caches, queues, or other long-lived state. -- Growth that remains across multiple snapshots captured after comparable GC activity. +- Paths from roots through registries, caches, queues, or other long-lived state. +- Objects that remain in the three-snapshot diff after comparable GC activity. + +### Dominator Analysis with Reap + +Reap builds a dominator tree from a single heap's reference graph. An object dominates another object when every path from a heap root to the second object passes through the first. This makes Reap useful for finding a small cache, queue, thread, or registry that keeps a much larger object graph alive. + +Reap does not currently accept Ruby's address-less `SHAPE` records. Disable them when capturing a dump for Reap: + +```bash +$ bake async:service:supervisor:memory_dump \ + connection_id=1 \ + path=/var/tmp/worker-1-reap.json \ + shapes=false +``` + +Install Reap with Cargo, then print the largest dominators and optionally generate an inverted flame graph: + +```bash +$ cargo install reap +$ reap /var/tmp/worker-1-reap.json \ + --count 20 \ + --flamegraph /var/tmp/worker-1-retained.svg +``` + +The report separates an object's shallow size from the total memory it dominates. Use `--root ADDRESS` to repeat the analysis for a suspicious subtree, or `--dot FILE` to write its dominator graph. Dumps captured with `shapes=false` remain compatible with `heap-profiler` and `sheap`. -The reported memory is shallow size and does not include every object referenced by a container. Treat a two-snapshot increase as evidence of growth, not necessarily a leak. Capture a third snapshot after another comparable workload and GC cycle to confirm that the same classes or allocation locations continue growing. +Reap does not compare snapshots or use allocation generations. Combine it with a `sheap` diff when you need both evidence of continued retention and the aggregate size of the retained graph. ## Recording Allocation Locations @@ -155,12 +228,13 @@ Look for fibers or threads that remain in every snapshot, queues that never drai 1. Confirm sustained growth with the `MemoryMonitor`, `ProcessMonitor`, or external process metrics. 2. List worker connection IDs and select one representative worker. -3. Capture a baseline heap dump after normal warm-up. -4. Start GC profiling. -5. Run representative traffic or wait through the suspected leak interval. +3. Enable allocation tracing before the workload if its overhead is acceptable. +4. Capture a baseline heap dump after normal warm-up, using `shapes=false` if you plan to use Reap. +5. Start GC profiling and run representative traffic or wait through the suspected leak interval. 6. Capture scheduler and thread dumps if work appears stuck or backlogged. -7. Stop GC profiling and capture a second heap dump from the same worker. -8. Compare heap types, allocation locations when tracing is enabled, and retained reference paths. -9. Repeat across another workload interval to distinguish sustained retention from normal cache warm-up. +7. Stop GC profiling and capture the second heap dump from the same worker. +8. Repeat the workload and capture a third dump for a stronger `sheap` retention signal. +9. Use `heap-profiler` for aggregate counts, `sheap` for retained objects and root paths, and Reap for dominator sizes. +10. Repeat the experiment if necessary to distinguish sustained retention from normal cache warm-up. Use the {ruby Async::Service::Supervisor::MemoryMonitor} to protect production from unbounded growth, but complete diagnostics before its configured limit restarts the worker. A restarted worker receives a new connection ID and loses the heap state you were observing. diff --git a/lib/async/service/supervisor/worker_controller.rb b/lib/async/service/supervisor/worker_controller.rb index fc0f912..59726b4 100644 --- a/lib/async/service/supervisor/worker_controller.rb +++ b/lib/async/service/supervisor/worker_controller.rb @@ -57,11 +57,12 @@ def scheduler_dump(path: nil, log: nil) # This is a heavyweight operation that dumps all objects in the heap. # # @parameter path [String] Optional file path to save the dump. - def memory_dump(path: nil) + # @parameter shapes [Boolean] Whether to include Ruby shape-tree records. + def memory_dump(path: nil, shapes: true) require "objspace" dump(path: path, buffer: false) do |file| - ObjectSpace.dump_all(output: file) + ObjectSpace.dump_all(output: file, shapes: shapes) end end @@ -118,4 +119,3 @@ def setup_utilization_observer(path, size, offset) end end end - diff --git a/test/bake/async/service/supervisor.rb b/test/bake/async/service/supervisor.rb index afd4718..e80c59f 100644 --- a/test/bake/async/service/supervisor.rb +++ b/test/bake/async/service/supervisor.rb @@ -29,10 +29,11 @@ def invoke(name, **options) it "dumps memory" do path = File.join(@root, "memory.json") - result = invoke("memory_dump", connection_id: connection_id, path: path) + result = invoke("memory_dump", connection_id: connection_id, path: path, shapes: false) expect(result).to be == {path: path} expect(File.size(path)).to be > 0 + expect(File.read(path)).not.to be(:include?, '"type":"SHAPE"') end it "dumps the scheduler" do From 5a23b0f8581aa653792cca22c33868b7c6c112f1 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 29 Jul 2026 13:55:56 +1200 Subject: [PATCH 9/9] Add bounded allocation tracing tasks Assisted-By: devx/d15f74b3-893f-4846-bbae-4dcdd10e8b8a --- bake/async/service/supervisor.rb | 20 +++++++++++ context/memory-diagnostics.md | 19 ++++++++--- guides/memory-diagnostics/readme.md | 19 ++++++++--- .../service/supervisor/worker_controller.rb | 34 +++++++++++++++++++ test/bake/async/service/supervisor.rb | 18 ++++++++++ 5 files changed, 100 insertions(+), 10 deletions(-) diff --git a/bake/async/service/supervisor.rb b/bake/async/service/supervisor.rb index 1719ad6..dd7c717 100644 --- a/bake/async/service/supervisor.rb +++ b/bake/async/service/supervisor.rb @@ -51,6 +51,26 @@ def memory_dump(connection_id:, path:, shapes: true) end end +# Start recording object allocation metadata in a worker. +# +# @parameter connection_id [Integer] The connection ID of the worker to target. +def allocation_trace_start(connection_id:) + with_worker(connection_id) do |worker| + worker.allocation_trace_start + end +end + +# Stop recording object allocations, dump the traced heap, and clear the metadata. +# +# @parameter connection_id [Integer] The connection ID of the worker to target. +# @parameter path [String] The file path where the worker should write the dump. +# @parameter shapes [Boolean] Whether to include Ruby shape-tree records. +def allocation_trace_stop(connection_id:, path:, shapes: true) + with_worker(connection_id) do |worker| + worker.allocation_trace_stop(path: path, shapes: shapes) + end +end + # Dump the fiber scheduler hierarchy of a worker. # # @parameter connection_id [Integer] The connection ID of the worker to target. diff --git a/context/memory-diagnostics.md b/context/memory-diagnostics.md index 15eef2b..9d72c10 100644 --- a/context/memory-diagnostics.md +++ b/context/memory-diagnostics.md @@ -169,14 +169,23 @@ Reap does not compare snapshots or use allocation generations. Combine it with a ## Recording Allocation Locations -By default, a heap dump may not contain allocation source locations. To record them, enable ObjectSpace allocation tracing in the worker before the workload you want to investigate: +By default, a heap dump may not contain allocation source locations. Start tracing on the selected worker before the workload you want to investigate: -```ruby -require "objspace" -ObjectSpace.trace_object_allocations_start +```bash +$ bake async:service:supervisor:allocation_trace_start connection_id=1 +``` + +Exercise the workload, then stop tracing and write the heap dump: + +```bash +$ bake async:service:supervisor:allocation_trace_stop \ + connection_id=1 \ + path=/var/tmp/worker-1-traced.json ``` -Allocation tracing adds runtime and memory overhead, so enable it selectively and measure its impact before using it in production. When tracing is enabled, heap records can include `file`, `line`, `method`, and allocation generation fields. `heap-profiler` automatically includes breakdowns by gem, file, and location when this information is present. +The stop operation disables tracing before dumping the heap, then clears the allocation metadata after the dump is complete. Start and stop must target the same live connection ID. Use `shapes=false` with the stop task when the dump will be analyzed by Reap. + +Allocation tracing is process-global and adds runtime and memory overhead, so enable it selectively and measure its impact before using it in production. The start task refuses to run if tracing is already active, avoiding interference with another profiler. While tracing is enabled, heap records can include `file`, `line`, `method`, and allocation generation fields. `heap-profiler` automatically includes breakdowns by gem, file, and location when this information is present. High allocation counts identify hot allocation sites. Compare the `heap-profiler` reports and focus on locations whose objects remain present and continue accumulating, rather than sites that merely allocate many short-lived objects. diff --git a/guides/memory-diagnostics/readme.md b/guides/memory-diagnostics/readme.md index 15eef2b..9d72c10 100644 --- a/guides/memory-diagnostics/readme.md +++ b/guides/memory-diagnostics/readme.md @@ -169,14 +169,23 @@ Reap does not compare snapshots or use allocation generations. Combine it with a ## Recording Allocation Locations -By default, a heap dump may not contain allocation source locations. To record them, enable ObjectSpace allocation tracing in the worker before the workload you want to investigate: +By default, a heap dump may not contain allocation source locations. Start tracing on the selected worker before the workload you want to investigate: -```ruby -require "objspace" -ObjectSpace.trace_object_allocations_start +```bash +$ bake async:service:supervisor:allocation_trace_start connection_id=1 +``` + +Exercise the workload, then stop tracing and write the heap dump: + +```bash +$ bake async:service:supervisor:allocation_trace_stop \ + connection_id=1 \ + path=/var/tmp/worker-1-traced.json ``` -Allocation tracing adds runtime and memory overhead, so enable it selectively and measure its impact before using it in production. When tracing is enabled, heap records can include `file`, `line`, `method`, and allocation generation fields. `heap-profiler` automatically includes breakdowns by gem, file, and location when this information is present. +The stop operation disables tracing before dumping the heap, then clears the allocation metadata after the dump is complete. Start and stop must target the same live connection ID. Use `shapes=false` with the stop task when the dump will be analyzed by Reap. + +Allocation tracing is process-global and adds runtime and memory overhead, so enable it selectively and measure its impact before using it in production. The start task refuses to run if tracing is already active, avoiding interference with another profiler. While tracing is enabled, heap records can include `file`, `line`, `method`, and allocation generation fields. `heap-profiler` automatically includes breakdowns by gem, file, and location when this information is present. High allocation counts identify hot allocation sites. Compare the `heap-profiler` reports and focus on locations whose objects remain present and continue accumulating, rather than sites that merely allocate many short-lived objects. diff --git a/lib/async/service/supervisor/worker_controller.rb b/lib/async/service/supervisor/worker_controller.rb index 59726b4..7e6d1c3 100644 --- a/lib/async/service/supervisor/worker_controller.rb +++ b/lib/async/service/supervisor/worker_controller.rb @@ -66,6 +66,40 @@ def memory_dump(path: nil, shapes: true) end end + # Start recording object allocation metadata. + # + # Allocation tracing is process-global and can add significant overhead. + def allocation_trace_start + require "objspace" + + raise "Object allocation tracing was already started by this controller!" if @allocation_trace_active + raise "Object allocation tracing is already active!" if ObjectSpace.allocation_sourcefile(Object.new) + + ObjectSpace.trace_object_allocations_start + @allocation_trace_active = true + + return {started: true} + end + + # Stop recording allocations, dump the traced heap, and clear the metadata. + # + # @parameter path [String] File path where the heap dump should be written. + # @parameter shapes [Boolean] Whether to include Ruby shape-tree records. + def allocation_trace_stop(path:, shapes: true) + require "objspace" + stopped = false + + raise "Object allocation tracing was not started by this controller!" unless @allocation_trace_active + + ObjectSpace.trace_object_allocations_stop + stopped = true + @allocation_trace_active = false + + memory_dump(path: path, shapes: shapes) + ensure + ObjectSpace.trace_object_allocations_clear if stopped + end + # Dump information about all running threads. # # Includes thread inspection and backtraces for debugging. diff --git a/test/bake/async/service/supervisor.rb b/test/bake/async/service/supervisor.rb index e80c59f..f29763b 100644 --- a/test/bake/async/service/supervisor.rb +++ b/test/bake/async/service/supervisor.rb @@ -36,6 +36,24 @@ def invoke(name, **options) expect(File.read(path)).not.to be(:include?, '"type":"SHAPE"') end + it "traces allocations and dumps memory" do + path = File.join(@root, "allocations.json") + + expect(invoke("allocation_trace_start", connection_id: connection_id)).to be == {started: true} + object = Object.new + expect(ObjectSpace.allocation_sourcefile(object)).to be_truthy + + result = invoke("allocation_trace_stop", connection_id: connection_id, path: path, shapes: false) + + expect(result).to be == {path: path} + expect(File.size(path)).to be > 0 + expect(File.read(path)).to be(:include?, '"file":') + expect(ObjectSpace.allocation_sourcefile(object)).to be_nil + ensure + ObjectSpace.trace_object_allocations_stop + ObjectSpace.trace_object_allocations_clear + end + it "dumps the scheduler" do path = File.join(@root, "scheduler.txt") result = invoke("scheduler_dump", connection_id: connection_id, path: path)