From 743248a5294f29aece27810d008121e0e63ebe4c Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 1 Aug 2026 02:25:35 +1200 Subject: [PATCH 01/13] Add ORCA load reporting --- async-service-supervisor-envoy.gemspec | 1 + gems.rb | 2 + guides/getting-started/readme.md | 24 +++ lib/async/service/supervisor/envoy.rb | 1 + lib/async/service/supervisor/envoy/monitor.rb | 156 ++++++++++++++++-- .../service/supervisor/envoy/orca_service.rb | 59 +++++++ readme.md | 1 + releases.md | 4 + .../async/service/supervisor/envoy/monitor.rb | 102 +++++++++++- .../service/supervisor/envoy/orca_service.rb | 30 ++++ 10 files changed, 368 insertions(+), 12 deletions(-) create mode 100644 lib/async/service/supervisor/envoy/orca_service.rb create mode 100644 test/async/service/supervisor/envoy/orca_service.rb diff --git a/async-service-supervisor-envoy.gemspec b/async-service-supervisor-envoy.gemspec index c0a9251..226ea1a 100644 --- a/async-service-supervisor-envoy.gemspec +++ b/async-service-supervisor-envoy.gemspec @@ -28,4 +28,5 @@ Gem::Specification.new do |spec| spec.add_dependency "async-grpc-xds", "~> 0.1" spec.add_dependency "async-http" spec.add_dependency "async-service-supervisor", "~> 0.18" + spec.add_dependency "process-metrics", "~> 0.12" end diff --git a/gems.rb b/gems.rb index 9efc822..12a3c64 100644 --- a/gems.rb +++ b/gems.rb @@ -5,6 +5,8 @@ source "https://rubygems.org" +gem "async-grpc-xds", github: "socketry/async-grpc-xds", branch: "add-orca-support" +gem "async-service-supervisor", github: "socketry/async-service-supervisor", branch: "expose-worker-utilization" gem "falcon" gemspec diff --git a/guides/getting-started/readme.md b/guides/getting-started/readme.md index 4d5fbd1..8294b17 100644 --- a/guides/getting-started/readme.md +++ b/guides/getting-started/readme.md @@ -94,3 +94,27 @@ Async::Service::Supervisor::Envoy::Monitor.new( ``` Disconnected workers are removed from EDS. Registered workers that fail the delegate health check remain in EDS with an unhealthy endpoint status. + +## Load-aware Balancing + +Enable out-of-band ORCA reporting to let Envoy weight independently bound workers using their current processor utilization and request throughput: + +``` ruby +Async::Service::Supervisor::Envoy::Monitor.new( + bind: "http://0.0.0.0:18000", + orca: true, + interval: 1 +) +``` + +The monitor samples each worker's processor usage with `process-metrics` and reads its `requests_total` counter from the supervisor utilization registry. It serves the resulting ORCA reports from the same HTTP/2 endpoint as ADS and configures each discovered cluster to use Envoy's client-side weighted-round-robin policy. + +The first sample establishes a baseline. Subsequent reports contain normalized `cpu_utilization` and `rps_fractional` values for each worker. Reports are removed immediately when a worker disconnects. + +Out-of-band ORCA requires: + + - Envoy 1.39 or later. + - A fixed TCP port for the monitor's `bind` address. + - Independently addressable TCP worker endpoints. Unix sockets and endpoints shared by several workers cannot provide distinct per-worker ORCA identities. + +The monitor address and worker addresses must be reachable from Envoy. In a sidecar deployment, binding the monitor to a fixed port in the shared network namespace satisfies this requirement. diff --git a/lib/async/service/supervisor/envoy.rb b/lib/async/service/supervisor/envoy.rb index a3c6ed7..d0b53b0 100644 --- a/lib/async/service/supervisor/envoy.rb +++ b/lib/async/service/supervisor/envoy.rb @@ -7,4 +7,5 @@ require_relative "envoy/delegate" require_relative "envoy/endpoint" require_relative "envoy/monitor" +require_relative "envoy/orca_service" require_relative "envoy/supervised" diff --git a/lib/async/service/supervisor/envoy/monitor.rb b/lib/async/service/supervisor/envoy/monitor.rb index 0d26f4f..836755d 100644 --- a/lib/async/service/supervisor/envoy/monitor.rb +++ b/lib/async/service/supervisor/envoy/monitor.rb @@ -5,12 +5,17 @@ require "async/http/endpoint" require "async/service/supervisor/monitor" +require "async/service/supervisor/utilization_monitor" require "async/grpc/xds/control_plane" +require "async/grpc/xds/resource_builder" require "async/grpc/xds/server" +require "process/metrics" +require "xds/data/orca/v3/orca_load_report_pb" require_relative "delegate" require_relative "endpoint" require_relative "endpoint_group" +require_relative "orca_service" module Async module Service @@ -23,21 +28,44 @@ class Monitor < Async::Service::Supervisor::Monitor # @parameter bind [String | Nil] The optional address for the xDS control plane server. # @parameter delegate [Delegate] The delegate used to map supervisor state into Envoy endpoints. # @parameter control_plane [Async::GRPC::XDS::ControlPlane] The xDS control plane to update. + # @parameter orca [Boolean] Whether to collect and serve per-worker ORCA load reports. + # @parameter processor [Process::Metrics::Processor | Nil] The optional process CPU sampler. + # @parameter utilization_monitor [Async::Service::Supervisor::UtilizationMonitor | Nil] The optional per-worker utilization monitor. + # @parameter interval [Numeric] The endpoint reconciliation and ORCA reporting interval in seconds. def initialize( bind: nil, delegate: Delegate.new, control_plane: Async::GRPC::XDS::ControlPlane.new, + orca: false, + processor: nil, + utilization_monitor: nil, + interval: 1, **options ) - super(**options) + super(interval: interval, **options) @bind = bind @delegate = delegate @control_plane = control_plane + @interval = interval + @orca = orca @controllers = {} @published_clusters = {} @server_task = nil @mutex = Mutex.new + + if @orca + raise ArgumentError, "ORCA reporting requires a TCP bind address!" unless @bind + + @orca_port = server_endpoint.url.port + raise ArgumentError, "ORCA reporting requires a fixed TCP port!" unless @orca_port&.positive? + + @processor = processor || Process::Metrics::Processor.new + @utilization_monitor = utilization_monitor || Async::Service::Supervisor::UtilizationMonitor.new(interval: interval) + @request_totals = {} + @load_reports = {} + @authorities = {} + end end # @attribute [Async::GRPC::XDS::ControlPlane] The xDS control plane receiving cluster and endpoint updates. @@ -50,8 +78,11 @@ def initialize( # @parameter supervisor_controller [Object] The supervisor controller describing the worker. # @returns [void] def register(supervisor_controller) + @utilization_monitor&.register(supervisor_controller) + @mutex.synchronize do @controllers[supervisor_controller.id] = supervisor_controller + @authorities[worker_hostname(supervisor_controller)] = supervisor_controller.id if @orca reconcile end end @@ -62,8 +93,16 @@ def register(supervisor_controller) def remove(supervisor_controller) @mutex.synchronize do @controllers.delete(supervisor_controller.id) + if @orca + hostname = worker_hostname(supervisor_controller) + @authorities.delete(hostname) + @load_reports.delete(hostname) + @request_totals.delete(supervisor_controller.id) + end reconcile end + + @utilization_monitor&.remove(supervisor_controller) end # Run the monitor and optional xDS server task. @@ -74,8 +113,9 @@ def run(parent: Async::Task.current) if @bind @server_task = parent.async do - endpoint = Async::HTTP::Endpoint.parse(@bind, protocol: Async::HTTP::Protocol::HTTP2) - Async::GRPC::XDS::Server.new(@control_plane).run(endpoint) + server = Async::GRPC::XDS::Server.new(@control_plane) + server.dispatcher.register(ORCAService.new(self, minimum_interval: @interval)) if @orca + server.run(server_endpoint) end end @@ -92,9 +132,29 @@ def as_json end end + # Determine whether an ORCA worker authority is currently registered. + # @parameter authority [String] The gRPC request authority. + # @returns [Boolean] Whether the authority identifies a live worker. + def worker?(authority) + return false unless @orca + + @mutex.synchronize{@authorities.key?(authority)} + end + + # Get the latest ORCA report for a worker authority. + # @parameter authority [String] The gRPC request authority. + # @returns [Xds::Data::Orca::V3::OrcaLoadReport | Nil] The latest valid report, if available. + def load_report(authority) + return unless @orca + + @mutex.synchronize{@load_reports[authority]} + end + # Refresh endpoint health and publish updated EDS state. # @returns [void] def run_once + sample_load_reports if @orca + @mutex.synchronize do reconcile end @@ -102,6 +162,14 @@ def run_once private + def server_endpoint + @server_endpoint ||= Async::HTTP::Endpoint.parse(@bind, protocol: Async::HTTP::Protocol::HTTP2) + end + + def worker_hostname(supervisor_controller) + "worker-#{supervisor_controller.id}" + end + def build_record(supervisor_controller, endpoint) cluster = @delegate.cluster(supervisor_controller, endpoint) return unless cluster @@ -148,14 +216,24 @@ def build_records_by_cluster def build_clusters(records_by_cluster = build_records_by_cluster) records_by_cluster.transform_values do |records| - groups = {} - - records.each do |record| - group = groups[record[:endpoint]] ||= EndpointGroup.new(record[:endpoint]) - group.add(record[:worker], healthy: record[:healthy]) + if @orca + records.map do |record| + { + addresses: record[:endpoint].addresses, + healthy: record[:healthy], + hostname: worker_hostname(record[:worker]), + } + end + else + groups = {} + + records.each do |record| + group = groups[record[:endpoint]] ||= EndpointGroup.new(record[:endpoint]) + group.add(record[:worker], healthy: record[:healthy]) + end + + groups.each_value.map(&:as_json) end - - groups.each_value.map(&:as_json) end end @@ -168,7 +246,63 @@ def cluster_configuration(records) raise ArgumentError, "Envoy cluster contains no common protocols: #{protocols.inspect}" if common_protocols.empty? raise ArgumentError, "HTTPS upstream endpoints are not yet supported!" if schemes.first == :https - {protocol: envoy_protocol(common_protocols)} + configuration = {protocol: envoy_protocol(common_protocols)} + + if @orca + if records.any?{|record| record[:endpoint].addresses.any?{|address| address[:path]}} + raise ArgumentError, "Out-of-band ORCA reporting requires IP endpoints!" + end + + configuration[:load_balancing_policy] = Async::GRPC::XDS::ResourceBuilder.client_side_weighted_round_robin( + @orca_port, + reporting_period: @interval + ) + end + + configuration + end + + def sample_load_reports + controllers = @mutex.synchronize{@controllers.dup} + workers = @utilization_monitor.sample_workers + process_ids = controllers.each_value.filter_map(&:process_id) + processor_samples = @processor.sample(process_ids) + request_totals = {} + + workers.each do |worker_id, worker| + requests_total = worker[:utilization][:requests_total] + if requests_total.is_a?(Numeric) && requests_total.finite? + request_totals[worker_id] = requests_total + end + end + + @mutex.synchronize do + controllers.each do |worker_id, controller| + next unless @controllers[worker_id].equal?(controller) + + hostname = worker_hostname(controller) + processor_sample = processor_samples[controller.process_id] + requests_total = request_totals[worker_id] + previous_requests_total = @request_totals[worker_id] + + if processor_sample && requests_total && previous_requests_total && requests_total >= previous_requests_total + rps = (requests_total - previous_requests_total).fdiv(processor_sample.duration) + cpu = processor_sample.utilization + + if rps.finite? && cpu.finite? + @load_reports[hostname] = Xds::Data::Orca::V3::OrcaLoadReport.new( + cpu_utilization: cpu, + rps_fractional: rps + ).freeze + next + end + end + + @load_reports.delete(hostname) + end + + @request_totals = request_totals + end end def envoy_protocol(protocols) diff --git a/lib/async/service/supervisor/envoy/orca_service.rb b/lib/async/service/supervisor/envoy/orca_service.rb new file mode 100644 index 0000000..7b8ef26 --- /dev/null +++ b/lib/async/service/supervisor/envoy/orca_service.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2026, by Samuel Williams. + +require "async/grpc/service" +require "xds/service/orca/v3/open_rca_service" + +module Async + module Service + module Supervisor + module Envoy + # Streams per-worker out-of-band ORCA load reports to Envoy. + class ORCAService < Async::GRPC::Service + SERVICE_NAME = "xds.service.orca.v3.OpenRcaService" + + # Initialize the ORCA service. + # @parameter monitor [Monitor] The monitor providing worker load reports. + # @parameter minimum_interval [Numeric] The minimum reporting interval in seconds. + def initialize(monitor, minimum_interval: 1) + super(Xds::Service::Orca::V3::OpenRcaService, SERVICE_NAME) + + @monitor = monitor + @minimum_interval = minimum_interval + end + + # Stream current load reports for the worker named by the request authority. + # @parameter input [Interface(:read)] The ORCA request stream. + # @parameter output [Interface(:write)] The ORCA report stream. + # @parameter call [Protocol::GRPC::Call] The gRPC call context. + # @asynchronous + def stream_core_metrics(input, output, call) + request = input.read + return unless request + + authority = call.request.authority + interval = [duration(request.report_interval), @minimum_interval].max + + while @monitor.worker?(authority) + if report = @monitor.load_report(authority) + output.write(report) + end + + sleep(interval) + end + end + + private + + def duration(value) + return 0 unless value + + value.seconds + value.nanos.fdiv(1_000_000_000) + end + end + end + end + end +end diff --git a/readme.md b/readme.md index 1849978..4ae9417 100644 --- a/readme.md +++ b/readme.md @@ -13,6 +13,7 @@ Provides an Envoy xDS monitor for `async-service-supervisor`. - **Multiple clusters** - Groups workers by `state[:name]` by default. - **Endpoint contract** - Converts concrete post-bind worker listeners into Envoy upstream endpoints, including grouped IP or Unix socket addresses. - **Delegate mapping** - Uses a delegate object for endpoint selection, cluster grouping, and health without active probing. + - **Load-aware balancing** - Optionally reports per-worker CPU utilization and request throughput to Envoy using out-of-band ORCA. ## Usage diff --git a/releases.md b/releases.md index 85b8bbb..66e3183 100644 --- a/releases.md +++ b/releases.md @@ -1,5 +1,9 @@ # Releases +## Unreleased + + - Add supervisor-driven out-of-band ORCA load reporting for independently addressable workers. + ## v0.2.0 - Accept the required Falcon listener as a positional worker preparation argument. diff --git a/test/async/service/supervisor/envoy/monitor.rb b/test/async/service/supervisor/envoy/monitor.rb index 62716d1..ca20c04 100644 --- a/test/async/service/supervisor/envoy/monitor.rb +++ b/test/async/service/supervisor/envoy/monitor.rb @@ -7,7 +7,7 @@ require "envoy/config/endpoint/v3/endpoint_pb" describe Async::Service::Supervisor::Envoy::Monitor do - Controller = Struct.new(:id, :state) + Controller = Struct.new(:id, :state, :process_id) let(:monitor) {subject.new} let(:control_plane) {monitor.control_plane} @@ -21,6 +21,20 @@ def endpoint_assignment(cluster) Envoy::Config::Endpoint::V3::ClusterLoadAssignment.decode(response.resources.first.value) end + def utilization_monitor(samples = []) + Object.new.tap do |monitor| + monitor.define_singleton_method(:register){|controller|} + monitor.define_singleton_method(:remove){|controller|} + monitor.define_singleton_method(:sample_workers){samples.shift || {}} + end + end + + def processor(samples = []) + Object.new.tap do |processor| + processor.define_singleton_method(:sample){|process_ids| samples.shift || {}} + end + end + it "publishes registered endpoints" do controller = Controller.new(1, { endpoint: {name: "myservice", scheme: "http", protocols: ["h2"], addresses: [{address: "127.0.0.1", port: 50051}]} @@ -36,6 +50,92 @@ def endpoint_assignment(cluster) expect(load_balancer_endpoint.endpoint.address.socket_address.port_value).to be == 50051 end + it "publishes ORCA worker identity and load-balancing configuration" do + monitor = subject.new( + bind: "http://127.0.0.1:18000", + orca: true, + processor: processor, + utilization_monitor: utilization_monitor + ) + controller = Controller.new(1, { + endpoint: {name: "myservice", scheme: "http", protocols: ["h2"], addresses: [{address: "127.0.0.1", port: 50051}]} + }, 123) + + monitor.register(controller) + + response = monitor.control_plane.response( + Async::GRPC::XDS::ControlPlane::ENDPOINT_TYPE, + ["myservice"] + ) + assignment = Envoy::Config::Endpoint::V3::ClusterLoadAssignment.decode(response.resources.first.value) + cluster = monitor.control_plane.resources(Async::GRPC::XDS::ControlPlane::CLUSTER_TYPE).first + typed_configuration = cluster.load_balancing_policy.policies.first.typed_extension_config + configuration = Envoy::Extensions::LoadBalancingPolicies::ClientSideWeightedRoundRobin::V3::ClientSideWeightedRoundRobin.decode( + typed_configuration.typed_config.value + ) + + expect(assignment.endpoints.first.lb_endpoints.first.endpoint.hostname).to be == "worker-1" + expect(configuration.enable_oob_load_report.value).to be == true + expect(configuration.oob_reporting_config.port_value).to be == 18000 + end + + it "samples per-worker ORCA load reports" do + utilization = utilization_monitor([ + {1 => {state: {}, utilization: {requests_total: 10}}}, + {1 => {state: {}, utilization: {requests_total: 14}}}, + ]) + processor_sample = Struct.new(:duration, :utilization).new(2.0, 0.5) + processor = processor([ + {}, + {123 => processor_sample}, + ]) + monitor = subject.new( + bind: "http://127.0.0.1:18000", + orca: true, + processor: processor, + utilization_monitor: utilization + ) + controller = Controller.new(1, { + endpoint: {name: "myservice", scheme: "http", protocols: ["h2"], addresses: [{address: "127.0.0.1", port: 50051}]} + }, 123) + + monitor.register(controller) + monitor.run_once + expect(monitor.load_report("worker-1")).to be_nil + + monitor.run_once + report = monitor.load_report("worker-1") + + expect(report.cpu_utilization).to be == 0.5 + expect(report.rps_fractional).to be == 2.0 + expect(monitor.worker?("worker-1")).to be == true + + monitor.remove(controller) + expect(monitor.worker?("worker-1")).to be == false + expect(monitor.load_report("worker-1")).to be_nil + end + + it "requires a fixed bind address for ORCA" do + expect do + subject.new(orca: true) + end.to raise_exception(ArgumentError) + end + + it "rejects Unix endpoints for out-of-band ORCA" do + monitor = subject.new( + bind: "http://127.0.0.1:18000", + orca: true, + processor: processor, + utilization_monitor: utilization_monitor + ) + + expect do + monitor.register(Controller.new(1, { + endpoint: {name: "myservice", scheme: "http", protocols: ["h2"], addresses: [{path: "/tmp/falcon.ipc"}]} + }, 123)) + end.to raise_exception(ArgumentError) + end + it "publishes a supervised Unix endpoint" do controller = Controller.new(1, { endpoint: { diff --git a/test/async/service/supervisor/envoy/orca_service.rb b/test/async/service/supervisor/envoy/orca_service.rb new file mode 100644 index 0000000..3cf8eb4 --- /dev/null +++ b/test/async/service/supervisor/envoy/orca_service.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2026, by Samuel Williams. + +require "async/service/supervisor/envoy/orca_service" +require "google/protobuf/duration_pb" + +describe Async::Service::Supervisor::Envoy::ORCAService do + it "streams reports for the requested worker authority" do + report = Xds::Data::Orca::V3::OrcaLoadReport.new(cpu_utilization: 0.5, rps_fractional: 2.0) + checks = [true, false] + monitor = Object.new + monitor.define_singleton_method(:worker?){|authority| authority == "worker-1" && checks.shift} + monitor.define_singleton_method(:load_report){|authority| report} + + request = Xds::Service::Orca::V3::OrcaLoadReportRequest.new( + report_interval: Google::Protobuf::Duration.new + ) + input = Object.new + input.define_singleton_method(:read){request} + output = [] + output.define_singleton_method(:write){|value| self << value} + call = Struct.new(:request).new(Struct.new(:authority).new("worker-1")) + + subject.new(monitor, minimum_interval: 0).stream_core_metrics(input, output, call) + + expect(output).to be == [report] + end +end From f6c54682a4bb0c09d0f438f9cefa0d479ead18f7 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 1 Aug 2026 02:40:41 +1200 Subject: [PATCH 02/13] Keep ORCA streams active during sampling --- .../service/supervisor/envoy/orca_service.rb | 8 +++-- .../service/supervisor/envoy/orca_service.rb | 34 +++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/lib/async/service/supervisor/envoy/orca_service.rb b/lib/async/service/supervisor/envoy/orca_service.rb index 7b8ef26..e5cd8f0 100644 --- a/lib/async/service/supervisor/envoy/orca_service.rb +++ b/lib/async/service/supervisor/envoy/orca_service.rb @@ -4,6 +4,7 @@ # Copyright, 2026, by Samuel Williams. require "async/grpc/service" +require "protocol/http/body/writable" require "xds/service/orca/v3/open_rca_service" module Async @@ -13,6 +14,7 @@ module Envoy # Streams per-worker out-of-band ORCA load reports to Envoy. class ORCAService < Async::GRPC::Service SERVICE_NAME = "xds.service.orca.v3.OpenRcaService" + EMPTY_REPORT = Xds::Data::Orca::V3::OrcaLoadReport.new.freeze # Initialize the ORCA service. # @parameter monitor [Monitor] The monitor providing worker load reports. @@ -37,12 +39,12 @@ def stream_core_metrics(input, output, call) interval = [duration(request.report_interval), @minimum_interval].max while @monitor.worker?(authority) - if report = @monitor.load_report(authority) - output.write(report) - end + output.write(@monitor.load_report(authority) || EMPTY_REPORT) sleep(interval) end + rescue Protocol::HTTP::Body::Writable::Closed + # The client closed the reporting stream. end private diff --git a/test/async/service/supervisor/envoy/orca_service.rb b/test/async/service/supervisor/envoy/orca_service.rb index 3cf8eb4..4ec61b6 100644 --- a/test/async/service/supervisor/envoy/orca_service.rb +++ b/test/async/service/supervisor/envoy/orca_service.rb @@ -27,4 +27,38 @@ expect(output).to be == [report] end + + it "streams an empty report while establishing a baseline" do + checks = [true, false] + monitor = Object.new + monitor.define_singleton_method(:worker?){|authority| authority == "worker-1" && checks.shift} + monitor.define_singleton_method(:load_report){|authority| nil} + + request = Xds::Service::Orca::V3::OrcaLoadReportRequest.new + input = Object.new + input.define_singleton_method(:read){request} + output = [] + output.define_singleton_method(:write){|value| self << value} + call = Struct.new(:request).new(Struct.new(:authority).new("worker-1")) + + subject.new(monitor, minimum_interval: 0).stream_core_metrics(input, output, call) + + expect(output).to be == [subject::EMPTY_REPORT] + end + + it "stops when the client closes the stream" do + monitor = Object.new + monitor.define_singleton_method(:worker?){|authority| true} + monitor.define_singleton_method(:load_report){|authority| nil} + + input = Object.new + input.define_singleton_method(:read){Xds::Service::Orca::V3::OrcaLoadReportRequest.new} + output = Object.new + output.define_singleton_method(:write){|value| raise Protocol::HTTP::Body::Writable::Closed} + call = Struct.new(:request).new(Struct.new(:authority).new("worker-1")) + + expect do + subject.new(monitor, minimum_interval: 0).stream_core_metrics(input, output, call) + end.not.to raise_exception + end end From 14931ea61a5f829037dbbb29a8669ca120c2d515 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 1 Aug 2026 02:44:08 +1200 Subject: [PATCH 03/13] Report ORCA sampling diagnostics --- lib/async/service/supervisor/envoy/monitor.rb | 7 +++++++ lib/async/service/supervisor/envoy/orca_service.rb | 4 +++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/async/service/supervisor/envoy/monitor.rb b/lib/async/service/supervisor/envoy/monitor.rb index 836755d..690564d 100644 --- a/lib/async/service/supervisor/envoy/monitor.rb +++ b/lib/async/service/supervisor/envoy/monitor.rb @@ -302,6 +302,13 @@ def sample_load_reports end @request_totals = request_totals + + Console.debug(self, "Sampled ORCA load reports.", + workers: workers.keys, + processes: processor_samples.keys, + requests: request_totals, + reports: @load_reports.keys, + ) end end diff --git a/lib/async/service/supervisor/envoy/orca_service.rb b/lib/async/service/supervisor/envoy/orca_service.rb index e5cd8f0..59ddcaa 100644 --- a/lib/async/service/supervisor/envoy/orca_service.rb +++ b/lib/async/service/supervisor/envoy/orca_service.rb @@ -14,7 +14,9 @@ module Envoy # Streams per-worker out-of-band ORCA load reports to Envoy. class ORCAService < Async::GRPC::Service SERVICE_NAME = "xds.service.orca.v3.OpenRcaService" - EMPTY_REPORT = Xds::Data::Orca::V3::OrcaLoadReport.new.freeze + EMPTY_REPORT = Xds::Data::Orca::V3::OrcaLoadReport.new( + named_metrics: {"orca.heartbeat" => 0.0} + ).freeze # Initialize the ORCA service. # @parameter monitor [Monitor] The monitor providing worker load reports. From 9d81c64c6f38cc0bf09d8b2c30359b2f2f2a7b77 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 1 Aug 2026 02:46:07 +1200 Subject: [PATCH 04/13] Keep idle ORCA reports active --- lib/async/service/supervisor/envoy/monitor.rb | 3 ++- test/async/service/supervisor/envoy/monitor.rb | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/async/service/supervisor/envoy/monitor.rb b/lib/async/service/supervisor/envoy/monitor.rb index 690564d..1bcfe0a 100644 --- a/lib/async/service/supervisor/envoy/monitor.rb +++ b/lib/async/service/supervisor/envoy/monitor.rb @@ -292,7 +292,8 @@ def sample_load_reports if rps.finite? && cpu.finite? @load_reports[hostname] = Xds::Data::Orca::V3::OrcaLoadReport.new( cpu_utilization: cpu, - rps_fractional: rps + rps_fractional: rps, + named_metrics: {"orca.heartbeat" => 0.0}, ).freeze next end diff --git a/test/async/service/supervisor/envoy/monitor.rb b/test/async/service/supervisor/envoy/monitor.rb index ca20c04..65d2ae3 100644 --- a/test/async/service/supervisor/envoy/monitor.rb +++ b/test/async/service/supervisor/envoy/monitor.rb @@ -108,6 +108,7 @@ def processor(samples = []) expect(report.cpu_utilization).to be == 0.5 expect(report.rps_fractional).to be == 2.0 + expect(report.named_metrics).to be == {"orca.heartbeat" => 0.0} expect(monitor.worker?("worker-1")).to be == true monitor.remove(controller) From a7b7baa17dee0c2437b33093df373a6d470bdfc2 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 1 Aug 2026 02:51:04 +1200 Subject: [PATCH 05/13] Avoid freezing protobuf messages --- lib/async/service/supervisor/envoy/monitor.rb | 2 +- lib/async/service/supervisor/envoy/orca_service.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/async/service/supervisor/envoy/monitor.rb b/lib/async/service/supervisor/envoy/monitor.rb index 1bcfe0a..68a7746 100644 --- a/lib/async/service/supervisor/envoy/monitor.rb +++ b/lib/async/service/supervisor/envoy/monitor.rb @@ -294,7 +294,7 @@ def sample_load_reports cpu_utilization: cpu, rps_fractional: rps, named_metrics: {"orca.heartbeat" => 0.0}, - ).freeze + ) next end end diff --git a/lib/async/service/supervisor/envoy/orca_service.rb b/lib/async/service/supervisor/envoy/orca_service.rb index 59ddcaa..c78a1a8 100644 --- a/lib/async/service/supervisor/envoy/orca_service.rb +++ b/lib/async/service/supervisor/envoy/orca_service.rb @@ -16,7 +16,7 @@ class ORCAService < Async::GRPC::Service SERVICE_NAME = "xds.service.orca.v3.OpenRcaService" EMPTY_REPORT = Xds::Data::Orca::V3::OrcaLoadReport.new( named_metrics: {"orca.heartbeat" => 0.0} - ).freeze + ) # Initialize the ORCA service. # @parameter monitor [Monitor] The monitor providing worker load reports. From 1283a7c078c13d050c28ce050fbbb4bd01e00176 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 1 Aug 2026 09:28:00 +1200 Subject: [PATCH 06/13] Use dedicated weighted round robin builder --- lib/async/service/supervisor/envoy/monitor.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/async/service/supervisor/envoy/monitor.rb b/lib/async/service/supervisor/envoy/monitor.rb index 68a7746..409e502 100644 --- a/lib/async/service/supervisor/envoy/monitor.rb +++ b/lib/async/service/supervisor/envoy/monitor.rb @@ -8,6 +8,7 @@ require "async/service/supervisor/utilization_monitor" require "async/grpc/xds/control_plane" require "async/grpc/xds/resource_builder" +require "async/grpc/xds/resources/client_side_weighted_round_robin" require "async/grpc/xds/server" require "process/metrics" require "xds/data/orca/v3/orca_load_report_pb" @@ -253,7 +254,7 @@ def cluster_configuration(records) raise ArgumentError, "Out-of-band ORCA reporting requires IP endpoints!" end - configuration[:load_balancing_policy] = Async::GRPC::XDS::ResourceBuilder.client_side_weighted_round_robin( + configuration[:load_balancing_policy] = Async::GRPC::XDS::Resources::ClientSideWeightedRoundRobin.build( @orca_port, reporting_period: @interval ) From 13997101bbad72b087b171c663c6e2b0497f5774 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 1 Aug 2026 09:51:16 +1200 Subject: [PATCH 07/13] Use top-level weighted round robin builder --- lib/async/service/supervisor/envoy/monitor.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/async/service/supervisor/envoy/monitor.rb b/lib/async/service/supervisor/envoy/monitor.rb index 409e502..2553454 100644 --- a/lib/async/service/supervisor/envoy/monitor.rb +++ b/lib/async/service/supervisor/envoy/monitor.rb @@ -6,9 +6,9 @@ require "async/http/endpoint" require "async/service/supervisor/monitor" require "async/service/supervisor/utilization_monitor" +require "async/grpc/xds/client_side_weighted_round_robin" require "async/grpc/xds/control_plane" require "async/grpc/xds/resource_builder" -require "async/grpc/xds/resources/client_side_weighted_round_robin" require "async/grpc/xds/server" require "process/metrics" require "xds/data/orca/v3/orca_load_report_pb" @@ -254,7 +254,7 @@ def cluster_configuration(records) raise ArgumentError, "Out-of-band ORCA reporting requires IP endpoints!" end - configuration[:load_balancing_policy] = Async::GRPC::XDS::Resources::ClientSideWeightedRoundRobin.build( + configuration[:load_balancing_policy] = Async::GRPC::XDS::ClientSideWeightedRoundRobin.build( @orca_port, reporting_period: @interval ) From 578b8a6ae0745abe3cf85c86f415cbb6e89287d1 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 1 Aug 2026 10:25:36 +1200 Subject: [PATCH 08/13] Update ORCA integration dependencies --- async-service-supervisor-envoy.gemspec | 2 +- gems.rb | 1 - lib/async/service/supervisor/envoy/monitor.rb | 3 +-- test/async/service/supervisor/envoy/monitor.rb | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/async-service-supervisor-envoy.gemspec b/async-service-supervisor-envoy.gemspec index 226ea1a..cd9ae60 100644 --- a/async-service-supervisor-envoy.gemspec +++ b/async-service-supervisor-envoy.gemspec @@ -25,7 +25,7 @@ Gem::Specification.new do |spec| spec.required_ruby_version = ">= 3.3" spec.add_dependency "async", "~> 2.38" - spec.add_dependency "async-grpc-xds", "~> 0.1" + spec.add_dependency "async-grpc-xds", "~> 0.2" spec.add_dependency "async-http" spec.add_dependency "async-service-supervisor", "~> 0.18" spec.add_dependency "process-metrics", "~> 0.12" diff --git a/gems.rb b/gems.rb index 12a3c64..4716f20 100644 --- a/gems.rb +++ b/gems.rb @@ -5,7 +5,6 @@ source "https://rubygems.org" -gem "async-grpc-xds", github: "socketry/async-grpc-xds", branch: "add-orca-support" gem "async-service-supervisor", github: "socketry/async-service-supervisor", branch: "expose-worker-utilization" gem "falcon" diff --git a/lib/async/service/supervisor/envoy/monitor.rb b/lib/async/service/supervisor/envoy/monitor.rb index 2553454..976b6a9 100644 --- a/lib/async/service/supervisor/envoy/monitor.rb +++ b/lib/async/service/supervisor/envoy/monitor.rb @@ -8,7 +8,6 @@ require "async/service/supervisor/utilization_monitor" require "async/grpc/xds/client_side_weighted_round_robin" require "async/grpc/xds/control_plane" -require "async/grpc/xds/resource_builder" require "async/grpc/xds/server" require "process/metrics" require "xds/data/orca/v3/orca_load_report_pb" @@ -265,7 +264,7 @@ def cluster_configuration(records) def sample_load_reports controllers = @mutex.synchronize{@controllers.dup} - workers = @utilization_monitor.sample_workers + workers = @utilization_monitor.sample_by_worker process_ids = controllers.each_value.filter_map(&:process_id) processor_samples = @processor.sample(process_ids) request_totals = {} diff --git a/test/async/service/supervisor/envoy/monitor.rb b/test/async/service/supervisor/envoy/monitor.rb index 65d2ae3..6b6d49c 100644 --- a/test/async/service/supervisor/envoy/monitor.rb +++ b/test/async/service/supervisor/envoy/monitor.rb @@ -25,7 +25,7 @@ def utilization_monitor(samples = []) Object.new.tap do |monitor| monitor.define_singleton_method(:register){|controller|} monitor.define_singleton_method(:remove){|controller|} - monitor.define_singleton_method(:sample_workers){samples.shift || {}} + monitor.define_singleton_method(:sample_by_worker){samples.shift || {}} end end From f84d02fe99d4da332c0210361090124b89451d94 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 1 Aug 2026 10:55:42 +1200 Subject: [PATCH 09/13] Compose ORCA utilization monitoring explicitly --- async-service-supervisor-envoy.gemspec | 2 +- gems.rb | 1 - guides/getting-started/readme.md | 19 +++++++++++++------ lib/async/service/supervisor/envoy/monitor.rb | 9 +++------ .../async/service/supervisor/envoy/monitor.rb | 8 ++++++-- 5 files changed, 23 insertions(+), 16 deletions(-) diff --git a/async-service-supervisor-envoy.gemspec b/async-service-supervisor-envoy.gemspec index cd9ae60..c28d7ae 100644 --- a/async-service-supervisor-envoy.gemspec +++ b/async-service-supervisor-envoy.gemspec @@ -27,6 +27,6 @@ Gem::Specification.new do |spec| spec.add_dependency "async", "~> 2.38" spec.add_dependency "async-grpc-xds", "~> 0.2" spec.add_dependency "async-http" - spec.add_dependency "async-service-supervisor", "~> 0.18" + spec.add_dependency "async-service-supervisor", "~> 0.20" spec.add_dependency "process-metrics", "~> 0.12" end diff --git a/gems.rb b/gems.rb index 4716f20..9efc822 100644 --- a/gems.rb +++ b/gems.rb @@ -5,7 +5,6 @@ source "https://rubygems.org" -gem "async-service-supervisor", github: "socketry/async-service-supervisor", branch: "expose-worker-utilization" gem "falcon" gemspec diff --git a/guides/getting-started/readme.md b/guides/getting-started/readme.md index 8294b17..c1ea351 100644 --- a/guides/getting-started/readme.md +++ b/guides/getting-started/readme.md @@ -100,14 +100,20 @@ Disconnected workers are removed from EDS. Registered workers that fail the dele Enable out-of-band ORCA reporting to let Envoy weight independently bound workers using their current processor utilization and request throughput: ``` ruby -Async::Service::Supervisor::Envoy::Monitor.new( - bind: "http://0.0.0.0:18000", - orca: true, - interval: 1 -) +utilization_monitor = Async::Service::Supervisor::UtilizationMonitor.new(interval: 1) + +[ + utilization_monitor, + Async::Service::Supervisor::Envoy::Monitor.new( + bind: "http://0.0.0.0:18000", + orca: true, + utilization_monitor: utilization_monitor, + interval: 1 + ) +] ``` -The monitor samples each worker's processor usage with `process-metrics` and reads its `requests_total` counter from the supervisor utilization registry. It serves the resulting ORCA reports from the same HTTP/2 endpoint as ADS and configures each discovered cluster to use Envoy's client-side weighted-round-robin policy. +The supervisor utilization monitor manages each worker's shared-memory allocation and registration. The Envoy monitor samples it through `sample_by_worker`, combines each worker's `requests_total` counter with processor usage from `process-metrics`, and serves the resulting ORCA reports from the same HTTP/2 endpoint as ADS. It also configures each discovered cluster to use Envoy's client-side weighted-round-robin policy. The first sample establishes a baseline. Subsequent reports contain normalized `cpu_utilization` and `rps_fractional` values for each worker. Reports are removed immediately when a worker disconnects. @@ -115,6 +121,7 @@ Out-of-band ORCA requires: - Envoy 1.39 or later. - A fixed TCP port for the monitor's `bind` address. + - A supervisor utilization monitor registered alongside the Envoy monitor. - Independently addressable TCP worker endpoints. Unix sockets and endpoints shared by several workers cannot provide distinct per-worker ORCA identities. The monitor address and worker addresses must be reachable from Envoy. In a sidecar deployment, binding the monitor to a fixed port in the shared network namespace satisfies this requirement. diff --git a/lib/async/service/supervisor/envoy/monitor.rb b/lib/async/service/supervisor/envoy/monitor.rb index 976b6a9..beab1fd 100644 --- a/lib/async/service/supervisor/envoy/monitor.rb +++ b/lib/async/service/supervisor/envoy/monitor.rb @@ -30,7 +30,7 @@ class Monitor < Async::Service::Supervisor::Monitor # @parameter control_plane [Async::GRPC::XDS::ControlPlane] The xDS control plane to update. # @parameter orca [Boolean] Whether to collect and serve per-worker ORCA load reports. # @parameter processor [Process::Metrics::Processor | Nil] The optional process CPU sampler. - # @parameter utilization_monitor [Async::Service::Supervisor::UtilizationMonitor | Nil] The optional per-worker utilization monitor. + # @parameter utilization_monitor [Async::Service::Supervisor::UtilizationMonitor | Nil] The per-worker utilization monitor used for ORCA reporting. # @parameter interval [Numeric] The endpoint reconciliation and ORCA reporting interval in seconds. def initialize( bind: nil, @@ -56,12 +56,13 @@ def initialize( if @orca raise ArgumentError, "ORCA reporting requires a TCP bind address!" unless @bind + raise ArgumentError, "ORCA reporting requires a utilization monitor!" unless utilization_monitor @orca_port = server_endpoint.url.port raise ArgumentError, "ORCA reporting requires a fixed TCP port!" unless @orca_port&.positive? @processor = processor || Process::Metrics::Processor.new - @utilization_monitor = utilization_monitor || Async::Service::Supervisor::UtilizationMonitor.new(interval: interval) + @utilization_monitor = utilization_monitor @request_totals = {} @load_reports = {} @authorities = {} @@ -78,8 +79,6 @@ def initialize( # @parameter supervisor_controller [Object] The supervisor controller describing the worker. # @returns [void] def register(supervisor_controller) - @utilization_monitor&.register(supervisor_controller) - @mutex.synchronize do @controllers[supervisor_controller.id] = supervisor_controller @authorities[worker_hostname(supervisor_controller)] = supervisor_controller.id if @orca @@ -101,8 +100,6 @@ def remove(supervisor_controller) end reconcile end - - @utilization_monitor&.remove(supervisor_controller) end # Run the monitor and optional xDS server task. diff --git a/test/async/service/supervisor/envoy/monitor.rb b/test/async/service/supervisor/envoy/monitor.rb index 6b6d49c..4724b23 100644 --- a/test/async/service/supervisor/envoy/monitor.rb +++ b/test/async/service/supervisor/envoy/monitor.rb @@ -23,8 +23,6 @@ def endpoint_assignment(cluster) def utilization_monitor(samples = []) Object.new.tap do |monitor| - monitor.define_singleton_method(:register){|controller|} - monitor.define_singleton_method(:remove){|controller|} monitor.define_singleton_method(:sample_by_worker){samples.shift || {}} end end @@ -122,6 +120,12 @@ def processor(samples = []) end.to raise_exception(ArgumentError) end + it "requires a utilization monitor for ORCA" do + expect do + subject.new(bind: "http://127.0.0.1:18000", orca: true) + end.to raise_exception(ArgumentError) + end + it "rejects Unix endpoints for out-of-band ORCA" do monitor = subject.new( bind: "http://127.0.0.1:18000", From ac0cce32f99aef48f08c6500973a4019aef45d72 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 1 Aug 2026 11:05:46 +1200 Subject: [PATCH 10/13] Wait for all control plane backends --- control-plane/test/envoy.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/control-plane/test/envoy.rb b/control-plane/test/envoy.rb index 7a391bf..cb563c4 100644 --- a/control-plane/test/envoy.rb +++ b/control-plane/test/envoy.rb @@ -29,8 +29,8 @@ def eventually(timeout: 20, interval: 0.5) it "routes requests through Envoy to supervised Falcon workers" do uri = envoy_uri - backend_ids = eventually do - 20.times.map do + eventually do + backend_ids = 20.times.map do response = Net::HTTP.get_response(uri) expect(response.code.to_i).to be == 200 @@ -38,9 +38,9 @@ def eventually(timeout: 20, interval: 0.5) response["x-backend-id"] end + + expect(backend_ids.compact.uniq.sort).to be == ["backend-a", "backend-b"] end - - expect(backend_ids.compact.uniq.sort).to be == ["backend-a", "backend-b"] end it "loads the xDS cluster from the supervisor monitor" do From a7bab020ad42c8d4224368a3e3d3daa700d3d6fc Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 1 Aug 2026 11:08:21 +1200 Subject: [PATCH 11/13] Make control plane readiness retryable --- control-plane/test/envoy.rb | 40 ++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/control-plane/test/envoy.rb b/control-plane/test/envoy.rb index cb563c4..def6acc 100644 --- a/control-plane/test/envoy.rb +++ b/control-plane/test/envoy.rb @@ -17,43 +17,43 @@ def eventually(timeout: 20, interval: 0.5) while Process.clock_gettime(Process::CLOCK_MONOTONIC) < deadline begin - return yield + return result if result = yield rescue => error - sleep interval end + + sleep interval end - raise error + raise error || "Condition was not met within #{timeout} seconds!" end it "routes requests through Envoy to supervised Falcon workers" do uri = envoy_uri - eventually do - backend_ids = 20.times.map do - response = Net::HTTP.get_response(uri) - - expect(response.code.to_i).to be == 200 - expect(response.body).to be =~ /Hello from backend-[ab]/ - - response["x-backend-id"] - end + responses = eventually do + responses = 20.times.map{Net::HTTP.get_response(uri)} + backend_ids = responses.filter_map{|response| response["x-backend-id"]}.uniq.sort - expect(backend_ids.compact.uniq.sort).to be == ["backend-a", "backend-b"] + responses if backend_ids == ["backend-a", "backend-b"] + end + + responses.each do |response| + expect(response.code.to_i).to be == 200 + expect(response.body).to be =~ /Hello from backend-[ab]/ end + + expect(responses.filter_map{|response| response["x-backend-id"]}.uniq.sort).to be == ["backend-a", "backend-b"] end it "loads the xDS cluster from the supervisor monitor" do uri = admin_uri + "/clusters?format=json" cluster_status = eventually do - response = Net::HTTP.get_response(uri) - - expect(response.code.to_i).to be == 200 - - clusters = JSON.parse(response.body) - clusters.fetch("cluster_statuses").find do |cluster| - cluster.fetch("name") == "app-http1" + if (response = Net::HTTP.get_response(uri)).code.to_i == 200 + clusters = JSON.parse(response.body) + clusters.fetch("cluster_statuses").find do |cluster| + cluster.fetch("name") == "app-http1" + end end end From bc2424a5c71afe5e80336d3c6f3925a95dfe948b Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 1 Aug 2026 11:23:15 +1200 Subject: [PATCH 12/13] Fix readiness result scoping --- control-plane/test/envoy.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/control-plane/test/envoy.rb b/control-plane/test/envoy.rb index def6acc..ede0bc5 100644 --- a/control-plane/test/envoy.rb +++ b/control-plane/test/envoy.rb @@ -17,7 +17,8 @@ def eventually(timeout: 20, interval: 0.5) while Process.clock_gettime(Process::CLOCK_MONOTONIC) < deadline begin - return result if result = yield + result = yield + return result if result rescue => error end From 63363c135cce9e851344d09b3b7b6dcb93e73db1 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 1 Aug 2026 11:27:19 +1200 Subject: [PATCH 13/13] Clarify readiness result handling --- control-plane/test/envoy.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/control-plane/test/envoy.rb b/control-plane/test/envoy.rb index ede0bc5..76f2258 100644 --- a/control-plane/test/envoy.rb +++ b/control-plane/test/envoy.rb @@ -17,8 +17,9 @@ def eventually(timeout: 20, interval: 0.5) while Process.clock_gettime(Process::CLOCK_MONOTONIC) < deadline begin - result = yield - return result if result + if result = yield + return result + end rescue => error end