diff --git a/async-service-supervisor-envoy.gemspec b/async-service-supervisor-envoy.gemspec index c0a9251..c28d7ae 100644 --- a/async-service-supervisor-envoy.gemspec +++ b/async-service-supervisor-envoy.gemspec @@ -25,7 +25,8 @@ Gem::Specification.new do |spec| spec.required_ruby_version = ">= 3.3" spec.add_dependency "async", "~> 2.38" - spec.add_dependency "async-grpc-xds", "~> 0.1" + spec.add_dependency "async-grpc-xds", "~> 0.2" spec.add_dependency "async-http" - spec.add_dependency "async-service-supervisor", "~> 0.18" + spec.add_dependency "async-service-supervisor", "~> 0.20" + spec.add_dependency "process-metrics", "~> 0.12" end diff --git a/control-plane/test/envoy.rb b/control-plane/test/envoy.rb index 7a391bf..76f2258 100644 --- a/control-plane/test/envoy.rb +++ b/control-plane/test/envoy.rb @@ -17,43 +17,45 @@ def eventually(timeout: 20, interval: 0.5) while Process.clock_gettime(Process::CLOCK_MONOTONIC) < deadline begin - return yield + if result = yield + return result + end rescue => error - sleep interval end + + sleep interval end - raise error + raise error || "Condition was not met within #{timeout} seconds!" end it "routes requests through Envoy to supervised Falcon workers" do uri = envoy_uri - backend_ids = eventually do - 20.times.map do - response = Net::HTTP.get_response(uri) - - expect(response.code.to_i).to be == 200 - expect(response.body).to be =~ /Hello from backend-[ab]/ - - response["x-backend-id"] - end + responses = eventually do + responses = 20.times.map{Net::HTTP.get_response(uri)} + backend_ids = responses.filter_map{|response| response["x-backend-id"]}.uniq.sort + + responses if backend_ids == ["backend-a", "backend-b"] end - expect(backend_ids.compact.uniq.sort).to be == ["backend-a", "backend-b"] + responses.each do |response| + expect(response.code.to_i).to be == 200 + expect(response.body).to be =~ /Hello from backend-[ab]/ + end + + expect(responses.filter_map{|response| response["x-backend-id"]}.uniq.sort).to be == ["backend-a", "backend-b"] end it "loads the xDS cluster from the supervisor monitor" do uri = admin_uri + "/clusters?format=json" cluster_status = eventually do - response = Net::HTTP.get_response(uri) - - expect(response.code.to_i).to be == 200 - - clusters = JSON.parse(response.body) - clusters.fetch("cluster_statuses").find do |cluster| - cluster.fetch("name") == "app-http1" + if (response = Net::HTTP.get_response(uri)).code.to_i == 200 + clusters = JSON.parse(response.body) + clusters.fetch("cluster_statuses").find do |cluster| + cluster.fetch("name") == "app-http1" + end end end diff --git a/guides/getting-started/readme.md b/guides/getting-started/readme.md index 4d5fbd1..c1ea351 100644 --- a/guides/getting-started/readme.md +++ b/guides/getting-started/readme.md @@ -94,3 +94,34 @@ Async::Service::Supervisor::Envoy::Monitor.new( ``` Disconnected workers are removed from EDS. Registered workers that fail the delegate health check remain in EDS with an unhealthy endpoint status. + +## Load-aware Balancing + +Enable out-of-band ORCA reporting to let Envoy weight independently bound workers using their current processor utilization and request throughput: + +``` ruby +utilization_monitor = Async::Service::Supervisor::UtilizationMonitor.new(interval: 1) + +[ + utilization_monitor, + Async::Service::Supervisor::Envoy::Monitor.new( + bind: "http://0.0.0.0:18000", + orca: true, + utilization_monitor: utilization_monitor, + interval: 1 + ) +] +``` + +The supervisor utilization monitor manages each worker's shared-memory allocation and registration. The Envoy monitor samples it through `sample_by_worker`, combines each worker's `requests_total` counter with processor usage from `process-metrics`, and serves the resulting ORCA reports from the same HTTP/2 endpoint as ADS. It also configures each discovered cluster to use Envoy's client-side weighted-round-robin policy. + +The first sample establishes a baseline. Subsequent reports contain normalized `cpu_utilization` and `rps_fractional` values for each worker. Reports are removed immediately when a worker disconnects. + +Out-of-band ORCA requires: + + - Envoy 1.39 or later. + - A fixed TCP port for the monitor's `bind` address. + - A supervisor utilization monitor registered alongside the Envoy monitor. + - Independently addressable TCP worker endpoints. Unix sockets and endpoints shared by several workers cannot provide distinct per-worker ORCA identities. + +The monitor address and worker addresses must be reachable from Envoy. In a sidecar deployment, binding the monitor to a fixed port in the shared network namespace satisfies this requirement. diff --git a/lib/async/service/supervisor/envoy.rb b/lib/async/service/supervisor/envoy.rb index a3c6ed7..d0b53b0 100644 --- a/lib/async/service/supervisor/envoy.rb +++ b/lib/async/service/supervisor/envoy.rb @@ -7,4 +7,5 @@ require_relative "envoy/delegate" require_relative "envoy/endpoint" require_relative "envoy/monitor" +require_relative "envoy/orca_service" require_relative "envoy/supervised" diff --git a/lib/async/service/supervisor/envoy/monitor.rb b/lib/async/service/supervisor/envoy/monitor.rb index 0d26f4f..beab1fd 100644 --- a/lib/async/service/supervisor/envoy/monitor.rb +++ b/lib/async/service/supervisor/envoy/monitor.rb @@ -5,12 +5,17 @@ require "async/http/endpoint" require "async/service/supervisor/monitor" +require "async/service/supervisor/utilization_monitor" +require "async/grpc/xds/client_side_weighted_round_robin" require "async/grpc/xds/control_plane" require "async/grpc/xds/server" +require "process/metrics" +require "xds/data/orca/v3/orca_load_report_pb" require_relative "delegate" require_relative "endpoint" require_relative "endpoint_group" +require_relative "orca_service" module Async module Service @@ -23,21 +28,45 @@ class Monitor < Async::Service::Supervisor::Monitor # @parameter bind [String | Nil] The optional address for the xDS control plane server. # @parameter delegate [Delegate] The delegate used to map supervisor state into Envoy endpoints. # @parameter control_plane [Async::GRPC::XDS::ControlPlane] The xDS control plane to update. + # @parameter orca [Boolean] Whether to collect and serve per-worker ORCA load reports. + # @parameter processor [Process::Metrics::Processor | Nil] The optional process CPU sampler. + # @parameter utilization_monitor [Async::Service::Supervisor::UtilizationMonitor | Nil] The per-worker utilization monitor used for ORCA reporting. + # @parameter interval [Numeric] The endpoint reconciliation and ORCA reporting interval in seconds. def initialize( bind: nil, delegate: Delegate.new, control_plane: Async::GRPC::XDS::ControlPlane.new, + orca: false, + processor: nil, + utilization_monitor: nil, + interval: 1, **options ) - super(**options) + super(interval: interval, **options) @bind = bind @delegate = delegate @control_plane = control_plane + @interval = interval + @orca = orca @controllers = {} @published_clusters = {} @server_task = nil @mutex = Mutex.new + + if @orca + raise ArgumentError, "ORCA reporting requires a TCP bind address!" unless @bind + raise ArgumentError, "ORCA reporting requires a utilization monitor!" unless utilization_monitor + + @orca_port = server_endpoint.url.port + raise ArgumentError, "ORCA reporting requires a fixed TCP port!" unless @orca_port&.positive? + + @processor = processor || Process::Metrics::Processor.new + @utilization_monitor = utilization_monitor + @request_totals = {} + @load_reports = {} + @authorities = {} + end end # @attribute [Async::GRPC::XDS::ControlPlane] The xDS control plane receiving cluster and endpoint updates. @@ -52,6 +81,7 @@ def initialize( def register(supervisor_controller) @mutex.synchronize do @controllers[supervisor_controller.id] = supervisor_controller + @authorities[worker_hostname(supervisor_controller)] = supervisor_controller.id if @orca reconcile end end @@ -62,6 +92,12 @@ def register(supervisor_controller) def remove(supervisor_controller) @mutex.synchronize do @controllers.delete(supervisor_controller.id) + if @orca + hostname = worker_hostname(supervisor_controller) + @authorities.delete(hostname) + @load_reports.delete(hostname) + @request_totals.delete(supervisor_controller.id) + end reconcile end end @@ -74,8 +110,9 @@ def run(parent: Async::Task.current) if @bind @server_task = parent.async do - endpoint = Async::HTTP::Endpoint.parse(@bind, protocol: Async::HTTP::Protocol::HTTP2) - Async::GRPC::XDS::Server.new(@control_plane).run(endpoint) + server = Async::GRPC::XDS::Server.new(@control_plane) + server.dispatcher.register(ORCAService.new(self, minimum_interval: @interval)) if @orca + server.run(server_endpoint) end end @@ -92,9 +129,29 @@ def as_json end end + # Determine whether an ORCA worker authority is currently registered. + # @parameter authority [String] The gRPC request authority. + # @returns [Boolean] Whether the authority identifies a live worker. + def worker?(authority) + return false unless @orca + + @mutex.synchronize{@authorities.key?(authority)} + end + + # Get the latest ORCA report for a worker authority. + # @parameter authority [String] The gRPC request authority. + # @returns [Xds::Data::Orca::V3::OrcaLoadReport | Nil] The latest valid report, if available. + def load_report(authority) + return unless @orca + + @mutex.synchronize{@load_reports[authority]} + end + # Refresh endpoint health and publish updated EDS state. # @returns [void] def run_once + sample_load_reports if @orca + @mutex.synchronize do reconcile end @@ -102,6 +159,14 @@ def run_once private + def server_endpoint + @server_endpoint ||= Async::HTTP::Endpoint.parse(@bind, protocol: Async::HTTP::Protocol::HTTP2) + end + + def worker_hostname(supervisor_controller) + "worker-#{supervisor_controller.id}" + end + def build_record(supervisor_controller, endpoint) cluster = @delegate.cluster(supervisor_controller, endpoint) return unless cluster @@ -148,14 +213,24 @@ def build_records_by_cluster def build_clusters(records_by_cluster = build_records_by_cluster) records_by_cluster.transform_values do |records| - groups = {} - - records.each do |record| - group = groups[record[:endpoint]] ||= EndpointGroup.new(record[:endpoint]) - group.add(record[:worker], healthy: record[:healthy]) + if @orca + records.map do |record| + { + addresses: record[:endpoint].addresses, + healthy: record[:healthy], + hostname: worker_hostname(record[:worker]), + } + end + else + groups = {} + + records.each do |record| + group = groups[record[:endpoint]] ||= EndpointGroup.new(record[:endpoint]) + group.add(record[:worker], healthy: record[:healthy]) + end + + groups.each_value.map(&:as_json) end - - groups.each_value.map(&:as_json) end end @@ -168,7 +243,71 @@ def cluster_configuration(records) raise ArgumentError, "Envoy cluster contains no common protocols: #{protocols.inspect}" if common_protocols.empty? raise ArgumentError, "HTTPS upstream endpoints are not yet supported!" if schemes.first == :https - {protocol: envoy_protocol(common_protocols)} + configuration = {protocol: envoy_protocol(common_protocols)} + + if @orca + if records.any?{|record| record[:endpoint].addresses.any?{|address| address[:path]}} + raise ArgumentError, "Out-of-band ORCA reporting requires IP endpoints!" + end + + configuration[:load_balancing_policy] = Async::GRPC::XDS::ClientSideWeightedRoundRobin.build( + @orca_port, + reporting_period: @interval + ) + end + + configuration + end + + def sample_load_reports + controllers = @mutex.synchronize{@controllers.dup} + workers = @utilization_monitor.sample_by_worker + process_ids = controllers.each_value.filter_map(&:process_id) + processor_samples = @processor.sample(process_ids) + request_totals = {} + + workers.each do |worker_id, worker| + requests_total = worker[:utilization][:requests_total] + if requests_total.is_a?(Numeric) && requests_total.finite? + request_totals[worker_id] = requests_total + end + end + + @mutex.synchronize do + controllers.each do |worker_id, controller| + next unless @controllers[worker_id].equal?(controller) + + hostname = worker_hostname(controller) + processor_sample = processor_samples[controller.process_id] + requests_total = request_totals[worker_id] + previous_requests_total = @request_totals[worker_id] + + if processor_sample && requests_total && previous_requests_total && requests_total >= previous_requests_total + rps = (requests_total - previous_requests_total).fdiv(processor_sample.duration) + cpu = processor_sample.utilization + + if rps.finite? && cpu.finite? + @load_reports[hostname] = Xds::Data::Orca::V3::OrcaLoadReport.new( + cpu_utilization: cpu, + rps_fractional: rps, + named_metrics: {"orca.heartbeat" => 0.0}, + ) + next + end + end + + @load_reports.delete(hostname) + end + + @request_totals = request_totals + + Console.debug(self, "Sampled ORCA load reports.", + workers: workers.keys, + processes: processor_samples.keys, + requests: request_totals, + reports: @load_reports.keys, + ) + end end def envoy_protocol(protocols) diff --git a/lib/async/service/supervisor/envoy/orca_service.rb b/lib/async/service/supervisor/envoy/orca_service.rb new file mode 100644 index 0000000..c78a1a8 --- /dev/null +++ b/lib/async/service/supervisor/envoy/orca_service.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2026, by Samuel Williams. + +require "async/grpc/service" +require "protocol/http/body/writable" +require "xds/service/orca/v3/open_rca_service" + +module Async + module Service + module Supervisor + module Envoy + # Streams per-worker out-of-band ORCA load reports to Envoy. + class ORCAService < Async::GRPC::Service + SERVICE_NAME = "xds.service.orca.v3.OpenRcaService" + EMPTY_REPORT = Xds::Data::Orca::V3::OrcaLoadReport.new( + named_metrics: {"orca.heartbeat" => 0.0} + ) + + # Initialize the ORCA service. + # @parameter monitor [Monitor] The monitor providing worker load reports. + # @parameter minimum_interval [Numeric] The minimum reporting interval in seconds. + def initialize(monitor, minimum_interval: 1) + super(Xds::Service::Orca::V3::OpenRcaService, SERVICE_NAME) + + @monitor = monitor + @minimum_interval = minimum_interval + end + + # Stream current load reports for the worker named by the request authority. + # @parameter input [Interface(:read)] The ORCA request stream. + # @parameter output [Interface(:write)] The ORCA report stream. + # @parameter call [Protocol::GRPC::Call] The gRPC call context. + # @asynchronous + def stream_core_metrics(input, output, call) + request = input.read + return unless request + + authority = call.request.authority + interval = [duration(request.report_interval), @minimum_interval].max + + while @monitor.worker?(authority) + output.write(@monitor.load_report(authority) || EMPTY_REPORT) + + sleep(interval) + end + rescue Protocol::HTTP::Body::Writable::Closed + # The client closed the reporting stream. + end + + private + + def duration(value) + return 0 unless value + + value.seconds + value.nanos.fdiv(1_000_000_000) + end + end + end + end + end +end diff --git a/readme.md b/readme.md index 1849978..4ae9417 100644 --- a/readme.md +++ b/readme.md @@ -13,6 +13,7 @@ Provides an Envoy xDS monitor for `async-service-supervisor`. - **Multiple clusters** - Groups workers by `state[:name]` by default. - **Endpoint contract** - Converts concrete post-bind worker listeners into Envoy upstream endpoints, including grouped IP or Unix socket addresses. - **Delegate mapping** - Uses a delegate object for endpoint selection, cluster grouping, and health without active probing. + - **Load-aware balancing** - Optionally reports per-worker CPU utilization and request throughput to Envoy using out-of-band ORCA. ## Usage diff --git a/releases.md b/releases.md index 85b8bbb..66e3183 100644 --- a/releases.md +++ b/releases.md @@ -1,5 +1,9 @@ # Releases +## Unreleased + + - Add supervisor-driven out-of-band ORCA load reporting for independently addressable workers. + ## v0.2.0 - Accept the required Falcon listener as a positional worker preparation argument. diff --git a/test/async/service/supervisor/envoy/monitor.rb b/test/async/service/supervisor/envoy/monitor.rb index 62716d1..4724b23 100644 --- a/test/async/service/supervisor/envoy/monitor.rb +++ b/test/async/service/supervisor/envoy/monitor.rb @@ -7,7 +7,7 @@ require "envoy/config/endpoint/v3/endpoint_pb" describe Async::Service::Supervisor::Envoy::Monitor do - Controller = Struct.new(:id, :state) + Controller = Struct.new(:id, :state, :process_id) let(:monitor) {subject.new} let(:control_plane) {monitor.control_plane} @@ -21,6 +21,18 @@ def endpoint_assignment(cluster) Envoy::Config::Endpoint::V3::ClusterLoadAssignment.decode(response.resources.first.value) end + def utilization_monitor(samples = []) + Object.new.tap do |monitor| + monitor.define_singleton_method(:sample_by_worker){samples.shift || {}} + end + end + + def processor(samples = []) + Object.new.tap do |processor| + processor.define_singleton_method(:sample){|process_ids| samples.shift || {}} + end + end + it "publishes registered endpoints" do controller = Controller.new(1, { endpoint: {name: "myservice", scheme: "http", protocols: ["h2"], addresses: [{address: "127.0.0.1", port: 50051}]} @@ -36,6 +48,99 @@ def endpoint_assignment(cluster) expect(load_balancer_endpoint.endpoint.address.socket_address.port_value).to be == 50051 end + it "publishes ORCA worker identity and load-balancing configuration" do + monitor = subject.new( + bind: "http://127.0.0.1:18000", + orca: true, + processor: processor, + utilization_monitor: utilization_monitor + ) + controller = Controller.new(1, { + endpoint: {name: "myservice", scheme: "http", protocols: ["h2"], addresses: [{address: "127.0.0.1", port: 50051}]} + }, 123) + + monitor.register(controller) + + response = monitor.control_plane.response( + Async::GRPC::XDS::ControlPlane::ENDPOINT_TYPE, + ["myservice"] + ) + assignment = Envoy::Config::Endpoint::V3::ClusterLoadAssignment.decode(response.resources.first.value) + cluster = monitor.control_plane.resources(Async::GRPC::XDS::ControlPlane::CLUSTER_TYPE).first + typed_configuration = cluster.load_balancing_policy.policies.first.typed_extension_config + configuration = Envoy::Extensions::LoadBalancingPolicies::ClientSideWeightedRoundRobin::V3::ClientSideWeightedRoundRobin.decode( + typed_configuration.typed_config.value + ) + + expect(assignment.endpoints.first.lb_endpoints.first.endpoint.hostname).to be == "worker-1" + expect(configuration.enable_oob_load_report.value).to be == true + expect(configuration.oob_reporting_config.port_value).to be == 18000 + end + + it "samples per-worker ORCA load reports" do + utilization = utilization_monitor([ + {1 => {state: {}, utilization: {requests_total: 10}}}, + {1 => {state: {}, utilization: {requests_total: 14}}}, + ]) + processor_sample = Struct.new(:duration, :utilization).new(2.0, 0.5) + processor = processor([ + {}, + {123 => processor_sample}, + ]) + monitor = subject.new( + bind: "http://127.0.0.1:18000", + orca: true, + processor: processor, + utilization_monitor: utilization + ) + controller = Controller.new(1, { + endpoint: {name: "myservice", scheme: "http", protocols: ["h2"], addresses: [{address: "127.0.0.1", port: 50051}]} + }, 123) + + monitor.register(controller) + monitor.run_once + expect(monitor.load_report("worker-1")).to be_nil + + monitor.run_once + report = monitor.load_report("worker-1") + + expect(report.cpu_utilization).to be == 0.5 + expect(report.rps_fractional).to be == 2.0 + expect(report.named_metrics).to be == {"orca.heartbeat" => 0.0} + expect(monitor.worker?("worker-1")).to be == true + + monitor.remove(controller) + expect(monitor.worker?("worker-1")).to be == false + expect(monitor.load_report("worker-1")).to be_nil + end + + it "requires a fixed bind address for ORCA" do + expect do + subject.new(orca: true) + end.to raise_exception(ArgumentError) + end + + it "requires a utilization monitor for ORCA" do + expect do + subject.new(bind: "http://127.0.0.1:18000", orca: true) + end.to raise_exception(ArgumentError) + end + + it "rejects Unix endpoints for out-of-band ORCA" do + monitor = subject.new( + bind: "http://127.0.0.1:18000", + orca: true, + processor: processor, + utilization_monitor: utilization_monitor + ) + + expect do + monitor.register(Controller.new(1, { + endpoint: {name: "myservice", scheme: "http", protocols: ["h2"], addresses: [{path: "/tmp/falcon.ipc"}]} + }, 123)) + end.to raise_exception(ArgumentError) + end + it "publishes a supervised Unix endpoint" do controller = Controller.new(1, { endpoint: { diff --git a/test/async/service/supervisor/envoy/orca_service.rb b/test/async/service/supervisor/envoy/orca_service.rb new file mode 100644 index 0000000..4ec61b6 --- /dev/null +++ b/test/async/service/supervisor/envoy/orca_service.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2026, by Samuel Williams. + +require "async/service/supervisor/envoy/orca_service" +require "google/protobuf/duration_pb" + +describe Async::Service::Supervisor::Envoy::ORCAService do + it "streams reports for the requested worker authority" do + report = Xds::Data::Orca::V3::OrcaLoadReport.new(cpu_utilization: 0.5, rps_fractional: 2.0) + checks = [true, false] + monitor = Object.new + monitor.define_singleton_method(:worker?){|authority| authority == "worker-1" && checks.shift} + monitor.define_singleton_method(:load_report){|authority| report} + + request = Xds::Service::Orca::V3::OrcaLoadReportRequest.new( + report_interval: Google::Protobuf::Duration.new + ) + input = Object.new + input.define_singleton_method(:read){request} + output = [] + output.define_singleton_method(:write){|value| self << value} + call = Struct.new(:request).new(Struct.new(:authority).new("worker-1")) + + subject.new(monitor, minimum_interval: 0).stream_core_metrics(input, output, call) + + expect(output).to be == [report] + end + + it "streams an empty report while establishing a baseline" do + checks = [true, false] + monitor = Object.new + monitor.define_singleton_method(:worker?){|authority| authority == "worker-1" && checks.shift} + monitor.define_singleton_method(:load_report){|authority| nil} + + request = Xds::Service::Orca::V3::OrcaLoadReportRequest.new + input = Object.new + input.define_singleton_method(:read){request} + output = [] + output.define_singleton_method(:write){|value| self << value} + call = Struct.new(:request).new(Struct.new(:authority).new("worker-1")) + + subject.new(monitor, minimum_interval: 0).stream_core_metrics(input, output, call) + + expect(output).to be == [subject::EMPTY_REPORT] + end + + it "stops when the client closes the stream" do + monitor = Object.new + monitor.define_singleton_method(:worker?){|authority| true} + monitor.define_singleton_method(:load_report){|authority| nil} + + input = Object.new + input.define_singleton_method(:read){Xds::Service::Orca::V3::OrcaLoadReportRequest.new} + output = Object.new + output.define_singleton_method(:write){|value| raise Protocol::HTTP::Body::Writable::Closed} + call = Struct.new(:request).new(Struct.new(:authority).new("worker-1")) + + expect do + subject.new(monitor, minimum_interval: 0).stream_core_metrics(input, output, call) + end.not.to raise_exception + end +end