From 4b8de9450b0718c6ce695a3b3094e19d4cd3294c Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Fri, 7 Aug 2026 19:09:31 +1200 Subject: [PATCH] Consolidate protobuf maintenance tasks --- async-grpc-xds.gemspec | 2 +- bake.rb | 75 ++++++++++++++++++++++++ bake/async/grpc/xds.rb | 126 ----------------------------------------- proto/readme.md | 32 +++-------- xds/update_protos.sh | 123 ---------------------------------------- 5 files changed, 84 insertions(+), 274 deletions(-) delete mode 100644 bake/async/grpc/xds.rb delete mode 100755 xds/update_protos.sh diff --git a/async-grpc-xds.gemspec b/async-grpc-xds.gemspec index 323435b..f9b6e54 100644 --- a/async-grpc-xds.gemspec +++ b/async-grpc-xds.gemspec @@ -20,7 +20,7 @@ Gem::Specification.new do |spec| "source_code_uri" => "https://github.com/socketry/async-grpc-xds.git", } - spec.files = Dir.glob(["{context,lib,proto,xds}/**/*", "*.md"], File::FNM_DOTMATCH, base: __dir__) + spec.files = Dir.glob(["{context,lib,proto}/**/*", "*.md"], File::FNM_DOTMATCH, base: __dir__) spec.required_ruby_version = ">= 3.3" diff --git a/bake.rb b/bake.rb index 9441211..7cd1311 100644 --- a/bake.rb +++ b/bake.rb @@ -3,6 +3,81 @@ # Released under the MIT License. # Copyright, 2025-2026, by Samuel Williams. +require "fileutils" +require "tmpdir" + +PROTOBUF_SOURCES = { + "envoy" => ["https://github.com/envoyproxy/data-plane-api.git", "84e84367f2560cdb47b9bb78fd3e615feb80c3e4"], + "google/protobuf" => ["https://github.com/protocolbuffers/protobuf.git", "68cb3eaca7cf86ee8eec7e1b54523643fd6aa344", "src"], + "google/rpc" => ["https://github.com/googleapis/api-common-protos.git", "3332dec527759859840a3a2ff108c67a54708130"], + "xds" => ["https://github.com/cncf/xds.git", "dba9d589def2cd10099a3a64887d859188c2f57a"], + "udpa" => ["https://github.com/cncf/udpa.git", "c52dc94e7fbe6449d8465faaeda22c76ca62d4ff"], + "validate" => ["https://github.com/envoyproxy/protoc-gen-validate.git", "414042a5ff2e98dc47f8161937316a25b1da5bba"], +}.freeze + +# Update the vendored protobuf definitions from pinned upstream revisions and regenerate their Ruby classes. +# +# @parameter proto_dir [String] The directory containing vendored protobuf definitions. +# @parameter output_dir [String] The directory containing generated Ruby classes. +def update_protos(proto_dir: "proto", output_dir: "lib") + proto_dir = File.expand_path(proto_dir) + files = Dir.glob(File.join(proto_dir, "**/*.proto")) + + Dir.mktmpdir("async-grpc-xds-protos") do |temporary_root| + repositories = {} + + PROTOBUF_SOURCES.each_value do |url, revision, source_root| + repository = File.join(temporary_root, repositories.size.to_s) + system("git", "init", "--quiet", repository) or raise "Could not initialize a repository for #{url}!" + system("git", "-C", repository, "remote", "add", "origin", url) or raise "Could not configure #{url}!" + system("git", "-C", repository, "fetch", "--quiet", "--depth", "1", "origin", revision) or raise "Could not fetch #{revision} from #{url}!" + system("git", "-C", repository, "checkout", "--quiet", "FETCH_HEAD") or raise "Could not check out #{revision} from #{url}!" + + repositories[url] = File.join(repository, source_root.to_s) + end + + files.each do |destination| + relative_path = destination.delete_prefix("#{proto_dir}/") + prefix, source = PROTOBUF_SOURCES.find{|prefix, _source| relative_path.start_with?("#{prefix}/")} + raise "No upstream source for #{relative_path}!" unless source + + url = source.first + source_path = File.join(repositories.fetch(url), relative_path) + raise "Could not find #{relative_path} in #{url}!" unless File.file?(source_path) + + FileUtils.cp(source_path, destination) + end + end + + { + updated: files.size, + sources: PROTOBUF_SOURCES.transform_values{|_url, revision, _source_root| revision}, + generated: generate_protos(proto_dir: proto_dir, output_dir: output_dir), + } +end + +# Generate the checked-in Ruby classes from the vendored protobuf definitions. +# +# @parameter proto_dir [String] The directory containing vendored protobuf definitions. +# @parameter output_dir [String] The output directory for generated Ruby classes. +def generate_protos(proto_dir: "proto", output_dir: "lib") + proto_dir = File.expand_path(proto_dir) + output_dir = File.expand_path(output_dir) + + files = Dir.glob(File.join(output_dir, "**/*_pb.rb")).map do |path| + relative_path = path.delete_prefix("#{output_dir}/").sub(/_pb\.rb\z/, ".proto") + File.join(proto_dir, relative_path) + end + + missing = files.reject{|path| File.file?(path)} + raise "Missing protobuf definitions: #{missing.join(', ')}" unless missing.empty? + + FileUtils.mkdir_p(output_dir) + system("protoc", "--ruby_out=#{output_dir}", "--proto_path=#{proto_dir}", *files) or raise "Could not generate Ruby protobuf classes!" + + files.size +end + # Update the project documentation with the new version number. # # @parameter version [String] The new version number. diff --git a/bake/async/grpc/xds.rb b/bake/async/grpc/xds.rb deleted file mode 100644 index 5af662a..0000000 --- a/bake/async/grpc/xds.rb +++ /dev/null @@ -1,126 +0,0 @@ -# frozen_string_literal: true - -# Released under the MIT License. -# Copyright, 2026, by Samuel Williams. - -# Generate Ruby protobuf classes from Envoy .proto files -# @parameter proto_dir [String] Directory containing .proto files (default: "proto") -# @parameter output_dir [String] Output directory for generated Ruby files (default: "lib") -def generate_protos(proto_dir: "proto", output_dir: "lib") - require "fileutils" - - proto_dir = File.expand_path(proto_dir) - output_dir = File.expand_path(output_dir) - - # Core discovery service files (most important) - discovery_files = [ - "envoy/service/discovery/v3/discovery.proto", - "envoy/service/discovery/v3/ads.proto" - ] - - orca_files = [ - "xds/data/orca/v3/orca_load_report.proto", - "xds/service/orca/v3/orca.proto", - "envoy/config/common/mutation_rules/v3/mutation_rules.proto", - "envoy/config/route/v3/route_components.proto", - "envoy/type/tracing/v3/custom_tag.proto", - "envoy/extensions/load_balancing_policies/common/v3/common.proto", - "envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.proto" - ] - - # Core config files needed for discovery - config_files = [ - "envoy/config/core/v3/base.proto", - "envoy/config/core/v3/address.proto", - "envoy/config/core/v3/config_source.proto", - "envoy/config/cluster/v3/cluster.proto", - "envoy/config/endpoint/v3/endpoint.proto" - ] - - # Google protobuf well-known types - google_files = [ - "google/protobuf/any.proto", - "google/protobuf/duration.proto", - "google/protobuf/timestamp.proto", - "google/protobuf/struct.proto", - "google/protobuf/empty.proto", - "google/protobuf/wrappers.proto", - "google/rpc/status.proto" - ] - - all_files = discovery_files + orca_files + config_files + google_files - - # Create output directories - FileUtils.mkdir_p(output_dir) - - # Generate Ruby code - all_files.each do |proto_file| - full_path = File.join(proto_dir, proto_file) - next unless File.exist?(full_path) - - Console.info{"Generating #{proto_file}..."} - - system( - "protoc", - "--ruby_out=#{output_dir}", - "--proto_path=#{proto_dir}", - "--proto_path=#{File.join(proto_dir, 'google')}", - full_path, - out: File::NULL, - err: File::NULL - ) or begin - Console.warn{"Failed to generate #{proto_file} (may have missing dependencies)"} - end - end - - # Count generated files - generated = Dir.glob(File.join(output_dir, "**/*_pb.rb")).count - - Console.info{"Generated #{generated} protobuf Ruby files in #{output_dir}"} -end - -# Generate all protobuf files (including optional dependencies) -# This will attempt to generate all .proto files, even if some fail -# @parameter proto_dir [String] Directory containing .proto files (default: "proto") -# @parameter output_dir [String] Output directory for generated Ruby files (default: "lib") -def generate_all_protos(proto_dir: "proto", output_dir: "lib") - require "fileutils" - - proto_dir = File.expand_path(proto_dir) - output_dir = File.expand_path(output_dir) - - # Find all .proto files - proto_files = Dir.glob(File.join(proto_dir, "**/*.proto")) - - Console.info{"Found #{proto_files.count} .proto files"} - - # Generate each file - success_count = 0 - fail_count = 0 - - proto_files.each do |proto_file| - relative_path = proto_file.sub(/^#{proto_dir}\//, "") - - Console.debug{"Generating #{relative_path}..."} - - if system( - "protoc", - "--ruby_out=#{output_dir}", - "--proto_path=#{proto_dir}", - "--proto_path=#{File.join(proto_dir, 'google')}", - proto_file, - out: File::NULL, - err: File::NULL - ) - success_count += 1 - else - fail_count += 1 - Console.debug{"Failed: #{relative_path}"} - end - end - - # Count generated files - generated = Dir.glob(File.join(output_dir, "**/*_pb.rb")).count - - Console.info{"Generated #{generated} protobuf Ruby files (#{success_count} succeeded, #{fail_count} failed)"} -end diff --git a/proto/readme.md b/proto/readme.md index 4f141d8..a8de0a6 100644 --- a/proto/readme.md +++ b/proto/readme.md @@ -25,45 +25,29 @@ These files come from [envoyproxy/data-plane-api](https://github.com/envoyproxy/ To update these files, run: ```bash -./xds/update_protos.sh +bundle exec bake update_protos ``` -Or manually: - -```bash -# Clone envoy data-plane-api -git clone --depth 1 https://github.com/envoyproxy/data-plane-api.git /tmp/envoy-api - -# Copy needed files -cp -r /tmp/envoy-api/envoy proto/ -cp -r /tmp/envoy-api/google proto/ - -# Cleanup -rm -rf /tmp/envoy-api -``` +This fetches the pinned upstream revisions defined in `bake.rb`, updates the vendored definitions, and regenerates the checked-in Ruby classes. ## Generating Ruby Code After updating proto files, generate Ruby classes: ```bash -bundle exec bake async:grpc:xds:generate_protos +bundle exec bake generate_protos ``` ## Version -These files are from the latest `main` branch of: +The source repositories and revisions are defined by `PROTOBUF_SOURCES` in `bake.rb`. They include: + - `envoyproxy/data-plane-api` - Envoy API definitions - `protocolbuffers/protobuf` - Google protobuf well-known types - `googleapis/api-common-protos` - Google RPC status - -To lock to a specific version, modify `xds/update_protos.sh` to check out specific tags: - -```bash -cd /tmp/envoy-api -git checkout v1.30.0 # Use specific Envoy version -# Then copy files -``` +- `cncf/xds` - xDS API definitions +- `cncf/udpa` - UDPA annotations +- `envoyproxy/protoc-gen-validate` - Validation annotations ## Note on Dependencies diff --git a/xds/update_protos.sh b/xds/update_protos.sh deleted file mode 100755 index 129375b..0000000 --- a/xds/update_protos.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/bash -# Update Envoy protobuf definitions -# This script clones the envoy data-plane-api and copies only the needed .proto files - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -PROTO_DIR="$PROJECT_ROOT/proto" -TEMP_DIR="/tmp/envoy-api-$$" - -echo "Cloning envoyproxy/data-plane-api..." - -# Clone with sparse checkout to only get what we need -git clone --depth 1 --filter=blob:none --sparse https://github.com/envoyproxy/data-plane-api.git "$TEMP_DIR" - -cd "$TEMP_DIR" - -echo "Setting up sparse checkout..." - -# First get envoy config files -git sparse-checkout set \ - envoy/config/cluster/v3 \ - envoy/config/endpoint/v3 \ - envoy/config/listener/v3 \ - envoy/config/route/v3 \ - envoy/config/core/v3 \ - envoy/extensions/transport_sockets/tls/v3 - -# Copy envoy config files -cp -r envoy "$PROTO_DIR/" - -# Now get discovery service and google protobuf files -git sparse-checkout set \ - envoy/service/discovery/v3 \ - google/protobuf - -# Copy discovery service -cp -r envoy/service "$PROTO_DIR/envoy/" - -# Copy google protobuf (if exists in repo) -if [ -d "google" ]; then - cp -r google "$PROTO_DIR/" -fi - -# Get Google protobuf well-known types from protobuf repo -echo "Fetching Google protobuf well-known types..." -git clone --depth 1 https://github.com/protocolbuffers/protobuf.git /tmp/protobuf-$$ -mkdir -p "$PROTO_DIR/google/protobuf" -cp /tmp/protobuf-$$/src/google/protobuf/{any,duration,timestamp,struct,empty,wrappers}.proto "$PROTO_DIR/google/protobuf/" 2>/dev/null || true -rm -rf /tmp/protobuf-$$ - -# Get google/rpc/status.proto from api-common-protos -echo "Fetching google/rpc/status.proto..." -git clone --depth 1 https://github.com/googleapis/api-common-protos.git /tmp/api-common-$$ -mkdir -p "$PROTO_DIR/google/rpc" -cp /tmp/api-common-$$/google/rpc/status.proto "$PROTO_DIR/google/rpc/" 2>/dev/null || true -rm -rf /tmp/api-common-$$ - -# Get envoy/type/v3 and xds/core/v3 from data-plane-api -echo "Fetching envoy/type/v3 and xds/core/v3..." -git clone --depth 1 --filter=blob:none --sparse https://github.com/envoyproxy/data-plane-api.git /tmp/envoy-types-$$ -cd /tmp/envoy-types-$$ -git sparse-checkout set envoy/type/v3 envoy/type/matcher/v3 envoy/type/metadata/v3 xds/core/v3 -mkdir -p "$PROTO_DIR/envoy/type/v3" "$PROTO_DIR/envoy/type/matcher/v3" "$PROTO_DIR/envoy/type/metadata/v3" "$PROTO_DIR/xds/core/v3" -cp -r envoy/type "$PROTO_DIR/envoy/" 2>/dev/null || true -cp -r xds/core "$PROTO_DIR/xds/" 2>/dev/null || true -rm -rf /tmp/envoy-types-$$ - -# Get xds/type/matcher/v3 and xds/core/v3 from cncf/xds repo -echo "Fetching xds/type/matcher/v3 and xds/core/v3..." -git clone --depth 1 https://github.com/cncf/xds.git /tmp/xds-types-$$ -mkdir -p "$PROTO_DIR/xds/type/matcher/v3" "$PROTO_DIR/xds/core/v3" -find /tmp/xds-types-$$/xds/type/matcher/v3 -name "*.proto" ! -name "*cel*" -exec cp {} "$PROTO_DIR/xds/type/matcher/v3/" \; -find /tmp/xds-types-$$/xds/core/v3 -name "*.proto" -exec cp {} "$PROTO_DIR/xds/core/v3/" \; -rm -rf /tmp/xds-types-$$ - -# Get udpa annotations -echo "Fetching udpa annotations..." -git clone --depth 1 https://github.com/cncf/udpa.git /tmp/udpa-$$ -mkdir -p "$PROTO_DIR/udpa/annotations" -cp /tmp/udpa-$$/udpa/annotations/*.proto "$PROTO_DIR/udpa/annotations/" 2>/dev/null || true -rm -rf /tmp/udpa-$$ - -# Get validate annotations -echo "Fetching validate annotations..." -git clone --depth 1 https://github.com/envoyproxy/protoc-gen-validate.git /tmp/validate-$$ -mkdir -p "$PROTO_DIR/validate" -cp /tmp/validate-$$/validate/validate.proto "$PROTO_DIR/validate/" 2>/dev/null || true -rm -rf /tmp/validate-$$ - -# Get envoy annotations -echo "Fetching envoy annotations..." -git clone --depth 1 --filter=blob:none --sparse https://github.com/envoyproxy/envoy.git /tmp/envoy-annotations-$$ -cd /tmp/envoy-annotations-$$ -git sparse-checkout set api/envoy/annotations -mkdir -p "$PROTO_DIR/envoy/annotations" -cp api/envoy/annotations/*.proto "$PROTO_DIR/envoy/annotations/" 2>/dev/null || true -rm -rf /tmp/envoy-annotations-$$ - -echo "Copying .proto files to $PROTO_DIR..." - -# Create directories -mkdir -p "$PROTO_DIR/envoy/service/discovery/v3" -mkdir -p "$PROTO_DIR/envoy/config/cluster/v3" -mkdir -p "$PROTO_DIR/envoy/config/endpoint/v3" -mkdir -p "$PROTO_DIR/envoy/config/listener/v3" -mkdir -p "$PROTO_DIR/envoy/config/route/v3" -mkdir -p "$PROTO_DIR/envoy/config/core/v3" -mkdir -p "$PROTO_DIR/envoy/extensions/transport_sockets/tls/v3" -mkdir -p "$PROTO_DIR/google/protobuf" - -# Copy files -cp -r envoy "$PROTO_DIR/" -cp -r google "$PROTO_DIR/" - -# Cleanup -rm -rf "$TEMP_DIR" - -echo "Done! Proto files updated in $PROTO_DIR" -echo "" -echo "To generate Ruby code, run:" -echo " bundle exec bake async:grpc:xds:generate_protos"