From 044be016c7e4ba065230082a627c73b431b041cf Mon Sep 17 00:00:00 2001 From: Maxim Tretyakov Date: Thu, 12 Feb 2026 19:31:49 +0500 Subject: [PATCH 1/2] Some optimizations for S3 cli --- lib/cloud_storage/wrappers/s3.rb | 45 +++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/lib/cloud_storage/wrappers/s3.rb b/lib/cloud_storage/wrappers/s3.rb index 73967ca..55b11af 100644 --- a/lib/cloud_storage/wrappers/s3.rb +++ b/lib/cloud_storage/wrappers/s3.rb @@ -24,18 +24,37 @@ def initialize(client, resource, bucket_name, **opts) @opts = opts end - def each + def each # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength return to_enum unless block_given? - @client.list_objects(bucket: @bucket_name, **@opts).contents.each do |item| - yield Objects::S3.new \ - item, - bucket_name: @bucket_name, - resource: @resource, - client: @client + counter = 0 + limit_exceeded = false + + @client.list_objects_v2(bucket: @bucket_name, **@opts).each do |page| + page.contents.each do |item| + yield build_object(item) + next unless @opts[:max_keys] + + counter += 1 + + limit_exceeded = counter >= @opts[:max_keys] + break if limit_exceeded + end + + break if limit_exceeded end rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound, Aws::S3::Errors::InvalidBucketName end + + private + + def build_object(item) + Objects::S3.new \ + item, + bucket_name: @bucket_name, + resource: @resource, + client: @client + end end def files(**opts) @@ -101,17 +120,13 @@ def resource def upload_file_or_io(key, file_or_io, **opts) if file_or_io.respond_to?(:path) - transfer_manager.upload_file(file_or_io.path, bucket: @bucket_name, key: key, **opts) + Aws::S3::TransferManager.new(client: client).upload_file \ + file_or_io.path, bucket: @bucket_name, key: key, **opts else - transfer_manager.upload_stream(bucket: @bucket_name, key: key, **opts) do |write_stream| - IO.copy_stream(file_or_io, write_stream) - end + client.put_object \ + bucket: @bucket_name, key: key, body: file_or_io, **opts end end - - def transfer_manager - @transfer_manager ||= Aws::S3::TransferManager.new(client: client) - end end end end From 3a879fbe3541b70ad276f69a5aa8588931235c15 Mon Sep 17 00:00:00 2001 From: Maxim Tretyakov Date: Thu, 12 Feb 2026 20:44:46 +0500 Subject: [PATCH 2/2] Bump version to 0.3.2 --- lib/cloud_storage/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cloud_storage/version.rb b/lib/cloud_storage/version.rb index a7abc3c..317e363 100644 --- a/lib/cloud_storage/version.rb +++ b/lib/cloud_storage/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module CloudStorage - VERSION = '0.3.1' + VERSION = '0.3.2' end