diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..74482a8 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,58 @@ +name: CI + +on: + pull_request: + branches: ['*'] + +jobs: + test: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + ruby-version: ['3.0', '3.1', '3.4', '4.0'] + + env: + S3_ENDPOINT: http://localhost:9000 + S3_BUCKET: test + GCS_ENDPOINT: http://localhost:8080/ + GCS_BUCKET: some-bucket + + steps: + - uses: actions/checkout@v4 + + - name: Start MinIO + run: | + docker run -d --name minio \ + -p 9000:9000 \ + --entrypoint sh \ + minio/minio:RELEASE.2025-04-22T22-12-26Z \ + -c 'mkdir -p /data/test && minio server /data --json' + + - name: Start fake-gcs-server + run: | + docker run -d --name gcs \ + -p 8080:8080 \ + --entrypoint sh \ + fsouza/fake-gcs-server \ + -c 'mkdir -p /data/some-bucket && /bin/fake-gcs-server -port 8080 -scheme http -external-url=http://localhost:8080 -public-host=localhost:8080 -filesystem-root /data' + + - name: Wait for services + run: | + for i in $(seq 1 30); do + curl -sf http://localhost:9000/minio/health/live && curl -sf http://localhost:8080/ && break + sleep 1 + done + + - name: Set up Ruby ${{ matrix.ruby-version }} + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true + + - name: Rubocop + run: bundle exec rubocop --display-style-guide --extra-details + + - name: RSpec + run: bundle exec rspec diff --git a/.pryrc b/.pryrc index 222e0ff..593ecf6 100644 --- a/.pryrc +++ b/.pryrc @@ -6,8 +6,8 @@ require 'cloud_storage' require 'cloud_storage/wrappers/gcs' require 'cloud_storage/wrappers/s3' -require_relative './spec/rspec_helpers/gcs' -require_relative './spec/rspec_helpers/s3' +require_relative 'spec/rspec_helpers/gcs' +require_relative 'spec/rspec_helpers/s3' # rubocop:disable Style/MixinUsage include RSpecHelpers::Gcs diff --git a/.rubocop.yml b/.rubocop.yml index b907bcf..4c3ae4a 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,7 +1,7 @@ inherit_from: - .rubocop_todo.yml -require: +plugins: - rubocop-rspec - rubocop-performance @@ -11,6 +11,7 @@ AllCops: Exclude: - 'bin/**/*' - 'uploads/**/*' + - 'vendor/**/*' Layout/LineLength: Max: 120 diff --git a/Gemfile b/Gemfile index a500a47..69fadb2 100644 --- a/Gemfile +++ b/Gemfile @@ -11,3 +11,9 @@ gem 'rubocop-rspec', require: false gem 'aws-sdk-s3', require: false gem 'google-cloud-storage', require: false + +gem 'pry-byebug' +gem 'rake' +gem 'rspec' +gem 'simplecov' +gem 'webmock' diff --git a/cloud_storage.gemspec b/cloud_storage.gemspec index 770cafa..7280025 100644 --- a/cloud_storage.gemspec +++ b/cloud_storage.gemspec @@ -16,12 +16,6 @@ Gem::Specification.new do |spec| spec.metadata['homepage_uri'] = spec.homepage spec.metadata['source_code_uri'] = 'https://github.com/wallarm/cloud_storage' - spec.add_development_dependency 'pry-byebug' - spec.add_development_dependency 'rake' - spec.add_development_dependency 'rspec' - spec.add_development_dependency 'simplecov' - spec.add_development_dependency 'webmock' - spec.files = Dir.chdir(File.expand_path(__dir__)) do `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) } end diff --git a/dip.yml b/dip.yml index 6546cfb..a45b238 100644 --- a/dip.yml +++ b/dip.yml @@ -1,7 +1,7 @@ version: '2' environment: - DOCKER_RUBY_VERSION: 3.0 + DOCKER_RUBY_VERSION: 3.4 S3_ENDPOINT: http://s3:9000 S3_BUCKET: wallarm-devtmp-ipfeeds-presigned-urls-research GCS_ENDPOINT: http://gcs:8080/ @@ -26,7 +26,6 @@ interaction: rspec: service: app - environment: command: bundle exec rspec rubocop: diff --git a/docker/Dockerfile.dip b/docker/Dockerfile.dip index f4ad700..593b655 100644 --- a/docker/Dockerfile.dip +++ b/docker/Dockerfile.dip @@ -1,5 +1,5 @@ ARG DOCKER_RUBY_VERSION -FROM ruby:${DOCKER_RUBY_VERSION}-alpine +FROM ruby:${DOCKER_RUBY_VERSION:-3.4}-alpine RUN gem update --system RUN apk add --update --no-cache less git build-base curl mc htop diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 3fa1eda..a43b10d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -24,7 +24,7 @@ services: - gcs s3: - image: minio/minio:edge + image: minio/minio:RELEASE.2025-04-22T22-12-26Z volumes: - ../uploads/s3:/data command: server /data --json @@ -40,5 +40,5 @@ services: volumes: bundler-data: - external: - name: bundler_data + name: bundler_data + external: true diff --git a/lib/cloud_storage.rb b/lib/cloud_storage.rb index e36ce4a..007b603 100644 --- a/lib/cloud_storage.rb +++ b/lib/cloud_storage.rb @@ -6,7 +6,8 @@ require_relative 'cloud_storage/objects/base' module CloudStorage - ObjectNotFound = Class.new(StandardError) + class ObjectNotFound < StandardError + end class << self def register_wrapper(klass) diff --git a/lib/cloud_storage/objects/gcs.rb b/lib/cloud_storage/objects/gcs.rb index 866fe8b..e48f8b9 100644 --- a/lib/cloud_storage/objects/gcs.rb +++ b/lib/cloud_storage/objects/gcs.rb @@ -4,7 +4,7 @@ module CloudStorage module Objects class Gcs < Base def initialize(internal, uri:) - super internal + super(internal) @uri = uri end diff --git a/lib/cloud_storage/objects/s3.rb b/lib/cloud_storage/objects/s3.rb index af77471..1fa0ac8 100644 --- a/lib/cloud_storage/objects/s3.rb +++ b/lib/cloud_storage/objects/s3.rb @@ -6,7 +6,7 @@ class S3 < Base attr_reader :bucket_name def initialize(internal, resource:, client:, bucket_name:) - super internal + super(internal) @bucket_name = bucket_name @resource = resource diff --git a/lib/cloud_storage/wrappers/s3.rb b/lib/cloud_storage/wrappers/s3.rb index 42d44f0..73967ca 100644 --- a/lib/cloud_storage/wrappers/s3.rb +++ b/lib/cloud_storage/wrappers/s3.rb @@ -34,7 +34,7 @@ def each resource: @resource, client: @client end - rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound + rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound, Aws::S3::Errors::InvalidBucketName end end @@ -44,19 +44,22 @@ def files(**opts) def exist?(key) resource.bucket(@bucket_name).object(key).exists? + rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound, + Aws::S3::Errors::InvalidBucketName, Aws::S3::Errors::BadRequest + false end def upload_file(key:, file:, **opts) - obj = resource.bucket(@bucket_name).object(key) + return unless upload_file_or_io(key, file, **opts) - return unless upload_file_or_io(obj, file, **opts) + obj = resource.bucket(@bucket_name).object(key) Objects::S3.new \ obj, bucket_name: @bucket_name, resource: resource, client: client - rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound + rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound, Aws::S3::Errors::InvalidBucketName raise ObjectNotFound, @bucket_name end @@ -70,6 +73,9 @@ def find(key) bucket_name: @bucket_name, resource: resource, client: client + rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound, + Aws::S3::Errors::InvalidBucketName, Aws::S3::Errors::BadRequest + raise ObjectNotFound, key end def delete_files(keys) @@ -80,7 +86,7 @@ def delete_files(keys) objects: keys.map { |key| { key: key } }, quiet: true } - rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound + rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound, Aws::S3::Errors::InvalidBucketName end private @@ -93,13 +99,19 @@ def resource @resource ||= Aws::S3::Resource.new(@options) end - def upload_file_or_io(obj, file_or_io, **opts) + def upload_file_or_io(key, file_or_io, **opts) if file_or_io.respond_to?(:path) - obj.upload_file(file_or_io.path, **opts) + transfer_manager.upload_file(file_or_io.path, bucket: @bucket_name, key: key, **opts) else - obj.upload_stream(**opts) { |write_stream| IO.copy_stream(file_or_io, write_stream) } + transfer_manager.upload_stream(bucket: @bucket_name, key: key, **opts) do |write_stream| + IO.copy_stream(file_or_io, write_stream) + end end end + + def transfer_manager + @transfer_manager ||= Aws::S3::TransferManager.new(client: client) + end end end end diff --git a/spec/cloud_storage/objects/gcs_spec.rb b/spec/cloud_storage/objects/gcs_spec.rb index 28d7a69..b0a78c3 100644 --- a/spec/cloud_storage/objects/gcs_spec.rb +++ b/spec/cloud_storage/objects/gcs_spec.rb @@ -22,7 +22,8 @@ context 'when with some internal options' do subject(:url) { obj.signed_url(expires_in: 30, issuer: 'max@tretyakov-ma.ru', signing_key: key, version: :v2) } - it { is_expected.to match(%r{\Ahttps://storage.googleapis.com/#{ENV.fetch('GCS_BUCKET')}/test_1.txt}) } + it { is_expected.to match(%r{\A#{ENV.fetch('GCS_ENDPOINT')}#{ENV.fetch('GCS_BUCKET')}/test_1.txt}) } + it { is_expected.to match(/GoogleAccessId=/) } end end