From e24b263929536e963beecb9fb1db555e67965338 Mon Sep 17 00:00:00 2001 From: Emilio Date: Fri, 27 Aug 2021 10:51:45 +0100 Subject: [PATCH 1/4] Writes items in batches of 25 items due to dynamodb batch_write_item limitation --- lib/quarantine/databases/dynamo_db.rb | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/lib/quarantine/databases/dynamo_db.rb b/lib/quarantine/databases/dynamo_db.rb index 0a81da2..107491b 100644 --- a/lib/quarantine/databases/dynamo_db.rb +++ b/lib/quarantine/databases/dynamo_db.rb @@ -12,6 +12,8 @@ module Databases class DynamoDB < Base extend T::Sig + BATCH_WRITE_MAXIMUM_ITEMS = T.let(25, Integer) + Attribute = T.type_alias { { attribute_name: String, attribute_type: String, key_type: String } } sig { returns(Aws::DynamoDB::Client) } @@ -42,17 +44,19 @@ def fetch_items(table_name) ).void end def write_items(table_name, items) - @dynamodb.batch_write_item( - request_items: { - table_name => items.map do |item| - { - put_request: { - item: item + items.each_slice(BATCH_WRITE_MAXIMUM_ITEMS) do |batch_items| + @dynamodb.batch_write_item( + request_items: { + table_name => batch_items.map do |item| + { + put_request: { + item: item + } } - } - end - } - ) + end + } + ) + end rescue Aws::DynamoDB::Errors::ServiceError raise Quarantine::DatabaseError end From 9d66a8cbb6270bf3236ea1556dba343f63cee499 Mon Sep 17 00:00:00 2001 From: Emilio Date: Fri, 27 Aug 2021 11:19:31 +0100 Subject: [PATCH 2/4] Add sorbet-runtime to cli to fix uninitialized constant T --- bin/quarantine_dynamodb | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/quarantine_dynamodb b/bin/quarantine_dynamodb index 679fe3c..ca98c96 100755 --- a/bin/quarantine_dynamodb +++ b/bin/quarantine_dynamodb @@ -1,6 +1,7 @@ #!/usr/bin/env ruby require 'optparse' +require 'sorbet-runtime' require_relative '../lib/quarantine/cli' From 54f8c9273cbca1ac3497326fffc1a7be0fbf1352 Mon Sep 17 00:00:00 2001 From: Emilio Date: Fri, 27 Aug 2021 14:37:57 +0100 Subject: [PATCH 3/4] Add endpoint option to cli --- lib/quarantine/cli.rb | 7 ++++++- spec/quarantine/cli_spec.rb | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/lib/quarantine/cli.rb b/lib/quarantine/cli.rb index ef0984e..3609ffe 100644 --- a/lib/quarantine/cli.rb +++ b/lib/quarantine/cli.rb @@ -30,6 +30,10 @@ def parse @options[:region] = region end + parser.on('-e', '--endpoint [ENDPOINT]', String, 'Specify the aws endpoint for DynamoDB') do |endpoint| + @options[:endpoint] = endpoint + end + parser.on( '-qTABLE', '--quarantine_table=TABLE', @@ -55,7 +59,8 @@ def parse # TODO: eventually move to a separate file & create_table by db type when my db adapters sig { void } def create_tables - dynamodb = Quarantine::Databases::DynamoDB.new(region: @options[:region]) + dynamodb_params = @options.slice(:region, :endpoint).compact + dynamodb = Quarantine::Databases::DynamoDB.new(dynamodb_params) attributes = [ { attribute_name: 'id', attribute_type: 'S', key_type: 'HASH' } diff --git a/spec/quarantine/cli_spec.rb b/spec/quarantine/cli_spec.rb index 8fbaf95..b9671bc 100644 --- a/spec/quarantine/cli_spec.rb +++ b/spec/quarantine/cli_spec.rb @@ -37,6 +37,18 @@ expect(cli.options[:test_statuses_table_name]).to eq('foo') end + it 'define the endpoint' do + endpoint = 'http://localhost:4569' + cli = Quarantine::CLI.new + + ARGV << '-r' << 'us-west-1' + ARGV << '-e' << endpoint + + cli.parse + + expect(cli.options[:endpoint]).to eq(endpoint) + end + context '#create_tables' do let(:dynamodb) { Quarantine::Databases::DynamoDB.new(region: 'us-west-1') } let(:cli) { Quarantine::CLI.new } @@ -63,6 +75,26 @@ cli.create_tables end + it 'called with endpoint' do + endpoint = 'localhost:4569' + dynamodb_double = double('dynamodb instance') + + allow(dynamodb_double).to receive(:create_table).with(any_args) + allow(Quarantine::Databases::DynamoDB) + .to receive(:new).and_return(dynamodb_double) + + ARGV << '-r' << 'us-west-1' + ARGV << '-e' << endpoint + + cli.parse + cli.create_tables + + expect(dynamodb_double).to have_received(:create_table) + expect(Quarantine::Databases::DynamoDB) + .to have_received(:new) + .with(hash_including(endpoint: endpoint)) + end + it 'raises exception if Quarantine::DatabaseError exception occurs' do allow(Quarantine::Databases::DynamoDB).to receive(:new).and_return(dynamodb) allow(dynamodb).to receive(:create_table).and_raise(Quarantine::DatabaseError.new('db error')) From 1112263cd9422967ccfe571bc26ed76c3b773e83 Mon Sep 17 00:00:00 2001 From: Emilio Date: Fri, 27 Aug 2021 14:42:49 +0100 Subject: [PATCH 4/4] Bump version to 2.3.0 --- Gemfile.lock | 2 +- lib/quarantine/version.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 0854a3c..3e1da69 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - quarantine (2.2.2) + quarantine (2.3.0) rspec (~> 3.0) rspec-retry (~> 0.6) sorbet-runtime (= 0.5.6338) diff --git a/lib/quarantine/version.rb b/lib/quarantine/version.rb index f443bed..31cb098 100644 --- a/lib/quarantine/version.rb +++ b/lib/quarantine/version.rb @@ -1,3 +1,3 @@ class Quarantine - VERSION = '2.2.2'.freeze + VERSION = '2.3.0'.freeze end