From 8eab2eb195294c6b6905d2b6ab012fc77310fefb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l=20Andre=20Sundt?= Date: Tue, 4 Jul 2023 11:09:17 +0200 Subject: [PATCH 01/11] Rails 4 support --- lib/backy/railtie.rb | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/lib/backy/railtie.rb b/lib/backy/railtie.rb index 3940823..77d9ac1 100644 --- a/lib/backy/railtie.rb +++ b/lib/backy/railtie.rb @@ -1,14 +1,24 @@ module Backy class Railtie < Rails::Railtie initializer "railtie.configure_rails_initialization" do - Backy.configure do |config| - config.pg_host = ActiveRecord::Base.connection_db_config.configuration_hash[:host] - config.pg_port = ActiveRecord::Base.connection_db_config.configuration_hash[:port] - config.pg_database = ActiveRecord::Base.connection_db_config.configuration_hash[:database] - config.pg_username = ActiveRecord::Base.connection_db_config.configuration_hash[:username] - config.pg_password = ActiveRecord::Base.connection_db_config.configuration_hash[:password] - config.app_name = Rails.application.class.name.split("::").first.underscore - config.environment = Rails.env + begin + Backy.configure do |config| + config.pg_host = ActiveRecord::Base.connection_db_config.configuration_hash[:host] + config.pg_port = ActiveRecord::Base.connection_db_config.configuration_hash[:port] + config.pg_database = ActiveRecord::Base.connection_db_config.configuration_hash[:database] + config.pg_username = ActiveRecord::Base.connection_db_config.configuration_hash[:username] + config.pg_password = ActiveRecord::Base.connection_db_config.configuration_hash[:password] + config.app_name = Rails.application.class.name.split("::").first.underscore + config.environment = Rails.env + end + rescue NoMethodError # Older rails version does not have the connection_db_config method + config.pg_host = ActiveRecord::Base.connection_config[:host] + config.pg_port = ActiveRecord::Base.connection_config[:port] + config.pg_database = ActiveRecord::Base.connection_config[:database] + config.pg_username = ActiveRecord::Base.connection_config[:username] + config.pg_password = ActiveRecord::Base.connection_config[:password] + config.app_name = Rails.application.class.name.split("::").first.underscore + config.environment = Rails.env end end From 99d4ca725c43e0a85db6cb0e249938ff66ad2e81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l=20Andre=20Sundt?= Date: Tue, 4 Jul 2023 11:48:31 +0200 Subject: [PATCH 02/11] Make s3 folder configurable --- lib/backy/configuration.rb | 5 +++++ lib/backy/s3.rb | 1 + lib/backy/s3_list.rb | 12 +----------- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/lib/backy/configuration.rb b/lib/backy/configuration.rb index 8aa7b7e..ec8ccf9 100644 --- a/lib/backy/configuration.rb +++ b/lib/backy/configuration.rb @@ -10,6 +10,7 @@ class Configuration :s3_access_key, :s3_secret, :s3_bucket, + :s3_folder, :app_name, :environment, :log_file @@ -51,6 +52,10 @@ def s3_bucket @s3_bucket ||= ENV["S3_BUCKET"] end + def s3_folder + @s3_folder ||= "db/dump/" + end + def app_name @app_name ||= "backy" end diff --git a/lib/backy/s3.rb b/lib/backy/s3.rb index cbfd28f..ee98b6c 100644 --- a/lib/backy/s3.rb +++ b/lib/backy/s3.rb @@ -11,6 +11,7 @@ module S3 def_delegator "Backy.configuration", :s3_secret, :secret def_delegator "Backy.configuration", :s3_bucket, :bucket def_delegator "Backy.configuration", :s3_access_key, :access_key + def_delegator "Backy.configuration", :s3_folder, :folder def s3 @s3 ||= Aws::S3::Client.new(region: region, credentials: s3_credentials) diff --git a/lib/backy/s3_list.rb b/lib/backy/s3_list.rb index 90e2ee1..6659e7b 100644 --- a/lib/backy/s3_list.rb +++ b/lib/backy/s3_list.rb @@ -2,16 +2,10 @@ module Backy class S3List include S3 - DEFAULT_PREFIX = "db/dump/" - - def initialize(prefix: nil) - @prefix = prefix || DEFAULT_PREFIX - end - def call return [] unless s3_configured? - response = s3.list_objects(prefix: prefix, bucket: bucket) + response = s3.list_objects(prefix: folder, bucket: bucket) result = response.contents.map(&:key) @@ -23,9 +17,5 @@ def call result.sort end - - private - - attr_reader :prefix end end From 7fc29eeb14ec4831f2f2e91ab5a4037f73d775dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l=20Andre=20Sundt?= Date: Thu, 6 Jul 2023 11:55:26 +0200 Subject: [PATCH 03/11] Only use certain options if pg_dump supports them --- lib/backy/db.rb | 4 ++++ lib/backy/pg_dump.rb | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/backy/db.rb b/lib/backy/db.rb index f30732d..b66d355 100644 --- a/lib/backy/db.rb +++ b/lib/backy/db.rb @@ -25,5 +25,9 @@ def pg_credentials args_string end + + def use_pg_dump_option_if_supported(option) + system("pg_dump --help | grep -q -- '#{option}'") ? option : "" + end end end diff --git a/lib/backy/pg_dump.rb b/lib/backy/pg_dump.rb index 2cf5b02..d810b1c 100644 --- a/lib/backy/pg_dump.rb +++ b/lib/backy/pg_dump.rb @@ -4,7 +4,7 @@ class PgDump include AppConfig DUMP_DIR = "db/dump" - DUMP_CMD_OPTS = "--no-acl --no-owner --no-subscriptions --no-publications --exclude-table=awsdms_ddl_audit" + DUMP_CMD_OPTS = "--no-acl --no-owner --exclude-table=awsdms_ddl_audit #{use_pg_dump_option_if_supported('--no-subscriptions')} #{use_pg_dump_option_if_supported('--no-publications')}" def call FileUtils.mkdir_p(DUMP_DIR) From 7ed9e2328974b1ba05b85353a41ed6598f41dea9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l=20Andre=20Sundt?= Date: Thu, 6 Jul 2023 15:11:15 +0200 Subject: [PATCH 04/11] Create the folder in /tmp before creating tempfile --- lib/backy/s3_load.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/backy/s3_load.rb b/lib/backy/s3_load.rb index 422ae51..21b39fb 100644 --- a/lib/backy/s3_load.rb +++ b/lib/backy/s3_load.rb @@ -12,6 +12,7 @@ def call print "Loading #{key} from S3 ... " + FileUtils.mkdir_p("/tmp/#{File.dirname(file_name)}") Tempfile.create(file_name) do |tempfile| response_target = tempfile.path From 57a77b7432688092a3bfb9aeae5b1a9fc8e261fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l=20Andre=20Sundt?= Date: Thu, 6 Jul 2023 15:21:00 +0200 Subject: [PATCH 05/11] Add class method --- lib/backy/db.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/backy/db.rb b/lib/backy/db.rb index b66d355..e87428c 100644 --- a/lib/backy/db.rb +++ b/lib/backy/db.rb @@ -4,6 +4,12 @@ module Backy module Db extend Forwardable + def self.included(base) + base.define_singleton_method :use_pg_dump_option_if_supported do |option| + system("pg_dump --help | grep -q -- '#{option}'") ? option : "" + end + end + private def_delegator "Backy.configuration", :pg_host, :host @@ -25,9 +31,5 @@ def pg_credentials args_string end - - def use_pg_dump_option_if_supported(option) - system("pg_dump --help | grep -q -- '#{option}'") ? option : "" - end end end From a3bc55ce31e248b02757248055f38fee5dd4ee77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l=20Andre=20Sundt?= Date: Thu, 6 Jul 2023 15:42:18 +0200 Subject: [PATCH 06/11] Tempfile w block already does this. Remove fluff --- lib/backy/s3_load.rb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/backy/s3_load.rb b/lib/backy/s3_load.rb index 21b39fb..c86dda4 100644 --- a/lib/backy/s3_load.rb +++ b/lib/backy/s3_load.rb @@ -22,11 +22,6 @@ def call FileUtils.mv(response_target, file_name) rescue Aws::S3::Errors::NoSuchKey puts "error. No such key #{key}" - ensure - if File.exist?(tempfile.path) - tempfile.close - File.delete(tempfile.path) - end end end From 2d239232602c045ab1f6004878f1f0c36e075f26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l=20Andre=20Sundt?= Date: Thu, 6 Jul 2023 15:50:31 +0200 Subject: [PATCH 07/11] Detect empty-ish files and return an error msg --- lib/backy/s3_save.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/backy/s3_save.rb b/lib/backy/s3_save.rb index 0474ebb..f94900c 100644 --- a/lib/backy/s3_save.rb +++ b/lib/backy/s3_save.rb @@ -19,6 +19,12 @@ def call return end + unless (File.file?(filename) && File.size(filename) > 25) + puts "error. #{file_name} seems to be more or less empty" + + return + end + File.open(file_name, "rb") do |body| s3.put_object(key: key, body: body, bucket: bucket, expires: expires) end From f969d1e5812ca540be91cf615fd7ad2ae402c80b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l=20Andre=20Sundt?= Date: Thu, 6 Jul 2023 16:04:11 +0200 Subject: [PATCH 08/11] Use cp to avoid error when Tempfile tries to delete file --- lib/backy/s3_load.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/backy/s3_load.rb b/lib/backy/s3_load.rb index c86dda4..6f288fe 100644 --- a/lib/backy/s3_load.rb +++ b/lib/backy/s3_load.rb @@ -19,7 +19,7 @@ def call begin s3.get_object(response_target: response_target, key: key, bucket: bucket) FileUtils.mkdir_p(File.dirname(file_name)) - FileUtils.mv(response_target, file_name) + FileUtils.cp(response_target, file_name) rescue Aws::S3::Errors::NoSuchKey puts "error. No such key #{key}" end From f87e7b3307a4f0635efe390646fbed1103f061e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l=20Andre=20Sundt?= Date: Fri, 29 Sep 2023 12:39:12 +0200 Subject: [PATCH 09/11] Call correct method name --- lib/backy/s3_save.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/backy/s3_save.rb b/lib/backy/s3_save.rb index f94900c..193129b 100644 --- a/lib/backy/s3_save.rb +++ b/lib/backy/s3_save.rb @@ -19,7 +19,7 @@ def call return end - unless (File.file?(filename) && File.size(filename) > 25) + unless (File.file?(file_name) && File.size(filename) > 25) puts "error. #{file_name} seems to be more or less empty" return From 2ce25100d0054bb1f64c14d0c055173ad6f42355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l=20Andre=20Sundt?= Date: Fri, 29 Sep 2023 12:39:28 +0200 Subject: [PATCH 10/11] Some configs use user instead of username --- lib/backy/railtie.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/backy/railtie.rb b/lib/backy/railtie.rb index 77d9ac1..0ec5103 100644 --- a/lib/backy/railtie.rb +++ b/lib/backy/railtie.rb @@ -15,7 +15,7 @@ class Railtie < Rails::Railtie config.pg_host = ActiveRecord::Base.connection_config[:host] config.pg_port = ActiveRecord::Base.connection_config[:port] config.pg_database = ActiveRecord::Base.connection_config[:database] - config.pg_username = ActiveRecord::Base.connection_config[:username] + config.pg_username = ActiveRecord::Base.connection_config[:username] || ActiveRecord::Base.connection_config[:user] config.pg_password = ActiveRecord::Base.connection_config[:password] config.app_name = Rails.application.class.name.split("::").first.underscore config.environment = Rails.env From 7b292e981ec41b1bbefb9d79bf41f2be7fdeb930 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l=20Andre=20Sundt?= Date: Fri, 29 Sep 2023 14:08:56 +0200 Subject: [PATCH 11/11] Fix bad method name (called undefined method) --- lib/backy/s3_save.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/backy/s3_save.rb b/lib/backy/s3_save.rb index 193129b..d1b5065 100644 --- a/lib/backy/s3_save.rb +++ b/lib/backy/s3_save.rb @@ -19,7 +19,7 @@ def call return end - unless (File.file?(file_name) && File.size(filename) > 25) + unless (File.file?(file_name) && File.size(file_name) > 25) puts "error. #{file_name} seems to be more or less empty" return