From 4aca6d5f8b03518c1aa9191b8fc71b083dced61d Mon Sep 17 00:00:00 2001 From: Yorgos Saslis Date: Sun, 31 Jan 2016 17:46:34 +0200 Subject: [PATCH 1/7] auto fixes with `rubocop -a` --- attributes/default.rb | 1 - metadata.rb | 10 +++++----- recipes/default.rb | 26 +++++++++++++------------- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/attributes/default.rb b/attributes/default.rb index aaa5823..6b72a61 100644 --- a/attributes/default.rb +++ b/attributes/default.rb @@ -6,7 +6,6 @@ # https://github.com/fred/chef-s3cmd # - # Url to download the tarball from latest master branch from github. default['s3cmd']['url'] = 'https://github.com/s3tools/s3cmd/archive/master.tar.gz' default['s3cmd']['gpg_passphrase'] = 'abcdefgabcdefgabcdefgabcdefg' diff --git a/metadata.rb b/metadata.rb index 32de829..6c106df 100644 --- a/metadata.rb +++ b/metadata.rb @@ -1,7 +1,7 @@ -name 's3cmd' -maintainer 'Frederico Araujo' +name 's3cmd' +maintainer 'Frederico Araujo' maintainer_email 'fred.the.master@gmail.com' -license 'All rights reserved' -description 'Installs latest s3cmd from master branch at github (alpha)' +license 'All rights reserved' +description 'Installs latest s3cmd from master branch at github (alpha)' long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) -version '0.2.1' +version '0.2.1' diff --git a/recipes/default.rb b/recipes/default.rb index f0b35b6..0f2bba0 100644 --- a/recipes/default.rb +++ b/recipes/default.rb @@ -5,13 +5,13 @@ # https://github.com/fred/chef-s3cmd # -package "python" -package "python-setuptools" -package "python-distutils-extra" -package "python-dateutil" -package "python-requests" +package 'python' +package 'python-setuptools' +package 'python-distutils-extra' +package 'python-dateutil' +package 'python-requests' -package "s3cmd" +package 's3cmd' if node['s3cmd']['config_dir'] home_folder = node['s3cmd']['config_dir'] @@ -20,14 +20,14 @@ end template "#{home_folder}/.s3cfg" do - source "s3cfg.erb" + source 's3cfg.erb' variables( - :access_key => node['s3cmd']['access_key'], - :secret_key => node['s3cmd']['secret_key'], - :gpg_passphrase => node['s3cmd']['gpg_passphrase'], - :bucket_location => node['s3cmd']['bucket_location'], - :https => node['s3cmd']['https'], - :encrypt => node['s3cmd']['encrypt'] + access_key: node['s3cmd']['access_key'], + secret_key: node['s3cmd']['secret_key'], + gpg_passphrase: node['s3cmd']['gpg_passphrase'], + bucket_location: node['s3cmd']['bucket_location'], + https: node['s3cmd']['https'], + encrypt: node['s3cmd']['encrypt'] ) owner node['s3cmd']['user'] group node['s3cmd']['user'] From 6ef7bf98727b0ab483c4643e5bd20be9f8269ddc Mon Sep 17 00:00:00 2001 From: Yorgos Saslis Date: Sun, 31 Jan 2016 19:33:08 +0200 Subject: [PATCH 2/7] Added useful utility files (.gitignore, chefignore, Gemfile) Used default values from `berks init .` --- .gitignore | 20 ++++++++++++++++++++ Gemfile | 18 ++++++++++++++++++ chefignore | 1 + 3 files changed, 39 insertions(+) create mode 100644 .gitignore create mode 100644 Gemfile create mode 100644 chefignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cbccf1e --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +*~ +*# +.#* +\#*# +.*.sw[a-z] +*.un~ +pkg/ + +# Berkshelf +.vagrant +/cookbooks +Berksfile.lock + +# Bundler +Gemfile.lock +bin/* +.bundle/* + +.kitchen/ +.kitchen.local.yml diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..dc26e3c --- /dev/null +++ b/Gemfile @@ -0,0 +1,18 @@ +source 'https://rubygems.org' + +gem 'berkshelf' + +# Uncomment these lines if you want to live on the Edge: +# +# group :development do +# gem "berkshelf", github: "berkshelf/berkshelf" +# gem "vagrant", github: "mitchellh/vagrant", tag: "v1.6.3" +# end +# +# group :plugins do +# gem "vagrant-berkshelf", github: "berkshelf/vagrant-berkshelf" +# gem "vagrant-omnibus", github: "schisamo/vagrant-omnibus" +# end + +gem 'test-kitchen' +gem 'kitchen-vagrant' diff --git a/chefignore b/chefignore new file mode 100644 index 0000000..7be3c6d --- /dev/null +++ b/chefignore @@ -0,0 +1 @@ +.kitchen From 4c8fc17fc2de60058421f1a9aa79684ca41016d2 Mon Sep 17 00:00:00 2001 From: Yorgos Saslis Date: Sun, 31 Jan 2016 19:35:34 +0200 Subject: [PATCH 3/7] Added dependency on chef-vault cookbook This will allow us to use chef-vault, to securely retrieve values for aws access / secret keys, etc. --- Berksfile | 5 +++++ metadata.json | 1 + metadata.rb | 2 ++ 3 files changed, 8 insertions(+) create mode 100644 Berksfile diff --git a/Berksfile b/Berksfile new file mode 100644 index 0000000..cd5f8bf --- /dev/null +++ b/Berksfile @@ -0,0 +1,5 @@ +source "https://supermarket.chef.io" + +metadata + +cookbook 'chef-vault', '~> 1.3.2' diff --git a/metadata.json b/metadata.json index bd31ead..ac849a7 100644 --- a/metadata.json +++ b/metadata.json @@ -8,6 +8,7 @@ "platforms": { }, "dependencies": { + "chef-vault": "~> 1.3.2" }, "recommendations": { }, diff --git a/metadata.rb b/metadata.rb index 6c106df..838de6a 100644 --- a/metadata.rb +++ b/metadata.rb @@ -5,3 +5,5 @@ description 'Installs latest s3cmd from master branch at github (alpha)' long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) version '0.2.1' + +depends 'chef-vault', '~> 1.3.2' \ No newline at end of file From 442bacf74e121c70d2944234bc8724ce632e2717 Mon Sep 17 00:00:00 2001 From: Yorgos Saslis Date: Sun, 31 Jan 2016 19:42:52 +0200 Subject: [PATCH 4/7] Changed the way we try to load the values for the config file attributes Recipe now first attempts to load values from a Chef Vault. If it fails, it reverts to the old way where we use node attributes. --- recipes/default.rb | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/recipes/default.rb b/recipes/default.rb index 0f2bba0..18a5731 100644 --- a/recipes/default.rb +++ b/recipes/default.rb @@ -19,16 +19,36 @@ home_folder = node['etc']['passwd'][node['s3cmd']['user']]['dir'] end +include_recipe 'chef-vault' + +begin + # First try to load s3 config through Chef Vault, as the more secure way. + s3cfg_variables = { + access_key: chef_vault_item_for_environment('aws_creds', 's3_user')['access_key'], + secret_key: chef_vault_item_for_environment('aws_creds', 's3_user')['secret_key'], + gpg_passphrase: chef_vault_item_for_environment('aws_creds', 's3_user')['gpg_passphrase'], + bucket_location: chef_vault_item_for_environment('aws_creds', 's3_user')['bucket_location'], + https: chef_vault_item_for_environment('aws_creds', 's3_user')['https'], + encrypt: chef_vault_item_for_environment('aws_creds', 's3_user')['encrypt'] + } +rescue ChefVault::Exceptions::KeysNotFound, Net::HTTPServerException => e + if e || e.response_code == '404' + # if we couldn't find the data bag, revert to plain node attributes + s3cfg_variables = { + access_key: node['s3cmd']['access_key'], + secret_key: node['s3cmd']['secret_key'], + gpg_passphrase: node['s3cmd']['gpg_passphrase'], + bucket_location: node['s3cmd']['bucket_location'], + https: node['s3cmd']['https'], + encrypt: node['s3cmd']['encrypt'] + } + end +end + + template "#{home_folder}/.s3cfg" do source 's3cfg.erb' - variables( - access_key: node['s3cmd']['access_key'], - secret_key: node['s3cmd']['secret_key'], - gpg_passphrase: node['s3cmd']['gpg_passphrase'], - bucket_location: node['s3cmd']['bucket_location'], - https: node['s3cmd']['https'], - encrypt: node['s3cmd']['encrypt'] - ) + variables s3cfg_variables owner node['s3cmd']['user'] group node['s3cmd']['user'] mode 0600 From d274dfc820672b94f193133b29854aaed5a39509 Mon Sep 17 00:00:00 2001 From: Yorgos Saslis Date: Sun, 31 Jan 2016 19:45:19 +0200 Subject: [PATCH 5/7] Added a comment to explain user must exist --- attributes/default.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/attributes/default.rb b/attributes/default.rb index 6b72a61..7a63abd 100644 --- a/attributes/default.rb +++ b/attributes/default.rb @@ -14,4 +14,4 @@ default['s3cmd']['bucket_location'] = 'US' default['s3cmd']['encrypt'] = false default['s3cmd']['https'] = false -default['s3cmd']['user'] = 'ubuntu' +default['s3cmd']['user'] = 'ubuntu' # User *must* exist, otherwise don't expect this to work ; ) From 04e24947af2826e5c1188d098cc7878f5433ba2d Mon Sep 17 00:00:00 2001 From: Yorgos Saslis Date: Sun, 31 Jan 2016 19:45:39 +0200 Subject: [PATCH 6/7] Added test kitchen config file --- .kitchen.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .kitchen.yml diff --git a/.kitchen.yml b/.kitchen.yml new file mode 100644 index 0000000..b00dcd6 --- /dev/null +++ b/.kitchen.yml @@ -0,0 +1,19 @@ +--- +driver: + name: vagrant + +provisioner: + name: chef_zero + +platforms: + - name: ubuntu-14.04 +# - name: centos-7.1 + +suites: + - name: default + run_list: + - recipe[s3cmd::default] + attributes: + s3cmd: + user: vagrant + config_dir: /home/vagrant \ No newline at end of file From 06cf599bfa41fba6d5c94cc3b0cc472f0640cd90 Mon Sep 17 00:00:00 2001 From: Yorgos Saslis Date: Mon, 1 Feb 2016 00:32:40 +0200 Subject: [PATCH 7/7] Added new cookbook attribute, so that data bag name and item name can be changed Includes sample data bag (to be used by test kitchen). Note that the new attribute is left uninitialized by default (commented out in attributes), to maintain backwards compatibility --- .kitchen.yml | 7 +++++- attributes/default.rb | 4 ++++ recipes/default.rb | 22 ++++++++++++------- .../fixtures/data_bags/aws_creds/s3_user.json | 12 ++++++++++ 4 files changed, 36 insertions(+), 9 deletions(-) create mode 100644 test/fixtures/data_bags/aws_creds/s3_user.json diff --git a/.kitchen.yml b/.kitchen.yml index b00dcd6..90d25af 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -4,6 +4,7 @@ driver: provisioner: name: chef_zero + data_bags_path: test/fixtures/data_bags platforms: - name: ubuntu-14.04 @@ -16,4 +17,8 @@ suites: attributes: s3cmd: user: vagrant - config_dir: /home/vagrant \ No newline at end of file + config_dir: /home/vagrant + data_bag: { + name: 'aws_creds', + item: 's3_user' + } \ No newline at end of file diff --git a/attributes/default.rb b/attributes/default.rb index 7a63abd..6092e33 100644 --- a/attributes/default.rb +++ b/attributes/default.rb @@ -15,3 +15,7 @@ default['s3cmd']['encrypt'] = false default['s3cmd']['https'] = false default['s3cmd']['user'] = 'ubuntu' # User *must* exist, otherwise don't expect this to work ; ) +# default['s3cmd']['data_bag'] = { +# name: 'aws_creds', +# item: 's3_user' +# } \ No newline at end of file diff --git a/recipes/default.rb b/recipes/default.rb index 18a5731..5007033 100644 --- a/recipes/default.rb +++ b/recipes/default.rb @@ -23,14 +23,20 @@ begin # First try to load s3 config through Chef Vault, as the more secure way. - s3cfg_variables = { - access_key: chef_vault_item_for_environment('aws_creds', 's3_user')['access_key'], - secret_key: chef_vault_item_for_environment('aws_creds', 's3_user')['secret_key'], - gpg_passphrase: chef_vault_item_for_environment('aws_creds', 's3_user')['gpg_passphrase'], - bucket_location: chef_vault_item_for_environment('aws_creds', 's3_user')['bucket_location'], - https: chef_vault_item_for_environment('aws_creds', 's3_user')['https'], - encrypt: chef_vault_item_for_environment('aws_creds', 's3_user')['encrypt'] - } + if node['s3cmd']['data_bag'] + data_bag_name = node['s3cmd']['data_bag']['name'] + data_bag_item_name = node['s3cmd']['data_bag']['item'] + s3cfg_variables = { + access_key: chef_vault_item_for_environment(data_bag_name, data_bag_item_name)['access_key'], + secret_key: chef_vault_item_for_environment(data_bag_name, data_bag_item_name)['secret_key'], + gpg_passphrase: chef_vault_item_for_environment(data_bag_name, data_bag_item_name)['gpg_passphrase'], + bucket_location: chef_vault_item_for_environment(data_bag_name, data_bag_item_name)['bucket_location'], + https: chef_vault_item_for_environment(data_bag_name, data_bag_item_name)['https'], + encrypt: chef_vault_item_for_environment(data_bag_name, data_bag_item_name)['encrypt'] + } + else + raise ChefVault::Exceptions::KeysNotFound.new('data bag not declared, reverting to attributes') # caught immediately below + end rescue ChefVault::Exceptions::KeysNotFound, Net::HTTPServerException => e if e || e.response_code == '404' # if we couldn't find the data bag, revert to plain node attributes diff --git a/test/fixtures/data_bags/aws_creds/s3_user.json b/test/fixtures/data_bags/aws_creds/s3_user.json new file mode 100644 index 0000000..f695b13 --- /dev/null +++ b/test/fixtures/data_bags/aws_creds/s3_user.json @@ -0,0 +1,12 @@ +{ + "id": "s3_user", + "_default": { + "username": "your_user", + "access_key": "insert_your_access_key_here", + "secret_key": "insert_your_secret_key_here", + "https": true, + "encrypt": false, + "gpg_passphrase": "", + "bucket_location": "EU" + } +} \ No newline at end of file