From c9ea73c28511298bf91b73fce261e6a728b935a2 Mon Sep 17 00:00:00 2001 From: Mike Virata-Stone Date: Mon, 13 Mar 2017 01:52:29 -0700 Subject: [PATCH 1/9] Some initial chef stuff, but nothing completed --- chef/deploy.rb | 150 +++++++++++++++++++++++++++++++++++++++++ chef/remote/install.sh | 22 ++++++ chef/remote/solo.rb | 3 + 3 files changed, 175 insertions(+) create mode 100755 chef/deploy.rb create mode 100755 chef/remote/install.sh create mode 100644 chef/remote/solo.rb diff --git a/chef/deploy.rb b/chef/deploy.rb new file mode 100755 index 00000000..888401f0 --- /dev/null +++ b/chef/deploy.rb @@ -0,0 +1,150 @@ +#!/usr/bin/env ruby +require "json" +require "optparse" + +# This script is for deploying to the StockAid instance. Unless configured via +# options, it expects to connect via the "chef" user to the "stockaid" host (you +# can set up this host in your ~/.ssh/config file, or provide the real host via +# a command line option). + +REMOTE_DIR = File.expand_path("../remote", __FILE__) +CHEF_VERSION = "12.19.36" +SSH_DEFAULTS = { batch: true } + +DEFAULT_OPTIONS = { + user: "chef", + host: "stockaid", + port: 22, + identity: File.expand_path("~/.ssh/stockaid_rsa"), + files_dir: File.expand_path("../files", __FILE__), + repos_dir: File.expand_path("../repos", __FILE__), + json: File.expand_path("../stockaid.json", __FILE__), + clear_host: false, + ssh: false +} + +OPTIONS = Hash.new { |_hash, key| DEFAULT_OPTIONS[key] } + +def system_exec(cmd, options = {}) + puts cmd + + if options[:exec] + exec cmd + else + system cmd + end +end + +def ssh_options(options = {}) + options = SSH_DEFAULTS.merge(options) + result = [] + result << "-o 'BatchMode yes'" if options[:batch] + result.join(" ") +end + +def ssh(command, options = {}) + if command.kind_of?(Hash) + options = options.merge(command) + options[:batch] = false if options[:exec] + command = nil + end + + user = OPTIONS[:user] + host = OPTIONS[:host] + port = OPTIONS[:port] + identity_file = OPTIONS[:identity] + + if options[:exec] + system_exec "ssh '#{user}@#{host}' -p #{port} -i #{identity_file} #{ssh_options(options)}", exec: true + else + force_tty = "-t -t" unless options[:batch] + system_exec "ssh '#{user}@#{host}' -p #{port} -i #{identity_file} #{force_tty} #{ssh_options(options)} '#{command}'" + end +end + +def scp(from, to, options = {}) + user = OPTIONS[:user] + host = OPTIONS[:host] + port = OPTIONS[:port] + identity_file = OPTIONS[:identity] + server = "#{user}@#{host}" + from.sub! /\Aserver:/, "#{server}:" + to.sub! /\Aserver:/, "#{server}:" + recursive = "-r" if options[:recursive] + system_exec "scp #{recursive} -P #{port} -i #{identity_file} #{ssh_options(options)} '#{from}' '#{to}'" +end + +def rsync(from, to, options = {}) + user = OPTIONS[:user] + host = OPTIONS[:host] + port = OPTIONS[:port] + identity_file = OPTIONS[:identity] + server = "#{user}@#{host}" + from.sub! /\Aserver:/, "#{server}:" + to.sub! /\Aserver:/, "#{server}:" + system_exec %{rsync -e "ssh -p #{port} -i #{identity_file} #{ssh_options(options)}" -av '#{from}' '#{to}'} +end + +OptionParser.new do |opts| + opts.banner = "Usage: ./deploy.rb [options]" + + opts.on "-s", "--ssh", "Just ssh to the server, don't actually deploy" do |ssh| + OPTIONS[:ssh] = true + end + + opts.on "-c", "--clear-host", "Clear the host from known hosts" do |clear| + OPTIONS[:clear_host] = clear + end + + opts.on "-u", "--user USER", "User to ssh with" do |user| + OPTIONS[:user] = user + end + + opts.on "-h", "--host HOST", "Host to ssh with" do |host| + OPTIONS[:host] = host + end + + opts.on "-p", "--port PORT", "Port to ssh with" do |port| + OPTIONS[:port] = port.to_i + end + + opts.on "-i", "--identity FILE", "Identity file to ssh with" do |identity| + OPTIONS[:identity] = File.expand_path(identity) + end + + opts.on "-j", "--json FILE", "Chef json config file" do |config| + OPTIONS[:json] = File.expand_path(config) + end +end.parse! + +JSON.parse(File.read(OPTIONS[:json])).fetch("meta", {}).each do |key, value| + unless OPTIONS.include?(key.to_sym) + puts "Using JSON default #{key}: #{value.inspect}" + value = File.expand_path(value) if %w(identity files_dir repos_dir).include?(key) + OPTIONS[key.to_sym] = value + end +end + +ssh exec: true if OPTIONS[:ssh] + +system_exec "ssh-keygen -R '#{OPTIONS[:host]}'" if OPTIONS[:clear_host] + +unless ssh "echo ssh key is working" + pub_file = "#{OPTIONS[:identity]}.pub" + public_key = File.read(pub_file) + + unless ssh %{mkdir -p ~/.ssh && touch ~/.ssh/authorized_keys && echo "#{public_key}" >> ~/.ssh/authorized_keys}, batch: false + abort "Failed to save authorized key!" + end +end + +rsync "#{REMOTE_DIR}/", "server:~/next-stockaid-chef" +rsync OPTIONS[:files_dir], "server:~/chef_data/" +rsync OPTIONS[:repos_dir], "server:~/chef_data/" + +if OPTIONS[:json] + puts "Using json file: #{OPTIONS[:json]}" + scp OPTIONS[:json], "server:~/next-stockaid-chef/solo.json" +end + +ssh "sudo ~/next-stockaid-chef/install.sh '#{OPTIONS[:user]}' '#{CHEF_VERSION}'" diff --git a/chef/remote/install.sh b/chef/remote/install.sh new file mode 100755 index 00000000..f0753bab --- /dev/null +++ b/chef/remote/install.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +USERNAME="$1" +CHEF_VERSION="$2" + +if ! command -v ruby >/dev/null 2>&1 +then + echo "Installing Ruby" + DEBIAN_FRONTEND=noninteractive apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y ruby ruby-dev build-essential +fi + +if ! command -v chef-solo >/dev/null 2>&1 +then + echo "Installing Chef" + gem install --no-rdoc --no-ri chef --version $CHEF_VERSION +fi + +rm -rf /home/$USERNAME/stockaid-chef +mv /home/$USERNAME/next-stockaid-chef /home/$USERNAME/stockaid-chef +cd /home/$USERNAME/stockaid-chef +chef-solo -c solo.rb -j solo.json diff --git a/chef/remote/solo.rb b/chef/remote/solo.rb new file mode 100644 index 00000000..8a546892 --- /dev/null +++ b/chef/remote/solo.rb @@ -0,0 +1,3 @@ +root = File.absolute_path File.dirname(__FILE__) +file_cache_path root +cookbook_path File.join(root, "cookbooks") From 2f1dbd44e9332c4a340fe194a8579f3faa116b6b Mon Sep 17 00:00:00 2001 From: Mike Virata-Stone Date: Sun, 14 May 2017 15:10:11 -0700 Subject: [PATCH 2/9] Get the chef recipes to work --- .../cookbooks/stockaid/attributes/default.rb | 11 ++ chef/remote/cookbooks/stockaid/metadata.rb | 6 + .../cookbooks/stockaid/recipes/database.rb | 35 ++++++ .../cookbooks/stockaid/recipes/default.rb | 7 ++ .../cookbooks/stockaid/recipes/letsencrypt.rb | 34 +++++ .../cookbooks/stockaid/recipes/nginx.rb | 119 ++++++++++++++++++ .../cookbooks/stockaid/recipes/rails.rb | 86 +++++++++++++ .../remote/cookbooks/stockaid/recipes/repo.rb | 14 +++ chef/remote/cookbooks/stockaid/recipes/rvm.rb | 53 ++++++++ .../stockaid/recipes/self_signed_ssl.rb | 12 ++ .../stockaid_letsencrypt_renew.erb | 48 +++++++ .../default/nginx/default.nginx.conf.erb | 12 ++ .../default/nginx/stockaid.nginx.conf.erb | 33 +++++ config/database.yml | 1 + 14 files changed, 471 insertions(+) create mode 100644 chef/remote/cookbooks/stockaid/attributes/default.rb create mode 100644 chef/remote/cookbooks/stockaid/metadata.rb create mode 100644 chef/remote/cookbooks/stockaid/recipes/database.rb create mode 100644 chef/remote/cookbooks/stockaid/recipes/default.rb create mode 100644 chef/remote/cookbooks/stockaid/recipes/letsencrypt.rb create mode 100644 chef/remote/cookbooks/stockaid/recipes/nginx.rb create mode 100644 chef/remote/cookbooks/stockaid/recipes/rails.rb create mode 100644 chef/remote/cookbooks/stockaid/recipes/repo.rb create mode 100644 chef/remote/cookbooks/stockaid/recipes/rvm.rb create mode 100644 chef/remote/cookbooks/stockaid/recipes/self_signed_ssl.rb create mode 100644 chef/remote/cookbooks/stockaid/templates/default/letsencrypt/stockaid_letsencrypt_renew.erb create mode 100644 chef/remote/cookbooks/stockaid/templates/default/nginx/default.nginx.conf.erb create mode 100644 chef/remote/cookbooks/stockaid/templates/default/nginx/stockaid.nginx.conf.erb diff --git a/chef/remote/cookbooks/stockaid/attributes/default.rb b/chef/remote/cookbooks/stockaid/attributes/default.rb new file mode 100644 index 00000000..805ede0b --- /dev/null +++ b/chef/remote/cookbooks/stockaid/attributes/default.rb @@ -0,0 +1,11 @@ +default[:stockaid][:github_url] = "https://github.com/on-site/StockAid.git" +default[:stockaid][:home] = "/home/stockaid" +default[:stockaid][:user] = "stockaid" +default[:stockaid][:group] = "stockaid" +default[:stockaid][:dir] = node[:stockaid][:home] +default[:stockaid][:repo_dir] = File.join(node[:stockaid][:dir], "StockAid") +default[:stockaid][:domain] = "orders.gratefulgarment.org" +default[:stockaid][:use_letsencrypt] = true + +# This must be set for letsencrypt to work +default[:stockaid][:letsencrypt_email] = nil diff --git a/chef/remote/cookbooks/stockaid/metadata.rb b/chef/remote/cookbooks/stockaid/metadata.rb new file mode 100644 index 00000000..adafdb3e --- /dev/null +++ b/chef/remote/cookbooks/stockaid/metadata.rb @@ -0,0 +1,6 @@ +name "stockaid" +maintainer "Mike Virata-Stone" +maintainer_email "mike@virata-stone.com" +license "All rights reserved" +description "Sets up and configures StockAid" +version "0.0.1" diff --git a/chef/remote/cookbooks/stockaid/recipes/database.rb b/chef/remote/cookbooks/stockaid/recipes/database.rb new file mode 100644 index 00000000..5581e779 --- /dev/null +++ b/chef/remote/cookbooks/stockaid/recipes/database.rb @@ -0,0 +1,35 @@ +%w( + postgresql + libpq-dev +).each do |pkg| + package pkg +end + +password_file = File.join(node[:stockaid][:dir], ".stockaid-db-password") + +file password_file do + content lazy { `openssl rand -base64 18` } + user node[:stockaid][:user] + group node[:stockaid][:group] + mode "0600" + action :create_if_missing +end + +execute "create-postgres-user" do + command lazy { %{psql -c "CREATE USER \\"stockaid\\" WITH LOGIN PASSWORD '#{File.read(password_file).strip}'"} } + user "postgres" + not_if %{psql postgres -tAc "SELECT 1 FROM pg_roles WHERE LOWER(rolname) = 'stockaid'" | grep 1}, user: "postgres" +end + +execute "create-postgres-database" do + command lazy { %{psql -c "CREATE DATABASE \\"stockaid_production\\" WITH OWNER \\"stockaid\\" ENCODING 'unicode'"} } + user "postgres" + not_if %{psql postgres -tAc "SELECT 1 FROM pg_database WHERE LOWER(datname) = 'stockaid_production'" | grep 1}, user: "postgres" + notifies :run, "execute[grant-postgres-database-to-user]", :immediately +end + +execute "grant-postgres-database-to-user" do + command %{psql -c "GRANT ALL PRIVILEGES ON DATABASE \\"stockaid_production\\" TO \\"stockaid\\""} + user "postgres" + action :nothing +end diff --git a/chef/remote/cookbooks/stockaid/recipes/default.rb b/chef/remote/cookbooks/stockaid/recipes/default.rb new file mode 100644 index 00000000..7427fee6 --- /dev/null +++ b/chef/remote/cookbooks/stockaid/recipes/default.rb @@ -0,0 +1,7 @@ +include_recipe "stockaid::repo" +include_recipe "stockaid::rvm" +include_recipe "stockaid::database" +include_recipe "stockaid::rails" +include_recipe "stockaid::self_signed_ssl" +include_recipe "stockaid::nginx" +include_recipe "stockaid::letsencrypt" if node[:stockaid][:use_letsencrypt] diff --git a/chef/remote/cookbooks/stockaid/recipes/letsencrypt.rb b/chef/remote/cookbooks/stockaid/recipes/letsencrypt.rb new file mode 100644 index 00000000..8590dfcf --- /dev/null +++ b/chef/remote/cookbooks/stockaid/recipes/letsencrypt.rb @@ -0,0 +1,34 @@ +package "letsencrypt" + +raise "node[:stockaid][:letsencrypt_email] is not set!" unless node[:stockaid][:letsencrypt_email] + +template "/usr/bin/stockaid_letsencrypt_renew" do + source "letsencrypt/stockaid_letsencrypt_renew.erb" + owner "root" + group "root" + mode "0744" +atend + +file "/etc/cron.d/letsencrypt_renew" do + content "# This file is managed by chef +0 5 * * * root /usr/bin/stockaid_letsencrypt_renew >> /var/log/letsencrypt-renew.log +" + owner "root" + group "root" + mode "0644" +end + +directory "/var/www-letsencrypt/#{node[:stockaid][:domain]}" do + owner "www-data" + group "www-data" + mode "0744" + recursive true +end + +execute "setup-letsencrypt" do + command "letsencrypt certonly --non-interactive --agree-tos --email '#{node[:stockaid][:letsencrypt_email]}' " \ + "--webroot -w '/var/www-letsencrypt/#{node[:stockaid][:domain]}' -d '#{node[:stockaid][:domain]}'" + creates "/etc/letsencrypt/live/#{node[:stockaid][:domain]}" + notifies :run, "execute[reload-nginx]", :before + notifies :create, "template[/etc/nginx/sites-available/stockaid]", :immediately +end diff --git a/chef/remote/cookbooks/stockaid/recipes/nginx.rb b/chef/remote/cookbooks/stockaid/recipes/nginx.rb new file mode 100644 index 00000000..75612abe --- /dev/null +++ b/chef/remote/cookbooks/stockaid/recipes/nginx.rb @@ -0,0 +1,119 @@ +%w( + apt-transport-https + ca-certificates +).each do |pkg| + package pkg +end + +apt_key = "561F9B9CAC40B2F7" + +execute "add-passenger-apt-key" do + command "apt-key adv --keyserver 'hkp://keyserver.ubuntu.com:80' --recv-keys '#{apt_key}'" + + not_if do + `apt-key finger`.split("\n").map do |line| + line[/Key fingerprint = ([0-9A-F ]+)/, 1] + end.compact.map do |fingerprint| + fingerprint.split.join + end.any? do |fingerprint| + fingerprint.end_with? apt_key + end + end +end + +file "/etc/apt/sources.list.d/passenger.list" do + content lazy { + ubuntu_codename = `lsb_release -s -c`.strip + "deb https://oss-binaries.phusionpassenger.com/apt/passenger #{ubuntu_codename} main" + } + + owner "root" + group "root" + mode "0644" +end + +execute "update-apt" do + command "apt-get update" + action :nothing +end + +%w( + nginx-extras + passenger +).each do |pkg| + package pkg do + options "--force-yes" + notifies :run, "execute[update-apt]", :before + notifies :run, "execute[reload-nginx]", :immediately + end +end + +template "/etc/nginx/sites-available/default" do + source "nginx/default.nginx.conf.erb" + owner "root" + group "root" + mode "0644" + notifies :run, "execute[reload-nginx]" +end + +link "/etc/nginx/conf.d/passenger.conf" do + to "/etc/nginx/passenger.conf" + notifies :run, "execute[reload-nginx]" +end + +template "/etc/nginx/sites-available/stockaid" do + source "nginx/stockaid.nginx.conf.erb" + owner "root" + group "root" + mode "0644" + + variables( + lazy { + stockaid_ruby_file = File.join(node[:stockaid][:repo_dir], ".ruby-version") + ruby = File.read(stockaid_ruby_file).strip + database_password_file = File.join(node[:stockaid][:dir], ".stockaid-db-password") + database_password = File.read(database_password_file).strip + secret_key_base_file = File.join(node[:stockaid][:dir], ".stockaid-secret-key-base") + secret_key_base = File.read(secret_key_base_file) + devise_pepper_file = File.join(node[:stockaid][:dir], ".stockaid-devise-pepper") + devise_pepper = File.read(devise_pepper_file) + + if File.exist?("/etc/letsencrypt/live/#{node[:stockaid][:domain]}/fullchain.pem") + certificate = "/etc/letsencrypt/live/#{node[:stockaid][:domain]}/fullchain.pem" + certificate_key = "/etc/letsencrypt/live/#{node[:stockaid][:domain]}/privkey.pem" + else + certificate = "/etc/self-signed-ssl/stockaid.crt" + certificate_key = "/etc/self-signed-ssl/stockaid.key" + end + + { + domain: node[:stockaid][:domain], + passenger_ruby: File.join(node[:stockaid][:home], ".rvm/wrappers/#{ruby}/ruby"), + rails_root: node[:stockaid][:repo_dir], + certificate: certificate, + certificate_key: certificate_key, + env: { + STOCKAID_DATABASE_HOST: "localhost", + STOCKAID_DATABASE_USERNAME: "stockaid", + STOCKAID_DATABASE_PASSWORD: database_password, + STOCKAID_SECRET_KEY_BASE: secret_key_base, + STOCKAID_DEVISE_PEPPER: devise_pepper, + STOCKAID_ENV_SETUP: "3" + } + } + } + ) + + notifies :run, "execute[reload-nginx]", :immediately +end + +link "/etc/nginx/sites-enabled/stockaid" do + to "/etc/nginx/sites-available/stockaid" + notifies :run, "execute[reload-nginx]", :immediately +end + +execute "reload-nginx" do + command "systemctl reload nginx" + ignore_failure true + action :nothing +end diff --git a/chef/remote/cookbooks/stockaid/recipes/rails.rb b/chef/remote/cookbooks/stockaid/recipes/rails.rb new file mode 100644 index 00000000..1db950ca --- /dev/null +++ b/chef/remote/cookbooks/stockaid/recipes/rails.rb @@ -0,0 +1,86 @@ +# For execjs to work without using the ruby racer gem +package "nodejs" + +rvm_binary = File.join(node[:stockaid][:home], ".rvm/bin/rvm") +rvm_environment = { + "USER" => node[:stockaid][:user], + "HOME" => node[:stockaid][:home], + "TERM" => "dumb", + "RAILS_ENV" => "production" +} + +execute "install-bundler" do + command "#{rvm_binary} in '#{node[:stockaid][:repo_dir]}' do gem install bundler --no-ri --no-rdoc" + user node[:stockaid][:user] + group node[:stockaid][:group] + cwd node[:stockaid][:repo_dir] + not_if "#{rvm_binary} in '#{node[:stockaid][:repo_dir]}' do gem list | grep ^bundler\\\\s", user: node[:stockaid][:user], environment: rvm_environment +end + +execute "bundle-install" do + command "#{rvm_binary} in '#{node[:stockaid][:repo_dir]}' do bundle install --frozen --with production" + user node[:stockaid][:user] + group node[:stockaid][:group] + cwd node[:stockaid][:repo_dir] + not_if "#{rvm_binary} in '#{node[:stockaid][:repo_dir]}' do bundle check", user: node[:stockaid][:user], environment: rvm_environment + notifies :run, "execute[reload-nginx]" +end + +require "securerandom" +secret_key_base_file = File.join(node[:stockaid][:dir], ".stockaid-secret-key-base") + +file secret_key_base_file do + content SecureRandom.hex(64) + user node[:stockaid][:user] + group node[:stockaid][:group] + mode "0600" + action :create_if_missing +end + +devise_pepper_file = File.join(node[:stockaid][:dir], ".stockaid-devise-pepper") + +file devise_pepper_file do + content SecureRandom.hex(64) + user node[:stockaid][:user] + group node[:stockaid][:group] + mode "0600" + action :create_if_missing +end + +database_password_file = File.join(node[:stockaid][:dir], ".stockaid-db-password") + +file File.join(node[:stockaid][:repo_dir], ".ruby-env") do + content lazy { + secret_key_base = File.read(secret_key_base_file).strip + devise_pepper = File.read(devise_pepper_file).strip + db_password = File.read(database_password_file).strip + + "# This file is generated by Chef +RAILS_ENV=production +STOCKAID_SECRET_KEY_BASE=#{secret_key_base} +STOCKAID_DEVISE_PEPPER=#{devise_pepper} +STOCKAID_DATABASE_HOST=localhost +STOCKAID_DATABASE_USERNAME=stockaid +STOCKAID_DATABASE_PASSWORD=#{db_password} +STOCKAID_ENV_SETUP=3" + } +end + +asset_check_path = File.join(node[:stockaid][:dir], ".stockaid-assets-precompiled") + +execute "rake-assets-precompile" do + command "#{rvm_binary} in '#{node[:stockaid][:repo_dir]}' do rake assets:precompile && git log -1 --format=format:%H%n >> '#{asset_check_path}'" + user node[:stockaid][:user] + group node[:stockaid][:group] + cwd node[:stockaid][:repo_dir] + not_if { File.exist?(asset_check_path) && File.read(asset_check_path)[`cd '#{node[:stockaid][:repo_dir]}' && git log -1 --format=format:%H`] } + notifies :run, "execute[reload-nginx]" +end + +execute "rake-db-migrate" do + command "#{rvm_binary} in '#{node[:stockaid][:repo_dir]}' do bundle exec rake db:migrate" + user node[:stockaid][:user] + group node[:stockaid][:group] + cwd node[:stockaid][:repo_dir] + not_if "#{rvm_binary} in '#{node[:stockaid][:repo_dir]}' do bundle exec rake db:abort_if_pending_migrations", user: node[:stockaid][:user], environment: rvm_environment +end diff --git a/chef/remote/cookbooks/stockaid/recipes/repo.rb b/chef/remote/cookbooks/stockaid/recipes/repo.rb new file mode 100644 index 00000000..a1f5aaa3 --- /dev/null +++ b/chef/remote/cookbooks/stockaid/recipes/repo.rb @@ -0,0 +1,14 @@ +package "git" + +directory node[:stockaid][:dir] do + owner node[:stockaid][:user] + group node[:stockaid][:group] + recursive true +end + +execute "git-clone-stockaid" do + command "git clone '#{node[:stockaid][:github_url]}' '#{node[:stockaid][:repo_dir]}'" + user node[:stockaid][:user] + group node[:stockaid][:group] + not_if { File.exist?(File.join(node[:stockaid][:repo_dir], ".git")) } +end diff --git a/chef/remote/cookbooks/stockaid/recipes/rvm.rb b/chef/remote/cookbooks/stockaid/recipes/rvm.rb new file mode 100644 index 00000000..3955fac2 --- /dev/null +++ b/chef/remote/cookbooks/stockaid/recipes/rvm.rb @@ -0,0 +1,53 @@ +%w( + autoconf + automake + bison + curl + gawk + libffi-dev + libgdbm-dev + libncurses5-dev + libreadline6-dev + libsqlite3-dev + libssl-dev + libtool + libyaml-dev + pkg-config + sqlite3 + zlib1g-dev +).each do |pkg| + package pkg +end + +gpg_key = "409B6B1796C275462A1703113804BB82D39DC0E3" +stockaid_ruby_file = File.join(node[:stockaid][:repo_dir], ".ruby-version") +rvm_binary = File.join node[:stockaid][:home], ".rvm/bin/rvm" +rvm_environment = { + "USER" => node[:stockaid][:user], + "HOME" => node[:stockaid][:home], + "TERM" => "dumb" +} + +execute "install-rvm-key" do + command "gpg --keyserver hkp://keys.gnupg.net --recv-keys #{gpg_key}" + user node[:stockaid][:user] + group node[:stockaid][:group] + environment rvm_environment + not_if "gpg -k #{gpg_key}", user: node[:stockaid][:user], environment: rvm_environment +end + +execute "install-rvm" do + command "curl -sSL https://get.rvm.io | bash -s stable --autolibs=read-fail" + user node[:stockaid][:user] + group node[:stockaid][:group] + environment rvm_environment + not_if { File.exist?(File.join(node[:stockaid][:home], ".rvm")) } +end + +execute "rvm-install-ruby" do + command lazy { "#{rvm_binary} install #{File.read(stockaid_ruby_file).strip}" } + user node[:stockaid][:user] + group node[:stockaid][:group] + environment rvm_environment + not_if { File.exist?(File.join(node[:stockaid][:home], ".rvm/rubies/#{File.read(stockaid_ruby_file).strip}")) } +end diff --git a/chef/remote/cookbooks/stockaid/recipes/self_signed_ssl.rb b/chef/remote/cookbooks/stockaid/recipes/self_signed_ssl.rb new file mode 100644 index 00000000..0589ffa4 --- /dev/null +++ b/chef/remote/cookbooks/stockaid/recipes/self_signed_ssl.rb @@ -0,0 +1,12 @@ +# The self signed cert will be used until lets encrypt is ready +directory "/etc/self-signed-ssl" do + owner "root" + group "root" + mode "0700" + recursive true +end + +execute "generate-self-signed-cert" do + command %{openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/self-signed-ssl/stockaid.key -out /etc/self-signed-ssl/stockaid.crt -subj "/C=/ST=/L=/O=/OU=/CN=#{node[:stockaid][:domain]}"} + not_if { File.exist?("/etc/self-signed-ssl/stockaid.key") && File.exist?("/etc/self-signed-ssl/stockaid.crt") } +end diff --git a/chef/remote/cookbooks/stockaid/templates/default/letsencrypt/stockaid_letsencrypt_renew.erb b/chef/remote/cookbooks/stockaid/templates/default/letsencrypt/stockaid_letsencrypt_renew.erb new file mode 100644 index 00000000..322e7b44 --- /dev/null +++ b/chef/remote/cookbooks/stockaid/templates/default/letsencrypt/stockaid_letsencrypt_renew.erb @@ -0,0 +1,48 @@ +#!/bin/bash + +sleep_before_renew=true + +while test $# -gt 0 +do + case "$1" in + --no-sleep) sleep_before_renew=false + ;; + --*) echo "invalid option $1" + ;; + *) echo "invalid argument $1" + ;; + esac + shift +done + +ping_and_check_url() { + echo "*** Pinging $1 at $(date) ***" + + wget -qO- $1 &> /dev/null + if [ $? -ne 0 ]; then + echo "$1 FAILED!!!" + else + echo "$1 succeeded" + fi +} + +echo "*************************************************************************" +echo "*** Starting letsencrypt renew script at $(date) ***" + +if [ "$sleep_before_renew" = "true" ]; then + random_sleep=${RANDOM:0:2} + echo "*** Sleeping for $random_sleep minutes ***" + sleep ${random_sleep}m +fi + +echo "*** Renewing certificates at $(date) ***" +letsencrypt renew --non-interactive --agree-tos --email <%= node[:stockaid][:letsencrypt_email] %> +echo "*** Reloading nginx at $(date) ***" +systemctl reload nginx.service +echo "*** Sleeping to let nginx reload successfully ***" +sleep 5 + +# Ping the domains to make sure they came back up ok, and also to spin up the +# Rails processes +echo "*** Pinging urls at $(date) ***" +ping_and_check_url <%= node[:stockaid][:domain] %> diff --git a/chef/remote/cookbooks/stockaid/templates/default/nginx/default.nginx.conf.erb b/chef/remote/cookbooks/stockaid/templates/default/nginx/default.nginx.conf.erb new file mode 100644 index 00000000..ef0990bb --- /dev/null +++ b/chef/remote/cookbooks/stockaid/templates/default/nginx/default.nginx.conf.erb @@ -0,0 +1,12 @@ +# A blank server for the default site +server { + listen 80 default_server; + listen [::]:80 default_server; + server_name _; + server_tokens off; + + location / { + add_header Content-Type text/plain; + return 200 'Hello!'; + } +} diff --git a/chef/remote/cookbooks/stockaid/templates/default/nginx/stockaid.nginx.conf.erb b/chef/remote/cookbooks/stockaid/templates/default/nginx/stockaid.nginx.conf.erb new file mode 100644 index 00000000..51e6f581 --- /dev/null +++ b/chef/remote/cookbooks/stockaid/templates/default/nginx/stockaid.nginx.conf.erb @@ -0,0 +1,33 @@ +server { + listen 443 ssl; + server_name <%= @domain %>; + server_tokens off; + root <%= File.join(@rails_root, "public") %>; + gzip_static on; + passenger_enabled on; + passenger_ruby <%= @passenger_ruby %>; + + <% @env.each do |key, value| %> + passenger_env_var <%= key %> <%= value %>; + <% end %> + + rails_env production; + ssl on; + ssl_certificate <%= @certificate %>; + ssl_certificate_key <%= @certificate_key %>; + client_max_body_size 4m; +} + +server { + listen 80; + server_name <%= @domain %>; + server_tokens off; + + location / { + return 301 https://<%= @domain %>$request_uri; + } + + location ^~ /.well-known { + alias /var/www-letsencrypt/<%= @domain %>/.well-known; + } +} diff --git a/config/database.yml b/config/database.yml index 06f054a4..c97d9a39 100644 --- a/config/database.yml +++ b/config/database.yml @@ -2,6 +2,7 @@ default: &default adapter: postgresql encoding: unicode pool: 5 + host: <%= ENV['STOCKAID_DATABASE_HOST'] %> username: <%= ENV['STOCKAID_DATABASE_USERNAME'] %> password: <%= ENV['STOCKAID_DATABASE_PASSWORD'] %> From 5ac7470dacef7158a9dba4261fb6f70fba0ea342 Mon Sep 17 00:00:00 2001 From: Mike Virata-Stone Date: Sun, 14 May 2017 18:57:18 -0700 Subject: [PATCH 3/9] Refactoring and add more required environment --- .../cookbooks/stockaid/attributes/default.rb | 14 ++++++-- .../stockaid/libraries/stockaid_helper.rb | 36 +++++++++++++++++++ .../cookbooks/stockaid/recipes/default.rb | 3 +- .../cookbooks/stockaid/recipes/letsencrypt.rb | 4 +-- .../cookbooks/stockaid/recipes/nginx.rb | 15 +------- .../cookbooks/stockaid/recipes/rails.rb | 18 ++++------ .../cookbooks/stockaid/recipes/sidekiq.rb | 1 + .../stockaid_letsencrypt_renew.erb | 2 +- .../default/nginx/stockaid.nginx.conf.erb | 6 +++- 9 files changed, 66 insertions(+), 33 deletions(-) create mode 100644 chef/remote/cookbooks/stockaid/libraries/stockaid_helper.rb create mode 100644 chef/remote/cookbooks/stockaid/recipes/sidekiq.rb diff --git a/chef/remote/cookbooks/stockaid/attributes/default.rb b/chef/remote/cookbooks/stockaid/attributes/default.rb index 805ede0b..03da4033 100644 --- a/chef/remote/cookbooks/stockaid/attributes/default.rb +++ b/chef/remote/cookbooks/stockaid/attributes/default.rb @@ -5,7 +5,15 @@ default[:stockaid][:dir] = node[:stockaid][:home] default[:stockaid][:repo_dir] = File.join(node[:stockaid][:dir], "StockAid") default[:stockaid][:domain] = "orders.gratefulgarment.org" -default[:stockaid][:use_letsencrypt] = true +default[:stockaid][:site_name] = "The Grateful Garment Project" -# This must be set for letsencrypt to work -default[:stockaid][:letsencrypt_email] = nil +default[:stockaid][:mailgun][:enabled] = true +default[:stockaid][:mailgun][:domain] = "mg.gratefulgarment.org" +default[:stockaid][:mailgun][:api_key] = nil # This must be set for mailgun to work + +default[:stockaid][:letsencrypt][:enabled] = true +default[:stockaid][:letsencrypt][:email] = nil # This must be set for letsencrypt to work + +default[:stockaid][:newrelic][:enabled] = true +default[:stockaid][:newrelic][:app_name] = "grateful-garment" +default[:stockaid][:newrelic][:license_key] = nil # This must be set for newrelic to work diff --git a/chef/remote/cookbooks/stockaid/libraries/stockaid_helper.rb b/chef/remote/cookbooks/stockaid/libraries/stockaid_helper.rb new file mode 100644 index 00000000..74382809 --- /dev/null +++ b/chef/remote/cookbooks/stockaid/libraries/stockaid_helper.rb @@ -0,0 +1,36 @@ +module StockAid + module Helper + module_function def stockaid_environment(node) + database_password_file = File.join(node[:stockaid][:dir], ".stockaid-db-password") + database_password = File.read(database_password_file).strip + secret_key_base_file = File.join(node[:stockaid][:dir], ".stockaid-secret-key-base") + secret_key_base = File.read(secret_key_base_file) + devise_pepper_file = File.join(node[:stockaid][:dir], ".stockaid-devise-pepper") + devise_pepper = File.read(devise_pepper_file) + + {}.tap do |environment| + environment.merge!( + "STOCKAID_DATABASE_HOST" => "localhost", + "STOCKAID_DATABASE_USERNAME" => "stockaid", + "STOCKAID_DATABASE_PASSWORD" => database_password, + "STOCKAID_SECRET_KEY_BASE" => secret_key_base, + "STOCKAID_DEVISE_PEPPER" => devise_pepper, + "STOCKAID_ENV_SETUP" => "3", + "STOCKAID_SITE_NAME" => node[:stockaid][:site_name] + ) + + if node[:stockaid][:mailgun][:enabled] + environment["STOCKAID_MAILGUN_DOMAIN"] = node[:stockaid][:mailgun][:domain] + environment["STOCKAID_MAILGUN_API_KEY"] = node[:stockaid][:mailgun][:api_key] + end + + if node[:stockaid][:newrelic][:enabled] + environment["NEW_RELIC_APP_NAME"] = node[:stockaid][:newrelic][:app_name] + environment["NEW_RELIC_LICENSE_KEY"] = node[:stockaid][:newrelic][:license_key] + end + + environment + end + end + end +end diff --git a/chef/remote/cookbooks/stockaid/recipes/default.rb b/chef/remote/cookbooks/stockaid/recipes/default.rb index 7427fee6..69903229 100644 --- a/chef/remote/cookbooks/stockaid/recipes/default.rb +++ b/chef/remote/cookbooks/stockaid/recipes/default.rb @@ -2,6 +2,7 @@ include_recipe "stockaid::rvm" include_recipe "stockaid::database" include_recipe "stockaid::rails" +include_recipe "stockaid::sidekiq" include_recipe "stockaid::self_signed_ssl" include_recipe "stockaid::nginx" -include_recipe "stockaid::letsencrypt" if node[:stockaid][:use_letsencrypt] +include_recipe "stockaid::letsencrypt" if node[:stockaid][:letsencrypt][:enabled] diff --git a/chef/remote/cookbooks/stockaid/recipes/letsencrypt.rb b/chef/remote/cookbooks/stockaid/recipes/letsencrypt.rb index 8590dfcf..93c25631 100644 --- a/chef/remote/cookbooks/stockaid/recipes/letsencrypt.rb +++ b/chef/remote/cookbooks/stockaid/recipes/letsencrypt.rb @@ -1,6 +1,6 @@ package "letsencrypt" -raise "node[:stockaid][:letsencrypt_email] is not set!" unless node[:stockaid][:letsencrypt_email] +raise "node[:stockaid][:letsencrypt][:email] is not set!" unless node[:stockaid][:letsencrypt][:email] template "/usr/bin/stockaid_letsencrypt_renew" do source "letsencrypt/stockaid_letsencrypt_renew.erb" @@ -26,7 +26,7 @@ end execute "setup-letsencrypt" do - command "letsencrypt certonly --non-interactive --agree-tos --email '#{node[:stockaid][:letsencrypt_email]}' " \ + command "letsencrypt certonly --non-interactive --agree-tos --email '#{node[:stockaid][:letsencrypt][:email]}' " \ "--webroot -w '/var/www-letsencrypt/#{node[:stockaid][:domain]}' -d '#{node[:stockaid][:domain]}'" creates "/etc/letsencrypt/live/#{node[:stockaid][:domain]}" notifies :run, "execute[reload-nginx]", :before diff --git a/chef/remote/cookbooks/stockaid/recipes/nginx.rb b/chef/remote/cookbooks/stockaid/recipes/nginx.rb index 75612abe..da05bd3e 100644 --- a/chef/remote/cookbooks/stockaid/recipes/nginx.rb +++ b/chef/remote/cookbooks/stockaid/recipes/nginx.rb @@ -71,12 +71,6 @@ lazy { stockaid_ruby_file = File.join(node[:stockaid][:repo_dir], ".ruby-version") ruby = File.read(stockaid_ruby_file).strip - database_password_file = File.join(node[:stockaid][:dir], ".stockaid-db-password") - database_password = File.read(database_password_file).strip - secret_key_base_file = File.join(node[:stockaid][:dir], ".stockaid-secret-key-base") - secret_key_base = File.read(secret_key_base_file) - devise_pepper_file = File.join(node[:stockaid][:dir], ".stockaid-devise-pepper") - devise_pepper = File.read(devise_pepper_file) if File.exist?("/etc/letsencrypt/live/#{node[:stockaid][:domain]}/fullchain.pem") certificate = "/etc/letsencrypt/live/#{node[:stockaid][:domain]}/fullchain.pem" @@ -92,14 +86,7 @@ rails_root: node[:stockaid][:repo_dir], certificate: certificate, certificate_key: certificate_key, - env: { - STOCKAID_DATABASE_HOST: "localhost", - STOCKAID_DATABASE_USERNAME: "stockaid", - STOCKAID_DATABASE_PASSWORD: database_password, - STOCKAID_SECRET_KEY_BASE: secret_key_base, - STOCKAID_DEVISE_PEPPER: devise_pepper, - STOCKAID_ENV_SETUP: "3" - } + env: StockAid::Helper.stockaid_environment(node) } } ) diff --git a/chef/remote/cookbooks/stockaid/recipes/rails.rb b/chef/remote/cookbooks/stockaid/recipes/rails.rb index 1db950ca..3ea3cb89 100644 --- a/chef/remote/cookbooks/stockaid/recipes/rails.rb +++ b/chef/remote/cookbooks/stockaid/recipes/rails.rb @@ -51,18 +51,14 @@ file File.join(node[:stockaid][:repo_dir], ".ruby-env") do content lazy { - secret_key_base = File.read(secret_key_base_file).strip - devise_pepper = File.read(devise_pepper_file).strip - db_password = File.read(database_password_file).strip + [].tap do |lines| + lines << "# This file is generated by Chef" + lines << "RAILS_ENV=production" - "# This file is generated by Chef -RAILS_ENV=production -STOCKAID_SECRET_KEY_BASE=#{secret_key_base} -STOCKAID_DEVISE_PEPPER=#{devise_pepper} -STOCKAID_DATABASE_HOST=localhost -STOCKAID_DATABASE_USERNAME=stockaid -STOCKAID_DATABASE_PASSWORD=#{db_password} -STOCKAID_ENV_SETUP=3" + StockAid::Helper.stockaid_environment(node).each do |key, value| + lines << "#{key}=#{value}" + end + end.join("\n") } end diff --git a/chef/remote/cookbooks/stockaid/recipes/sidekiq.rb b/chef/remote/cookbooks/stockaid/recipes/sidekiq.rb new file mode 100644 index 00000000..46409041 --- /dev/null +++ b/chef/remote/cookbooks/stockaid/recipes/sidekiq.rb @@ -0,0 +1 @@ +# TODO diff --git a/chef/remote/cookbooks/stockaid/templates/default/letsencrypt/stockaid_letsencrypt_renew.erb b/chef/remote/cookbooks/stockaid/templates/default/letsencrypt/stockaid_letsencrypt_renew.erb index 322e7b44..9580f6f1 100644 --- a/chef/remote/cookbooks/stockaid/templates/default/letsencrypt/stockaid_letsencrypt_renew.erb +++ b/chef/remote/cookbooks/stockaid/templates/default/letsencrypt/stockaid_letsencrypt_renew.erb @@ -36,7 +36,7 @@ if [ "$sleep_before_renew" = "true" ]; then fi echo "*** Renewing certificates at $(date) ***" -letsencrypt renew --non-interactive --agree-tos --email <%= node[:stockaid][:letsencrypt_email] %> +letsencrypt renew --non-interactive --agree-tos --email <%= node[:stockaid][:letsencrypt][:email] %> echo "*** Reloading nginx at $(date) ***" systemctl reload nginx.service echo "*** Sleeping to let nginx reload successfully ***" diff --git a/chef/remote/cookbooks/stockaid/templates/default/nginx/stockaid.nginx.conf.erb b/chef/remote/cookbooks/stockaid/templates/default/nginx/stockaid.nginx.conf.erb index 51e6f581..01b196a9 100644 --- a/chef/remote/cookbooks/stockaid/templates/default/nginx/stockaid.nginx.conf.erb +++ b/chef/remote/cookbooks/stockaid/templates/default/nginx/stockaid.nginx.conf.erb @@ -8,7 +8,11 @@ server { passenger_ruby <%= @passenger_ruby %>; <% @env.each do |key, value| %> - passenger_env_var <%= key %> <%= value %>; + <% if value.is_a?(String) && value =~ /\s/ %> + passenger_env_var <%= key %> <%= value.inspect %>; + <% else %> + passenger_env_var <%= key %> <%= value %>; + <% end %> <% end %> rails_env production; From ecc944a5e6b201ba940ef353e7a10c0a7b222b55 Mon Sep 17 00:00:00 2001 From: Mike Virata-Stone Date: Sun, 14 May 2017 19:09:39 -0700 Subject: [PATCH 4/9] Add more config that was missing --- .../remote/cookbooks/stockaid/attributes/default.rb | 6 ++++++ .../cookbooks/stockaid/libraries/stockaid_helper.rb | 13 ++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/chef/remote/cookbooks/stockaid/attributes/default.rb b/chef/remote/cookbooks/stockaid/attributes/default.rb index 03da4033..1bf36b22 100644 --- a/chef/remote/cookbooks/stockaid/attributes/default.rb +++ b/chef/remote/cookbooks/stockaid/attributes/default.rb @@ -7,6 +7,12 @@ default[:stockaid][:domain] = "orders.gratefulgarment.org" default[:stockaid][:site_name] = "The Grateful Garment Project" +default[:stockaid][:google][:api_key] = nil # Set for some Google integrations +default[:stockaid][:google][:drive_json] = nil # Set for automatic DB backups to Google Drive + +default[:stockaid][:mailer][:default_from] = "noreply@gratefulgarment.org" +default[:stockaid][:mailer][:default_host] = "orders.gratefulgarment.org" + default[:stockaid][:mailgun][:enabled] = true default[:stockaid][:mailgun][:domain] = "mg.gratefulgarment.org" default[:stockaid][:mailgun][:api_key] = nil # This must be set for mailgun to work diff --git a/chef/remote/cookbooks/stockaid/libraries/stockaid_helper.rb b/chef/remote/cookbooks/stockaid/libraries/stockaid_helper.rb index 74382809..05b5139a 100644 --- a/chef/remote/cookbooks/stockaid/libraries/stockaid_helper.rb +++ b/chef/remote/cookbooks/stockaid/libraries/stockaid_helper.rb @@ -16,9 +16,20 @@ module Helper "STOCKAID_SECRET_KEY_BASE" => secret_key_base, "STOCKAID_DEVISE_PEPPER" => devise_pepper, "STOCKAID_ENV_SETUP" => "3", - "STOCKAID_SITE_NAME" => node[:stockaid][:site_name] + "STOCKAID_SITE_NAME" => node[:stockaid][:site_name], + "STOCKAID_ACTION_MAILER_DEFAULT_FROM" => node[:stockaid][:mailer][:default_from], + "STOCKAID_ACTION_MAILER_DEFAULT_HOST" => node[:stockaid][:mailer][:default_host] ) + if node[:stockaid][:google][:api_key] + environment["STOCKAID_GOOGLE_API_KEY"] = node[:stockaid][:google][:api_key] + end + + if node[:stockaid][:google][:drive_json] + require "json" + environment["STOCKAID_GOOGLE_DRIVE_SERVICE_ACCOUNT_JSON"] = node[:stockaid][:google][:drive_json].to_json + end + if node[:stockaid][:mailgun][:enabled] environment["STOCKAID_MAILGUN_DOMAIN"] = node[:stockaid][:mailgun][:domain] environment["STOCKAID_MAILGUN_API_KEY"] = node[:stockaid][:mailgun][:api_key] From 223df27193edb73774ad637ed1644610136a4b7e Mon Sep 17 00:00:00 2001 From: Mike Virata-Stone Date: Sun, 14 May 2017 22:40:07 -0700 Subject: [PATCH 5/9] Implement sidekiq --- .../stockaid/libraries/stockaid_helper.rb | 17 ++++++++++ .../cookbooks/stockaid/recipes/rails.rb | 7 +++- .../cookbooks/stockaid/recipes/sidekiq.rb | 23 ++++++++++++- .../default/sidekiq/sidekiq.service.erb | 34 +++++++++++++++++++ 4 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 chef/remote/cookbooks/stockaid/templates/default/sidekiq/sidekiq.service.erb diff --git a/chef/remote/cookbooks/stockaid/libraries/stockaid_helper.rb b/chef/remote/cookbooks/stockaid/libraries/stockaid_helper.rb index 05b5139a..a37de102 100644 --- a/chef/remote/cookbooks/stockaid/libraries/stockaid_helper.rb +++ b/chef/remote/cookbooks/stockaid/libraries/stockaid_helper.rb @@ -1,5 +1,22 @@ module StockAid module Helper + module_function def systemd_env_variable(key, value) + { + "\\" => "\\\\", + "\n" => "\\n" + }.each do |string, replacement| + value = value.gsub(string, replacement) + end + + if value.include?('"') && value.include?("'") + raise "Please avoid using both ' and \" in an environment variable: #{value.inspect}" + elsif value.include?('"') + "Environment='#{key}=#{value}'" + else + %(Environment="#{key}=#{value}") + end + end + module_function def stockaid_environment(node) database_password_file = File.join(node[:stockaid][:dir], ".stockaid-db-password") database_password = File.read(database_password_file).strip diff --git a/chef/remote/cookbooks/stockaid/recipes/rails.rb b/chef/remote/cookbooks/stockaid/recipes/rails.rb index 3ea3cb89..a5623b1c 100644 --- a/chef/remote/cookbooks/stockaid/recipes/rails.rb +++ b/chef/remote/cookbooks/stockaid/recipes/rails.rb @@ -1,5 +1,10 @@ # For execjs to work without using the ruby racer gem -package "nodejs" +%w( + nodejs + redis-server +).each do |pkg| + package pkg +end rvm_binary = File.join(node[:stockaid][:home], ".rvm/bin/rvm") rvm_environment = { diff --git a/chef/remote/cookbooks/stockaid/recipes/sidekiq.rb b/chef/remote/cookbooks/stockaid/recipes/sidekiq.rb index 46409041..d4acc573 100644 --- a/chef/remote/cookbooks/stockaid/recipes/sidekiq.rb +++ b/chef/remote/cookbooks/stockaid/recipes/sidekiq.rb @@ -1 +1,22 @@ -# TODO +template "/etc/systemd/system/sidekiq.service" do + source "sidekiq/sidekiq.service.erb" + owner "root" + group "root" + mode "0600" + + variables lazy { + require "yaml" + procfile_file = File.join(node[:stockaid][:repo_dir], "Procfile") + procfile = YAML.load_file(procfile_file) + + { + command: procfile["worker"], + env: StockAid::Helper.stockaid_environment(node) + } + } +end + +service "sidekiq" do + action [:enable, :start] + subscribes :reload, "template[/etc/systemd/system/sidekiq.service]", :immediately +end diff --git a/chef/remote/cookbooks/stockaid/templates/default/sidekiq/sidekiq.service.erb b/chef/remote/cookbooks/stockaid/templates/default/sidekiq/sidekiq.service.erb new file mode 100644 index 00000000..3db5966d --- /dev/null +++ b/chef/remote/cookbooks/stockaid/templates/default/sidekiq/sidekiq.service.erb @@ -0,0 +1,34 @@ +# This file is managed by Chef +# +# Based on: +# https://github.com/mperham/sidekiq/blob/e2f588d8bd66481e9fb2f7a7fd1378344c4709a9/examples/systemd/sidekiq.service +[Unit] +Description=sidekiq +# start us only once the network and logging subsystems are available, +# consider adding redis-server.service if Redis is local and systemd-managed. +After=network.target + +# See these pages for lots of options: +# http://0pointer.de/public/systemd-man/systemd.service.html +# http://0pointer.de/public/systemd-man/systemd.exec.html +[Service] +Type=simple +WorkingDirectory=<%= node[:stockaid][:repo_dir] %> +ExecStart=<%= File.join(node[:stockaid][:home], ".rvm/bin/rvm") %> in <%= node[:stockaid][:repo_dir] %> do <%= @command %> +User=<%= node[:stockaid][:user] %> +Group=<%= node[:stockaid][:user] %> +UMask=0002 + +<% @env.each do |key, value| -%> +<%= StockAid::Helper.systemd_env_variable(key, value) %> +<% end -%> + +# if we crash, restart +RestartSec=1 +Restart=on-failure + +# This will default to "bundler" if we don't specify it +SyslogIdentifier=sidekiq + +[Install] +WantedBy=multi-user.target From 849e787eeca33cc7d9a06368ab703f39ddd704e7 Mon Sep 17 00:00:00 2001 From: Mike Virata-Stone Date: Mon, 15 May 2017 01:06:01 -0700 Subject: [PATCH 6/9] Fix all the easy rubocop errors --- chef/deploy.rb | 23 +++++++++---------- .../cookbooks/stockaid/recipes/database.rb | 6 ++--- .../cookbooks/stockaid/recipes/nginx.rb | 4 ++-- .../stockaid/recipes/self_signed_ssl.rb | 2 +- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/chef/deploy.rb b/chef/deploy.rb index 888401f0..f8b913fb 100755 --- a/chef/deploy.rb +++ b/chef/deploy.rb @@ -8,8 +8,8 @@ # a command line option). REMOTE_DIR = File.expand_path("../remote", __FILE__) -CHEF_VERSION = "12.19.36" -SSH_DEFAULTS = { batch: true } +CHEF_VERSION = "12.19.36".freeze +SSH_DEFAULTS = { batch: true }.freeze DEFAULT_OPTIONS = { user: "chef", @@ -21,7 +21,7 @@ json: File.expand_path("../stockaid.json", __FILE__), clear_host: false, ssh: false -} +}.freeze OPTIONS = Hash.new { |_hash, key| DEFAULT_OPTIONS[key] } @@ -43,7 +43,7 @@ def ssh_options(options = {}) end def ssh(command, options = {}) - if command.kind_of?(Hash) + if command.is_a?(Hash) options = options.merge(command) options[:batch] = false if options[:exec] command = nil @@ -82,13 +82,13 @@ def rsync(from, to, options = {}) server = "#{user}@#{host}" from.sub! /\Aserver:/, "#{server}:" to.sub! /\Aserver:/, "#{server}:" - system_exec %{rsync -e "ssh -p #{port} -i #{identity_file} #{ssh_options(options)}" -av '#{from}' '#{to}'} + system_exec %(rsync -e "ssh -p #{port} -i #{identity_file} #{ssh_options(options)}" -av '#{from}' '#{to}') end OptionParser.new do |opts| opts.banner = "Usage: ./deploy.rb [options]" - opts.on "-s", "--ssh", "Just ssh to the server, don't actually deploy" do |ssh| + opts.on "-s", "--ssh", "Just ssh to the server, don't actually deploy" do |_ssh| OPTIONS[:ssh] = true end @@ -118,11 +118,10 @@ def rsync(from, to, options = {}) end.parse! JSON.parse(File.read(OPTIONS[:json])).fetch("meta", {}).each do |key, value| - unless OPTIONS.include?(key.to_sym) - puts "Using JSON default #{key}: #{value.inspect}" - value = File.expand_path(value) if %w(identity files_dir repos_dir).include?(key) - OPTIONS[key.to_sym] = value - end + next if OPTIONS.include?(key.to_sym) + puts "Using JSON default #{key}: #{value.inspect}" + value = File.expand_path(value) if %w(identity files_dir repos_dir).include?(key) + OPTIONS[key.to_sym] = value end ssh exec: true if OPTIONS[:ssh] @@ -133,7 +132,7 @@ def rsync(from, to, options = {}) pub_file = "#{OPTIONS[:identity]}.pub" public_key = File.read(pub_file) - unless ssh %{mkdir -p ~/.ssh && touch ~/.ssh/authorized_keys && echo "#{public_key}" >> ~/.ssh/authorized_keys}, batch: false + unless ssh %(mkdir -p ~/.ssh && touch ~/.ssh/authorized_keys && echo "#{public_key}" >> ~/.ssh/authorized_keys), batch: false abort "Failed to save authorized key!" end end diff --git a/chef/remote/cookbooks/stockaid/recipes/database.rb b/chef/remote/cookbooks/stockaid/recipes/database.rb index 5581e779..8e5ee504 100644 --- a/chef/remote/cookbooks/stockaid/recipes/database.rb +++ b/chef/remote/cookbooks/stockaid/recipes/database.rb @@ -16,20 +16,20 @@ end execute "create-postgres-user" do - command lazy { %{psql -c "CREATE USER \\"stockaid\\" WITH LOGIN PASSWORD '#{File.read(password_file).strip}'"} } + command lazy { %(psql -c "CREATE USER \\"stockaid\\" WITH LOGIN PASSWORD '#{File.read(password_file).strip}'") } user "postgres" not_if %{psql postgres -tAc "SELECT 1 FROM pg_roles WHERE LOWER(rolname) = 'stockaid'" | grep 1}, user: "postgres" end execute "create-postgres-database" do - command lazy { %{psql -c "CREATE DATABASE \\"stockaid_production\\" WITH OWNER \\"stockaid\\" ENCODING 'unicode'"} } + command lazy { %(psql -c "CREATE DATABASE \\"stockaid_production\\" WITH OWNER \\"stockaid\\" ENCODING 'unicode'") } user "postgres" not_if %{psql postgres -tAc "SELECT 1 FROM pg_database WHERE LOWER(datname) = 'stockaid_production'" | grep 1}, user: "postgres" notifies :run, "execute[grant-postgres-database-to-user]", :immediately end execute "grant-postgres-database-to-user" do - command %{psql -c "GRANT ALL PRIVILEGES ON DATABASE \\"stockaid_production\\" TO \\"stockaid\\""} + command %(psql -c "GRANT ALL PRIVILEGES ON DATABASE \\"stockaid_production\\" TO \\"stockaid\\"") user "postgres" action :nothing end diff --git a/chef/remote/cookbooks/stockaid/recipes/nginx.rb b/chef/remote/cookbooks/stockaid/recipes/nginx.rb index da05bd3e..96a7c36f 100644 --- a/chef/remote/cookbooks/stockaid/recipes/nginx.rb +++ b/chef/remote/cookbooks/stockaid/recipes/nginx.rb @@ -68,7 +68,7 @@ mode "0644" variables( - lazy { + lazy do stockaid_ruby_file = File.join(node[:stockaid][:repo_dir], ".ruby-version") ruby = File.read(stockaid_ruby_file).strip @@ -88,7 +88,7 @@ certificate_key: certificate_key, env: StockAid::Helper.stockaid_environment(node) } - } + end ) notifies :run, "execute[reload-nginx]", :immediately diff --git a/chef/remote/cookbooks/stockaid/recipes/self_signed_ssl.rb b/chef/remote/cookbooks/stockaid/recipes/self_signed_ssl.rb index 0589ffa4..f8fb551a 100644 --- a/chef/remote/cookbooks/stockaid/recipes/self_signed_ssl.rb +++ b/chef/remote/cookbooks/stockaid/recipes/self_signed_ssl.rb @@ -7,6 +7,6 @@ end execute "generate-self-signed-cert" do - command %{openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/self-signed-ssl/stockaid.key -out /etc/self-signed-ssl/stockaid.crt -subj "/C=/ST=/L=/O=/OU=/CN=#{node[:stockaid][:domain]}"} + command %(openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/self-signed-ssl/stockaid.key -out /etc/self-signed-ssl/stockaid.crt -subj "/C=/ST=/L=/O=/OU=/CN=#{node[:stockaid][:domain]}") not_if { File.exist?("/etc/self-signed-ssl/stockaid.key") && File.exist?("/etc/self-signed-ssl/stockaid.crt") } end From 4875bddd3a1171cae24ef4ba51f2c405ce5b5392 Mon Sep 17 00:00:00 2001 From: Mike Virata-Stone Date: Mon, 15 May 2017 01:35:21 -0700 Subject: [PATCH 7/9] Fix all the rubocop errors that require manual tweaks --- .rubocop.yml | 7 +- chef/deploy.rb | 14 +- .../stockaid/libraries/stockaid_helper.rb | 126 ++++++++++++------ .../cookbooks/stockaid/recipes/database.rb | 3 +- .../cookbooks/stockaid/recipes/letsencrypt.rb | 2 +- .../cookbooks/stockaid/recipes/nginx.rb | 10 +- .../cookbooks/stockaid/recipes/rails.rb | 11 +- 7 files changed, 110 insertions(+), 63 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index ae736962..6c15f6e5 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -15,11 +15,14 @@ Metrics/AbcSize: Max: 16 Metrics/ClassLength: Max: 150 -Metrics/LineLength: - Max: 120 Metrics/MethodLength: Max: 15 +Metrics/LineLength: + Max: 120 + Exclude: + - chef/**/* + Rails: Enabled: true diff --git a/chef/deploy.rb b/chef/deploy.rb index f8b913fb..da3fdc96 100755 --- a/chef/deploy.rb +++ b/chef/deploy.rb @@ -68,8 +68,8 @@ def scp(from, to, options = {}) port = OPTIONS[:port] identity_file = OPTIONS[:identity] server = "#{user}@#{host}" - from.sub! /\Aserver:/, "#{server}:" - to.sub! /\Aserver:/, "#{server}:" + from.sub!(/\Aserver:/, "#{server}:") + to.sub!(/\Aserver:/, "#{server}:") recursive = "-r" if options[:recursive] system_exec "scp #{recursive} -P #{port} -i #{identity_file} #{ssh_options(options)} '#{from}' '#{to}'" end @@ -80,8 +80,8 @@ def rsync(from, to, options = {}) port = OPTIONS[:port] identity_file = OPTIONS[:identity] server = "#{user}@#{host}" - from.sub! /\Aserver:/, "#{server}:" - to.sub! /\Aserver:/, "#{server}:" + from.sub!(/\Aserver:/, "#{server}:") + to.sub!(/\Aserver:/, "#{server}:") system_exec %(rsync -e "ssh -p #{port} -i #{identity_file} #{ssh_options(options)}" -av '#{from}' '#{to}') end @@ -131,10 +131,8 @@ def rsync(from, to, options = {}) unless ssh "echo ssh key is working" pub_file = "#{OPTIONS[:identity]}.pub" public_key = File.read(pub_file) - - unless ssh %(mkdir -p ~/.ssh && touch ~/.ssh/authorized_keys && echo "#{public_key}" >> ~/.ssh/authorized_keys), batch: false - abort "Failed to save authorized key!" - end + saved_key = ssh %(mkdir -p ~/.ssh && touch ~/.ssh/authorized_keys && echo "#{public_key}" >> ~/.ssh/authorized_keys), batch: false + abort "Failed to save authorized key!" unless saved_key end rsync "#{REMOTE_DIR}/", "server:~/next-stockaid-chef" diff --git a/chef/remote/cookbooks/stockaid/libraries/stockaid_helper.rb b/chef/remote/cookbooks/stockaid/libraries/stockaid_helper.rb index a37de102..d6ee6d21 100644 --- a/chef/remote/cookbooks/stockaid/libraries/stockaid_helper.rb +++ b/chef/remote/cookbooks/stockaid/libraries/stockaid_helper.rb @@ -1,4 +1,88 @@ +require "json" + module StockAid + class Environment + attr_reader :node + + def initialize(node) + @node = node + end + + def to_h + simple_env.merge(google_env).merge(mailgun_env).merge(newrelic_env) + end + + private + + def simple_env + { + "STOCKAID_DATABASE_HOST" => "localhost", + "STOCKAID_DATABASE_USERNAME" => "stockaid", + "STOCKAID_DATABASE_PASSWORD" => database_password, + "STOCKAID_SECRET_KEY_BASE" => secret_key_base, + "STOCKAID_DEVISE_PEPPER" => devise_pepper, + "STOCKAID_ENV_SETUP" => "3", + "STOCKAID_SITE_NAME" => node[:stockaid][:site_name], + "STOCKAID_ACTION_MAILER_DEFAULT_FROM" => node[:stockaid][:mailer][:default_from], + "STOCKAID_ACTION_MAILER_DEFAULT_HOST" => node[:stockaid][:mailer][:default_host] + } + end + + def database_password + File.read(File.join(node[:stockaid][:dir], ".stockaid-db-password")).strip + end + + def secret_key_base + File.read(File.join(node[:stockaid][:dir], ".stockaid-secret-key-base")) + end + + def devise_pepper + File.read(File.join(node[:stockaid][:dir], ".stockaid-devise-pepper")) + end + + def google_env + google_api_env.merge(google_drive_env) + end + + def google_api_env + if node[:stockaid][:google][:api_key] + { "STOCKAID_GOOGLE_API_KEY" => node[:stockaid][:google][:api_key] } + else + {} + end + end + + def google_drive_env + if node[:stockaid][:google][:drive_json] + { "STOCKAID_GOOGLE_DRIVE_SERVICE_ACCOUNT_JSON" => node[:stockaid][:google][:drive_json].to_json } + else + {} + end + end + + def mailgun_env + if node[:stockaid][:mailgun][:enabled] + { + "STOCKAID_MAILGUN_DOMAIN" => node[:stockaid][:mailgun][:domain], + "STOCKAID_MAILGUN_API_KEY" => node[:stockaid][:mailgun][:api_key] + } + else + {} + end + end + + def newrelic_env + if node[:stockaid][:newrelic][:enabled] + { + "NEW_RELIC_APP_NAME" => node[:stockaid][:newrelic][:app_name], + "NEW_RELIC_LICENSE_KEY" => node[:stockaid][:newrelic][:license_key] + } + else + {} + end + end + end + module Helper module_function def systemd_env_variable(key, value) { @@ -18,47 +102,7 @@ module Helper end module_function def stockaid_environment(node) - database_password_file = File.join(node[:stockaid][:dir], ".stockaid-db-password") - database_password = File.read(database_password_file).strip - secret_key_base_file = File.join(node[:stockaid][:dir], ".stockaid-secret-key-base") - secret_key_base = File.read(secret_key_base_file) - devise_pepper_file = File.join(node[:stockaid][:dir], ".stockaid-devise-pepper") - devise_pepper = File.read(devise_pepper_file) - - {}.tap do |environment| - environment.merge!( - "STOCKAID_DATABASE_HOST" => "localhost", - "STOCKAID_DATABASE_USERNAME" => "stockaid", - "STOCKAID_DATABASE_PASSWORD" => database_password, - "STOCKAID_SECRET_KEY_BASE" => secret_key_base, - "STOCKAID_DEVISE_PEPPER" => devise_pepper, - "STOCKAID_ENV_SETUP" => "3", - "STOCKAID_SITE_NAME" => node[:stockaid][:site_name], - "STOCKAID_ACTION_MAILER_DEFAULT_FROM" => node[:stockaid][:mailer][:default_from], - "STOCKAID_ACTION_MAILER_DEFAULT_HOST" => node[:stockaid][:mailer][:default_host] - ) - - if node[:stockaid][:google][:api_key] - environment["STOCKAID_GOOGLE_API_KEY"] = node[:stockaid][:google][:api_key] - end - - if node[:stockaid][:google][:drive_json] - require "json" - environment["STOCKAID_GOOGLE_DRIVE_SERVICE_ACCOUNT_JSON"] = node[:stockaid][:google][:drive_json].to_json - end - - if node[:stockaid][:mailgun][:enabled] - environment["STOCKAID_MAILGUN_DOMAIN"] = node[:stockaid][:mailgun][:domain] - environment["STOCKAID_MAILGUN_API_KEY"] = node[:stockaid][:mailgun][:api_key] - end - - if node[:stockaid][:newrelic][:enabled] - environment["NEW_RELIC_APP_NAME"] = node[:stockaid][:newrelic][:app_name] - environment["NEW_RELIC_LICENSE_KEY"] = node[:stockaid][:newrelic][:license_key] - end - - environment - end + StockAid::Environment.new(node).to_h end end end diff --git a/chef/remote/cookbooks/stockaid/recipes/database.rb b/chef/remote/cookbooks/stockaid/recipes/database.rb index 8e5ee504..985101d4 100644 --- a/chef/remote/cookbooks/stockaid/recipes/database.rb +++ b/chef/remote/cookbooks/stockaid/recipes/database.rb @@ -24,7 +24,8 @@ execute "create-postgres-database" do command lazy { %(psql -c "CREATE DATABASE \\"stockaid_production\\" WITH OWNER \\"stockaid\\" ENCODING 'unicode'") } user "postgres" - not_if %{psql postgres -tAc "SELECT 1 FROM pg_database WHERE LOWER(datname) = 'stockaid_production'" | grep 1}, user: "postgres" + not_if %{psql postgres -tAc "SELECT 1 FROM pg_database WHERE LOWER(datname) = 'stockaid_production'" | grep 1}, + user: "postgres" notifies :run, "execute[grant-postgres-database-to-user]", :immediately end diff --git a/chef/remote/cookbooks/stockaid/recipes/letsencrypt.rb b/chef/remote/cookbooks/stockaid/recipes/letsencrypt.rb index 93c25631..15c32015 100644 --- a/chef/remote/cookbooks/stockaid/recipes/letsencrypt.rb +++ b/chef/remote/cookbooks/stockaid/recipes/letsencrypt.rb @@ -7,7 +7,7 @@ owner "root" group "root" mode "0744" -atend +end file "/etc/cron.d/letsencrypt_renew" do content "# This file is managed by chef diff --git a/chef/remote/cookbooks/stockaid/recipes/nginx.rb b/chef/remote/cookbooks/stockaid/recipes/nginx.rb index 96a7c36f..239e1db2 100644 --- a/chef/remote/cookbooks/stockaid/recipes/nginx.rb +++ b/chef/remote/cookbooks/stockaid/recipes/nginx.rb @@ -11,13 +11,13 @@ command "apt-key adv --keyserver 'hkp://keyserver.ubuntu.com:80' --recv-keys '#{apt_key}'" not_if do - `apt-key finger`.split("\n").map do |line| + fingerprints = `apt-key finger`.split("\n").map do |line| line[/Key fingerprint = ([0-9A-F ]+)/, 1] - end.compact.map do |fingerprint| - fingerprint.split.join - end.any? do |fingerprint| - fingerprint.end_with? apt_key end + + fingerprints.compact! + fingerprints.map! { |fingerprint| fingerprint.split.join } + fingerprints.any? { |fingerprint| fingerprint.end_with? apt_key } end end diff --git a/chef/remote/cookbooks/stockaid/recipes/rails.rb b/chef/remote/cookbooks/stockaid/recipes/rails.rb index a5623b1c..5dcbf028 100644 --- a/chef/remote/cookbooks/stockaid/recipes/rails.rb +++ b/chef/remote/cookbooks/stockaid/recipes/rails.rb @@ -19,7 +19,8 @@ user node[:stockaid][:user] group node[:stockaid][:group] cwd node[:stockaid][:repo_dir] - not_if "#{rvm_binary} in '#{node[:stockaid][:repo_dir]}' do gem list | grep ^bundler\\\\s", user: node[:stockaid][:user], environment: rvm_environment + not_if "#{rvm_binary} in '#{node[:stockaid][:repo_dir]}' do gem list | grep ^bundler\\\\s", + user: node[:stockaid][:user], environment: rvm_environment end execute "bundle-install" do @@ -27,7 +28,8 @@ user node[:stockaid][:user] group node[:stockaid][:group] cwd node[:stockaid][:repo_dir] - not_if "#{rvm_binary} in '#{node[:stockaid][:repo_dir]}' do bundle check", user: node[:stockaid][:user], environment: rvm_environment + not_if "#{rvm_binary} in '#{node[:stockaid][:repo_dir]}' do bundle check", + user: node[:stockaid][:user], environment: rvm_environment notifies :run, "execute[reload-nginx]" end @@ -52,8 +54,6 @@ action :create_if_missing end -database_password_file = File.join(node[:stockaid][:dir], ".stockaid-db-password") - file File.join(node[:stockaid][:repo_dir], ".ruby-env") do content lazy { [].tap do |lines| @@ -83,5 +83,6 @@ user node[:stockaid][:user] group node[:stockaid][:group] cwd node[:stockaid][:repo_dir] - not_if "#{rvm_binary} in '#{node[:stockaid][:repo_dir]}' do bundle exec rake db:abort_if_pending_migrations", user: node[:stockaid][:user], environment: rvm_environment + not_if "#{rvm_binary} in '#{node[:stockaid][:repo_dir]}' do bundle exec rake db:abort_if_pending_migrations", + user: node[:stockaid][:user], environment: rvm_environment end From e97bb04dead19244bbb7e6851f6e8284ada4d88a Mon Sep 17 00:00:00 2001 From: Mike Virata-Stone Date: Sun, 21 May 2017 21:41:47 -0700 Subject: [PATCH 8/9] Ignore Chef JSON files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 276b1092..da1ea537 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ /log/* !/log/.keep /tmp + +# Ignore Chef JSON files +/chef/*.json From b5c2731c58776b24302cea5140b46b01b68bddbc Mon Sep 17 00:00:00 2001 From: Mike Virata-Stone Date: Tue, 30 May 2017 01:47:33 -0700 Subject: [PATCH 9/9] Auto update and log the updates --- .../stockaid/recipes/auto_updates.rb | 31 +++++++++++++++++++ .../cookbooks/stockaid/recipes/default.rb | 1 + 2 files changed, 32 insertions(+) create mode 100644 chef/remote/cookbooks/stockaid/recipes/auto_updates.rb diff --git a/chef/remote/cookbooks/stockaid/recipes/auto_updates.rb b/chef/remote/cookbooks/stockaid/recipes/auto_updates.rb new file mode 100644 index 00000000..64c57192 --- /dev/null +++ b/chef/remote/cookbooks/stockaid/recipes/auto_updates.rb @@ -0,0 +1,31 @@ +directory "/var/log" do + owner "root" + group "root" + mode "0775" + recursive true +end + +file "/etc/cron.weekly/apt-auto-updates" do + content %(# This file is managed by Chef +echo "**************" >> /var/log/apt-auto-updates.log +date >> /var/log/apt-auto-updates.log +apt-get update >> /var/log/apt-auto-updates.log +apt-get upgrade --assume-yes >> /var/log/apt-auto-updates.log +echo "Updates (if any) installed" +) + owner "root" + group "root" + mode "0755" +end + +file "/etc/logrotate.d/apt-auto-updates" do + content %(# This file is managed by Chef +/var/log/apt-auto-updates.log { + rotate 20 + weekly + size 500k + compress + notifempty +} +) +end diff --git a/chef/remote/cookbooks/stockaid/recipes/default.rb b/chef/remote/cookbooks/stockaid/recipes/default.rb index 69903229..efdbfa4f 100644 --- a/chef/remote/cookbooks/stockaid/recipes/default.rb +++ b/chef/remote/cookbooks/stockaid/recipes/default.rb @@ -6,3 +6,4 @@ include_recipe "stockaid::self_signed_ssl" include_recipe "stockaid::nginx" include_recipe "stockaid::letsencrypt" if node[:stockaid][:letsencrypt][:enabled] +include_recipe "stockaid::auto_updates"