diff --git a/Gemfile b/Gemfile index 1e602d0dc..96e9c29ab 100644 --- a/Gemfile +++ b/Gemfile @@ -15,8 +15,7 @@ gem "redis", "~> 5.4" gem "puma", "~> 8.0" # Jobs -gem "resque", "~> 2.7.0" -gem "resque-pool", "~> 0.7.1" +gem "solid_queue", "~> 1.7.2" # Assets gem "propshaft", github: "rails/propshaft" diff --git a/Gemfile.lock b/Gemfile.lock index 80909cb63..0359763ce 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -175,6 +175,8 @@ GEM drb (2.2.3) erb (6.0.7) erubi (1.13.1) + et-orbi (1.4.2) + tzinfo faker (3.8.0) i18n (>= 1.8.11, < 2) ffi (1.17.2) @@ -182,6 +184,9 @@ GEM ffi (1.17.2-arm64-darwin) ffi (1.17.2-x86_64-darwin) ffi (1.17.2-x86_64-linux-gnu) + fugit (1.14.0) + et-orbi (~> 1.4) + raabro (~> 1.4) geared_pagination (1.2.0) activesupport (>= 5.0) addressable (>= 2.5.0) @@ -237,10 +242,6 @@ GEM prism (~> 1.5) mocha (3.1.0) ruby2_keywords (>= 0.0.5) - mono_logger (1.1.2) - multi_json (1.17.0) - mustermann (3.0.4) - ruby2_keywords (~> 0.0.1) net-imap (0.6.7) date net-protocol @@ -278,12 +279,9 @@ GEM public_suffix (6.0.2) puma (8.0.3) nio4r (~> 2.0) + raabro (1.5.0) racc (1.8.1) rack (3.2.8) - rack-protection (4.2.1) - base64 (>= 0.1.0) - logger (>= 1.6.0) - rack (>= 3.0.0, < 4) rack-session (2.1.2) base64 (>= 0.1.0) rack (>= 3.0.0) @@ -318,19 +316,9 @@ GEM redis-client (>= 0.22.0) redis-client (0.30.1) connection_pool - redis-namespace (1.11.0) - redis (>= 4) regexp_parser (2.11.2) reline (0.7.0) io-console (~> 0.5) - resque (2.7.0) - mono_logger (~> 1) - multi_json (~> 1.0) - redis-namespace (~> 1.6) - sinatra (>= 0.9.2) - resque-pool (0.7.1) - rake (>= 10.0, < 14.0) - resque (>= 1.22, < 3) rexml (3.4.4) rqrcode (3.2.0) chunky_png (~> 1.0) @@ -383,13 +371,13 @@ GEM sentry-ruby (5.26.0) bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) - sinatra (4.2.1) - logger (>= 1.6.0) - mustermann (~> 3.0) - rack (>= 3.0.0, < 4) - rack-protection (= 4.2.1) - rack-session (>= 2.0.0, < 3) - tilt (~> 2.0) + solid_queue (1.7.2) + activejob (>= 7.1) + activerecord (>= 7.1) + concurrent-ruby (>= 1.3.1) + fugit (~> 1.11) + railties (>= 7.1) + thor (>= 1.3.1) sqlite3 (2.10.2-aarch64-linux-gnu) sqlite3 (2.10.2-arm64-darwin) sqlite3 (2.10.2-x86_64-darwin) @@ -402,7 +390,6 @@ GEM thruster (0.1.27-arm64-darwin) thruster (0.1.27-x86_64-darwin) thruster (0.1.27-x86_64-linux) - tilt (2.6.1) timeout (0.6.1) tsort (0.2.0) tzinfo (2.0.6) @@ -456,13 +443,12 @@ DEPENDENCIES rails! rails_autolink redis (~> 5.4) - resque (~> 2.7.0) - resque-pool (~> 0.7.1) rqrcode rubocop-rails-omakase selenium-webdriver sentry-rails sentry-ruby + solid_queue (~> 1.7.2) sqlite3 stimulus-rails surfguard! diff --git a/Procfile b/Procfile index 20e143048..07d876c77 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,3 @@ web: bundle exec thrust bin/start-app redis: redis-server config/redis.conf -workers: FORK_PER_JOB=false INTERVAL=0.1 bundle exec resque-pool +workers: bundle exec bin/jobs diff --git a/app/jobs/retry_interrupted_jobs_job.rb b/app/jobs/retry_interrupted_jobs_job.rb new file mode 100644 index 000000000..2c3304f38 --- /dev/null +++ b/app/jobs/retry_interrupted_jobs_job.rb @@ -0,0 +1,31 @@ +# Solid Queue fails the job a stopped or killed worker was running. Retry it where running it again is harmless. +class RetryInterruptedJobsJob < ApplicationJob + RETRYABLE = %w[ Room::DestroyJob ActiveStorage::PurgeJob ] + INTERRUPTIONS = %w[ + SolidQueue::Processes::ProcessExitError SolidQueue::Processes::ProcessPrunedError SolidQueue::Processes::ProcessMissingError + ] + MAX_RETRIES = 3 + + def perform + SolidQueue::FailedExecution.joins(:job).where(job: { class_name: RETRYABLE }).includes(:job).find_each do |execution| + retry_interrupted(execution) if execution.exception_class.in?(INTERRUPTIONS) + end + end + + private + # FailedExecution#retry resets the job's execution count, so it's counted here instead. + def retry_interrupted(execution) + job = execution.job + retries = job.arguments["executions"].to_i + return if retries >= MAX_RETRIES + + execution.with_lock do + job.arguments["executions"] = retries + 1 + job.save! + job.prepare_for_execution + execution.destroy! + end + rescue ActiveRecord::RecordNotFound + # Retried or discarded by hand in the meantime. + end +end diff --git a/bench/README.md b/bench/README.md index 820495245..cab9c007f 100644 --- a/bench/README.md +++ b/bench/README.md @@ -12,7 +12,8 @@ ruby bench/compare_http.rb --baseline PATH --seed SEED --paths sidebar,search -- Both drivers alternate before/after order and reset fixture storage for each run. Results default to ignored `tmp/rails-optimization/results/`; override with `--output PATH`. -Use `--image` to override `campfire-reference:app`, and `--cpus` to override server CPUs +Use `--image` to override `campfire-reference:app`, `--baseline-image` when the baseline needs +another bundle, and `--cpus` to override server CPUs `8-11`. HTTP clients use CPUs `12-15` by default (`--client-cpus`). `--help` lists options. The rendering probe checks exact response bodies, selected headers and unread payloads, diff --git a/bench/compare_http.rb b/bench/compare_http.rb index 519b016e8..9e3944c29 100644 --- a/bench/compare_http.rb +++ b/bench/compare_http.rb @@ -46,7 +46,7 @@ command.concat environment(RAILS_ENV: "production", SECRET_KEY_BASE: "isolated-benchmark-fixture-key", DISABLE_SSL: true, SKIP_TELEMETRY: true, RAILS_LOG_LEVEL: "fatal", WEB_CONCURRENCY: 1, JOB_CONCURRENCY: 1, RAILS_MAX_THREADS: 5, REDIS_URL: "redis://#{redis}:6379/0") - command.concat [ options[:image], "bundle", "exec", "puma", "-C", "config/puma.rb" ] + command.concat [ options[side == "before" ? :baseline_image : :image], "sh", "-c", "bin/rails db:prepare && exec bundle exec puma -C config/puma.rb" ] run(*command) deadline = clock + 45 until client.ready? diff --git a/bench/compare_message_hot_paths.rb b/bench/compare_message_hot_paths.rb index 225ee3e01..8825f6cf2 100644 --- a/bench/compare_message_hot_paths.rb +++ b/bench/compare_message_hot_paths.rb @@ -24,7 +24,7 @@ File.join(options[:seed], "labels.json") => "/bench-labels.json") command.concat environment(RAILS_ENV: "production", SECRET_KEY_BASE: "isolated-benchmark-fixture-key", DISABLE_SSL: true, SKIP_TELEMETRY: true, RAILS_LOG_LEVEL: "fatal", BENCH_LABELS: "/bench-labels.json") - command.concat [ options[:image], "bundle", "exec", "ruby", "-r", "/rails/config/environment.rb", "/bench/message_hot_paths.rb" ] + command.concat [ options[side == "before" ? :baseline_image : :image], "bundle", "exec", "ruby", "-r", "/rails/config/environment.rb", "/bench/message_hot_paths.rb" ] data = JSON.parse(run(*command)) runs[side] << data write_json(File.join(options[:output], "#{side}-#{iteration + 1}.json"), data) diff --git a/bench/support.rb b/bench/support.rb index c0c4089b7..8ff3478c4 100644 --- a/bench/support.rb +++ b/bench/support.rb @@ -8,7 +8,7 @@ module BenchmarkSupport WORK = File.join(ROOT, "tmp/rails-optimization") def parse_options(description, defaults) - options = { baseline: nil, seed: nil, image: "campfire-reference:app", cpus: "8-11", rounds: 2 }.merge(defaults) + options = { baseline: nil, seed: nil, image: "campfire-reference:app", baseline_image: nil, cpus: "8-11", rounds: 2 }.merge(defaults) parser = OptionParser.new do |parser| parser.banner = "Usage: ruby #{$PROGRAM_NAME} --baseline PATH --seed PATH [options]\n#{description}" options.each do |name, default| @@ -26,6 +26,7 @@ def parse_options(description, defaults) raise OptionParser::MissingArgument, "--baseline and --seed are required" unless options[:baseline] && options[:seed] raise OptionParser::InvalidArgument, "--rounds must be even and at least 2" unless options[:rounds] >= 2 && options[:rounds].even? %i[baseline seed output].each { |name| options[name] = File.expand_path(options.fetch(name)) } + options[:baseline_image] ||= options[:image] options rescue OptionParser::ParseError => error abort "#{error.message}\n#{parser}" diff --git a/bin/boot b/bin/boot index 5106115a7..c78c7a223 100755 --- a/bin/boot +++ b/bin/boot @@ -51,4 +51,5 @@ class MonitoredProcess end end +system "bin/rails db:prepare", exception: true ProcessMonitor.new("Procfile") diff --git a/bin/jobs b/bin/jobs new file mode 100755 index 000000000..dcf59f309 --- /dev/null +++ b/bin/jobs @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "solid_queue/cli" + +SolidQueue::Cli.start(ARGV) diff --git a/bin/start-app b/bin/start-app index 9a751c2f4..c1af71202 100755 --- a/bin/start-app +++ b/bin/start-app @@ -1,5 +1,4 @@ #!/bin/bash -e rm -f tmp/pids/server.pid -./bin/rails db:prepare ./bin/rails server diff --git a/config/bundler-audit.yml b/config/bundler-audit.yml index 5068d2f2d..e74b3af94 100644 --- a/config/bundler-audit.yml +++ b/config/bundler-audit.yml @@ -3,8 +3,3 @@ ignore: - CVE-THAT-DOES-NOT-APPLY - # resque-web advisories. Campfire doesn't mount resque-web. Remove when we move to resque 3. - - GHSA-5r7x-x5pm-94xj - - GHSA-m39q-9jm7-xp4q - - GHSA-r23w-h6cg-h95q - - GHSA-r5h4-xh93-f339 diff --git a/config/database.yml b/config/database.yml index 845eb70f2..07162a66c 100644 --- a/config/database.yml +++ b/config/database.yml @@ -27,13 +27,32 @@ test: primary: <<: *default database: storage/db/test.sqlite3 + queue: + <<: *default + database: storage/db/test_queue.sqlite3 + migrations_paths: db/queue_migrate performance: primary: <<: *default database: storage/db/performance.sqlite3 + queue: + <<: *default + database: storage/db/performance_queue.sqlite3 + migrations_paths: db/queue_migrate + pragmas: + wal_autocheckpoint: 1000 production: primary: <<: *default database: storage/db/production.sqlite3 + queue: + <<: *default + database: storage/db/production_queue.sqlite3 + # Three worker threads plus the poller and heartbeat, whatever the web server's thread count. + pool: 5 + migrations_paths: db/queue_migrate + # The checkpoint contender only checkpoints the primary database. + pragmas: + wal_autocheckpoint: 1000 diff --git a/config/environments/development.rb b/config/environments/development.rb index a25cb6297..67c2c3cbd 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -64,9 +64,6 @@ # Uncomment if you wish to allow Action Cable access from any origin. # config.action_cable.disable_request_forgery_protection = true - # Uncomment to test with production class job queue - # config.active_job.queue_adapter = :resque - # Highlight code that enqueued background job in logs. config.active_job.verbose_enqueue_logs = true diff --git a/config/environments/production.rb b/config/environments/production.rb index 369bb5bf0..017c72dbd 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -93,5 +93,6 @@ # Only use :id for inspections in production. config.active_record.attributes_for_inspect = [ :id ] - config.active_job.queue_adapter = :resque + config.active_job.queue_adapter = :solid_queue + config.solid_queue.connects_to = { database: { writing: :queue } } end diff --git a/config/environments/test.rb b/config/environments/test.rb index 70deabfdf..1393f4f6b 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -34,6 +34,9 @@ # Disable request forgery protection in test environment. config.action_controller.allow_forgery_protection = false + # Jobs run on the test adapter; the queue database is for tests of Solid Queue's own records. + config.solid_queue.connects_to = { database: { writing: :queue } } + # Store uploaded files on the local file system in a temporary directory. config.active_storage.service = :test diff --git a/config/initializers/resque.rb b/config/initializers/resque.rb deleted file mode 100644 index 48b9c7d4a..000000000 --- a/config/initializers/resque.rb +++ /dev/null @@ -1 +0,0 @@ -Resque.redis = ENV.fetch("REDIS_URL") { "redis://localhost:6379" } diff --git a/config/initializers/solid_queue.rb b/config/initializers/solid_queue.rb new file mode 100644 index 000000000..97c732d7e --- /dev/null +++ b/config/initializers/solid_queue.rb @@ -0,0 +1,3 @@ +# Stop before forking, as Puma does, so workers don't inherit the contender's flock. +SolidQueue.on_start { SqliteWalCheckpoint.stop } +SolidQueue.on_worker_start { SqliteWalCheckpoint.start } diff --git a/config/initializers/sqlite_wal_checkpoint.rb b/config/initializers/sqlite_wal_checkpoint.rb index aa14b8036..d858fcc1d 100644 --- a/config/initializers/sqlite_wal_checkpoint.rb +++ b/config/initializers/sqlite_wal_checkpoint.rb @@ -1,5 +1,5 @@ # Loaded from lib/rails_ext (autoload_lib ignore list) so reloads do not orphan -# the contender thread. Non-test processes start here; Puma/Resque stop before +# the contender thread. Non-test processes start here; Puma/Solid Queue stop before # fork and start again in the child. require Rails.root.join("lib/rails_ext/sqlite_wal_checkpoint") diff --git a/config/queue.yml b/config/queue.yml new file mode 100644 index 000000000..2b0212f33 --- /dev/null +++ b/config/queue.yml @@ -0,0 +1,21 @@ +default: &default + dispatchers: + - polling_interval: 1 + batch_size: 500 + workers: + - queues: "*" + threads: 3 + processes: <%= ENV.fetch("JOB_CONCURRENCY", 1) %> + polling_interval: 0.1 + +development: + <<: *default + +test: + <<: *default + +performance: + <<: *default + +production: + <<: *default diff --git a/config/recurring.yml b/config/recurring.yml new file mode 100644 index 000000000..ce3ff6083 --- /dev/null +++ b/config/recurring.yml @@ -0,0 +1,7 @@ +production: + clear_solid_queue_finished_jobs: + command: "SolidQueue::Job.clear_finished_in_batches(sleep_between_batches: 0.3)" + schedule: every hour at minute 12 + retry_interrupted_jobs: + class: RetryInterruptedJobsJob + schedule: every minute diff --git a/config/resque-pool.yml b/config/resque-pool.yml deleted file mode 100644 index 64f569bde..000000000 --- a/config/resque-pool.yml +++ /dev/null @@ -1 +0,0 @@ -default: <%= (Concurrent.processor_count * 0.5).ceil %> diff --git a/db/queue_schema.rb b/db/queue_schema.rb new file mode 100644 index 000000000..f9a71dabb --- /dev/null +++ b/db/queue_schema.rb @@ -0,0 +1,160 @@ +ActiveRecord::Schema[7.1].define(version: 1) do + create_table "solid_queue_blocked_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.string "concurrency_key", null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.index [ "concurrency_key", "priority", "job_id" ], name: "index_solid_queue_blocked_executions_for_release" + t.index [ "expires_at", "concurrency_key" ], name: "index_solid_queue_blocked_executions_for_maintenance" + t.index [ "job_id" ], name: "index_solid_queue_blocked_executions_on_job_id", unique: true + end + + create_table "solid_queue_claimed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.bigint "process_id" + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_claimed_executions_on_job_id", unique: true + t.index [ "process_id", "job_id" ], name: "index_solid_queue_claimed_executions_on_process_id_and_job_id" + end + + create_table "solid_queue_failed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.text "error" + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_failed_executions_on_job_id", unique: true + end + + create_table "solid_queue_jobs", force: :cascade do |t| + t.string "queue_name", null: false + t.string "class_name", null: false + t.text "arguments" + t.integer "priority", default: 0, null: false + t.string "active_job_id" + t.datetime "scheduled_at" + t.datetime "finished_at" + t.string "concurrency_key" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.bigint "batch_id" + t.index [ "active_job_id" ], name: "index_solid_queue_jobs_on_active_job_id" + t.index [ "batch_id" ], name: "index_solid_queue_jobs_on_batch_id" + t.index [ "class_name" ], name: "index_solid_queue_jobs_on_class_name" + t.index [ "finished_at" ], name: "index_solid_queue_jobs_on_finished_at" + t.index [ "queue_name", "finished_at" ], name: "index_solid_queue_jobs_for_filtering" + t.index [ "scheduled_at", "finished_at" ], name: "index_solid_queue_jobs_for_alerting" + end + + create_table "solid_queue_pauses", force: :cascade do |t| + t.string "queue_name", null: false + t.datetime "created_at", null: false + t.index [ "queue_name" ], name: "index_solid_queue_pauses_on_queue_name", unique: true + end + + create_table "solid_queue_processes", force: :cascade do |t| + t.string "kind", null: false + t.datetime "last_heartbeat_at", null: false + t.bigint "supervisor_id" + t.integer "pid", null: false + t.string "hostname" + t.text "metadata" + t.datetime "created_at", null: false + t.string "name", null: false + t.index [ "last_heartbeat_at" ], name: "index_solid_queue_processes_on_last_heartbeat_at" + t.index [ "name", "supervisor_id" ], name: "index_solid_queue_processes_on_name_and_supervisor_id", unique: true + t.index [ "supervisor_id" ], name: "index_solid_queue_processes_on_supervisor_id" + end + + create_table "solid_queue_ready_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_ready_executions_on_job_id", unique: true + t.index [ "priority", "job_id" ], name: "index_solid_queue_poll_all" + t.index [ "queue_name", "priority", "job_id" ], name: "index_solid_queue_poll_by_queue" + end + + create_table "solid_queue_recurring_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "task_key", null: false + t.datetime "run_at", null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_recurring_executions_on_job_id", unique: true + t.index [ "task_key", "run_at" ], name: "index_solid_queue_recurring_executions_on_task_key_and_run_at", unique: true + end + + create_table "solid_queue_recurring_tasks", force: :cascade do |t| + t.string "key", null: false + t.string "schedule", null: false + t.string "command", limit: 2048 + t.string "class_name" + t.text "arguments" + t.string "queue_name" + t.integer "priority", default: 0 + t.boolean "static", default: true, null: false + t.text "description" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index [ "key" ], name: "index_solid_queue_recurring_tasks_on_key", unique: true + t.index [ "static" ], name: "index_solid_queue_recurring_tasks_on_static" + end + + create_table "solid_queue_scheduled_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "scheduled_at", null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_scheduled_executions_on_job_id", unique: true + t.index [ "scheduled_at", "priority", "job_id" ], name: "index_solid_queue_dispatch_all" + end + + create_table "solid_queue_semaphores", force: :cascade do |t| + t.string "key", null: false + t.integer "value", default: 1, null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index [ "expires_at" ], name: "index_solid_queue_semaphores_on_expires_at" + t.index [ "key", "value" ], name: "index_solid_queue_semaphores_on_key_and_value" + t.index [ "key" ], name: "index_solid_queue_semaphores_on_key", unique: true + end + + create_table "solid_queue_batches", force: :cascade do |t| + t.string "active_job_batch_id" + t.string "description" + t.text "on_finish" + t.text "on_success" + t.text "on_failure" + t.text "metadata" + t.integer "total_jobs", default: 0, null: false + t.integer "completed_jobs", default: 0, null: false + t.integer "failed_jobs", default: 0, null: false + t.datetime "enqueued_at" + t.datetime "finished_at" + t.datetime "failed_at" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index [ "active_job_batch_id" ], name: "index_solid_queue_batches_on_active_job_batch_id", unique: true + t.index [ "finished_at" ], name: "index_solid_queue_batches_on_finished_at" + end + + create_table "solid_queue_batch_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.bigint "batch_id", null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_batch_executions_on_job_id", unique: true + t.index [ "batch_id" ], name: "index_solid_queue_batch_executions_on_batch_id" + end + + add_foreign_key "solid_queue_batch_executions", "solid_queue_batches", column: "batch_id", on_delete: :cascade + add_foreign_key "solid_queue_batch_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_ready_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_recurring_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_scheduled_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade +end diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 5398e0c5b..92de9bc87 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -196,7 +196,7 @@ docker run --rm \ This gives you a `campfire-backup.tar.gz` in your current directory containing the database snapshot and all uploaded files. Copy it somewhere safe, ideally off the machine. -To restore, extract the archive back into a (stopped) instance's volume, and replace the live database with the snapshot: +To restore, extract the archive back into a (stopped) instance's volume, and run the restore hook, which replaces the live database with the snapshot: ```sh docker run --rm \ @@ -205,8 +205,7 @@ docker run --rm \ --volume "$PWD":/backup \ ghcr.io/basecamp/once-campfire:latest \ bash -c "tar xzf /backup/campfire-backup.tar.gz -C /rails && - cp /rails/storage/backups/production.sqlite3 /rails/storage/db/production.sqlite3 && - rm -f /rails/storage/db/production.sqlite3-wal /rails/storage/db/production.sqlite3-shm && + /hooks/post-restore && chown -R rails:rails /rails/storage" ``` diff --git a/hooks/post-restore b/hooks/post-restore index 0b9138d1e..06bd3c22c 100755 --- a/hooks/post-restore +++ b/hooks/post-restore @@ -1,12 +1,13 @@ #!/bin/bash set -euo pipefail -db_name="${RAILS_ENV:-production}.sqlite3" -snapshot="/rails/storage/backups/$db_name" -db="/rails/storage/db/$db_name" +for db_name in "${RAILS_ENV:-production}.sqlite3" "${RAILS_ENV:-production}_queue.sqlite3"; do + snapshot="/rails/storage/backups/$db_name" + db="/rails/storage/db/$db_name" -if [ -f "$snapshot" ]; then - mkdir -p "$(dirname "$db")" - cp "$snapshot" "$db" - rm -f "$db-wal" "$db-shm" -fi + if [ -f "$snapshot" ]; then + mkdir -p "$(dirname "$db")" + cp "$snapshot" "$db" + rm -f "$db-wal" "$db-shm" + fi +done diff --git a/lib/rails_ext/sqlite_wal_checkpoint.rb b/lib/rails_ext/sqlite_wal_checkpoint.rb index 7b1e35bff..b5c58050c 100644 --- a/lib/rails_ext/sqlite_wal_checkpoint.rb +++ b/lib/rails_ext/sqlite_wal_checkpoint.rb @@ -2,7 +2,7 @@ # writer and fsyncs during the request. database.yml sets wal_autocheckpoint=0; # this module copies pages off the request thread with PASSIVE checkpoints. # -# Every non-test process starts a contender. Callers that fork (Puma, Resque pool) +# Every non-test process starts a contender. Callers that fork (Puma, Solid Queue) # must stop before fork and start again in the child so the flock is never shared # across an inherited file descriptor. module SqliteWalCheckpoint diff --git a/lib/tasks/resque.rake b/lib/tasks/resque.rake deleted file mode 100644 index 6e04c7470..000000000 --- a/lib/tasks/resque.rake +++ /dev/null @@ -1,16 +0,0 @@ -task "resque:setup" do - require_relative "../../config/environment" -end - -task "resque:pool:setup" do - ActiveRecord::Base.connection.disconnect! - # Initializer may have started a contender in the pool master; drop it before - # workers fork so they do not inherit the flock. - SqliteWalCheckpoint.stop - - Resque::Pool.after_prefork do |job| - ActiveRecord::Base.establish_connection - Resque.redis.client.close - SqliteWalCheckpoint.start - end -end diff --git a/script/admin/prepare-backup b/script/admin/prepare-backup index 7701124d3..c61b728e8 100755 --- a/script/admin/prepare-backup +++ b/script/admin/prepare-backup @@ -4,22 +4,22 @@ require File.expand_path("../../config/environment", File.dirname(__FILE__)) class Backup class << self def create - dest = SQLite3::Database.new(backup_filename) - backup = SQLite3::Backup.new(dest, "main", Room.connection.raw_connection, "main") - - backup.step(-1) - backup.finish - dest.close + snapshot SolidQueue::Record + snapshot ApplicationRecord end private - def backup_filename - Rails.root.join("storage", "backups").tap(&:mkpath).join(primary_database_filename) + def snapshot(model) + dest = SQLite3::Database.new(backup_filename(model)) + backup = SQLite3::Backup.new(dest, "main", model.connection.raw_connection, "main") + + backup.step(-1) + backup.finish + dest.close end - def primary_database_filename - path = Rails.application.config.database_configuration[Rails.env]["primary"]["database"] - File.basename(path) + def backup_filename(model) + Rails.root.join("storage", "backups").tap(&:mkpath).join(File.basename(model.connection_db_config.database)) end end end diff --git a/test/jobs/retry_interrupted_jobs_job_test.rb b/test/jobs/retry_interrupted_jobs_job_test.rb new file mode 100644 index 000000000..2d2a5b943 --- /dev/null +++ b/test/jobs/retry_interrupted_jobs_job_test.rb @@ -0,0 +1,33 @@ +require "test_helper" + +class RetryInterruptedJobsJobTest < ActiveSupport::TestCase + test "retries only interrupted jobs that are safe to run again, a few times" do + interrupted_destroy = failed Room::DestroyJob.new(rooms(:watercooler), []), interrupted + broken_destroy = failed Room::DestroyJob.new(rooms(:hq), []), RuntimeError.new("boom") + interrupted_webhook = failed Bot::WebhookJob.new(users(:bender), messages(:first)), interrupted + interrupted_removal = failed RemoveBannedContentJob.new(users(:kevin)), interrupted + exhausted_destroy = failed Room::DestroyJob.new(rooms(:pets), []), interrupted, executions: 3 + + RetryInterruptedJobsJob.perform_now + + assert interrupted_destroy.reload.ready? + assert_equal 1, interrupted_destroy.arguments["executions"] + assert_not broken_destroy.reload.ready? + assert_not interrupted_webhook.reload.ready? + assert_not interrupted_removal.reload.ready? + assert_not exhausted_destroy.reload.ready? + end + + private + def interrupted + SolidQueue::Processes::ProcessPrunedError.new(1.minute.ago) + end + + def failed(active_job, error, executions: 0) + active_job.executions = executions + SolidQueue::Job.enqueue(active_job).tap do |job| + job.ready_execution.destroy! + job.failed_with(error) + end + end +end diff --git a/test/script/admin/prepare_backup_test.rb b/test/script/admin/prepare_backup_test.rb new file mode 100644 index 000000000..ca3a7be9e --- /dev/null +++ b/test/script/admin/prepare_backup_test.rb @@ -0,0 +1,35 @@ +require "test_helper" + +class PrepareBackupScriptTest < ActiveSupport::TestCase + # SQLite can't back up a connection in the middle of a write transaction. + self.use_transactional_tests = false + + teardown do + SolidQueue::Job.delete_all + SolidQueue::Process.delete_all + backups.each { |path| FileUtils.rm_f(path) } + end + + test "backs up the database and its queue, with the jobs running at the time" do + running = SolidQueue::Job.enqueue(RemoveBannedContentJob.new(users(:kevin))) + worker = SolidQueue::Process.create!(kind: "Worker", name: "worker", pid: 1, hostname: "campfire", last_heartbeat_at: Time.current) + SolidQueue::ReadyExecution.claim("*", 1, worker.id) + queued = SolidQueue::Job.enqueue(Room::DestroyJob.new(rooms(:watercooler), [])) + + load Rails.root.join("script/admin/prepare-backup").to_s + + database, queue = backups.map { |path| SQLite3::Database.new(path.to_s) } + assert_equal [ [ rooms(:watercooler).id ] ], database.execute("SELECT id FROM rooms WHERE id = ?", rooms(:watercooler).id) + assert_equal [ running.id, queued.id ].sort, queue.execute("SELECT id FROM solid_queue_jobs").flatten.sort + assert_equal [ [ running.id, worker.id ] ], queue.execute("SELECT job_id, process_id FROM solid_queue_claimed_executions") + ensure + [ database, queue ].compact.each(&:close) + end + + private + def backups + [ ApplicationRecord, SolidQueue::Record ].map do |model| + Rails.root.join("storage/backups", File.basename(model.connection_db_config.database)) + end + end +end