From 272d7a52ed139aa6ffb4921ec9997d852195b49c Mon Sep 17 00:00:00 2001 From: Thomas Klemm Date: Thu, 8 Oct 2026 11:13:41 +0000 Subject: [PATCH 1/2] Replace Resque with Solid Queue Jobs move from Resque in the container's Redis to Solid Queue in its own SQLite database, so jobs queued when the container stops are still there when it starts again. Redis doesn't persist (save "", appendonly no), so jobs still queued in Resque at the upgrade are lost, as on any restart. The queue database is backed up and restored with the primary. A room deletion or banned user's content removal still queued at the backup would otherwise never run after a restore. The two databases are copied one after the other and the app writes them in separate commits, so a job enqueued while the backup runs can be missing from it: a room deleted in that moment comes back closed and without members, its messages left. A job running at the backup comes back claimed by its process, and fails once that process is pruned as dead. bin/boot prepares the databases before starting the web server and the workers, which now start together and both need the queue's tables. Solid Queue forks its workers from a supervisor, so the supervisor stops its checkpoint contender before forking and each worker starts its own, as Puma's workers do. Workers poll every 0.1 s, as Resque did. The queue's pool is five, for the worker's three threads plus its poller and heartbeat, independent of the web server's RAILS_MAX_THREADS. The test environment gets a queue database, so the backup test can use real Solid Queue records; jobs still run on the test adapter. The benchmark drivers take --baseline-image, since a baseline from before this change needs Resque's gems, which the current image no longer has, and prepare the databases before starting Puma, so a seed made before the queue database existed still boots. --- Gemfile | 3 +- Gemfile.lock | 42 ++--- Procfile | 2 +- bench/README.md | 3 +- bench/compare_http.rb | 2 +- bench/compare_message_hot_paths.rb | 2 +- bench/support.rb | 3 +- bin/boot | 1 + bin/jobs | 6 + bin/start-app | 1 - config/bundler-audit.yml | 5 - config/database.yml | 19 +++ config/environments/development.rb | 3 - config/environments/production.rb | 3 +- config/environments/test.rb | 3 + config/initializers/resque.rb | 1 - config/initializers/solid_queue.rb | 3 + config/initializers/sqlite_wal_checkpoint.rb | 2 +- config/queue.yml | 21 +++ config/recurring.yml | 4 + config/resque-pool.yml | 1 - db/queue_schema.rb | 160 +++++++++++++++++++ docs/self-hosting.md | 5 +- hooks/post-restore | 17 +- lib/rails_ext/sqlite_wal_checkpoint.rb | 2 +- lib/tasks/resque.rake | 16 -- script/admin/prepare-backup | 22 +-- test/script/admin/prepare_backup_test.rb | 35 ++++ 28 files changed, 300 insertions(+), 87 deletions(-) create mode 100755 bin/jobs delete mode 100644 config/initializers/resque.rb create mode 100644 config/initializers/solid_queue.rb create mode 100644 config/queue.yml create mode 100644 config/recurring.yml delete mode 100644 config/resque-pool.yml create mode 100644 db/queue_schema.rb delete mode 100644 lib/tasks/resque.rake create mode 100644 test/script/admin/prepare_backup_test.rb diff --git a/Gemfile b/Gemfile index 1e602d0dc..96e9c29ab 100644 --- a/Gemfile +++ b/Gemfile @@ -15,8 +15,7 @@ gem "redis", "~> 5.4" gem "puma", "~> 8.0" # Jobs -gem "resque", "~> 2.7.0" -gem "resque-pool", "~> 0.7.1" +gem "solid_queue", "~> 1.7.2" # Assets gem "propshaft", github: "rails/propshaft" diff --git a/Gemfile.lock b/Gemfile.lock index 80909cb63..0359763ce 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -175,6 +175,8 @@ GEM drb (2.2.3) erb (6.0.7) erubi (1.13.1) + et-orbi (1.4.2) + tzinfo faker (3.8.0) i18n (>= 1.8.11, < 2) ffi (1.17.2) @@ -182,6 +184,9 @@ GEM ffi (1.17.2-arm64-darwin) ffi (1.17.2-x86_64-darwin) ffi (1.17.2-x86_64-linux-gnu) + fugit (1.14.0) + et-orbi (~> 1.4) + raabro (~> 1.4) geared_pagination (1.2.0) activesupport (>= 5.0) addressable (>= 2.5.0) @@ -237,10 +242,6 @@ GEM prism (~> 1.5) mocha (3.1.0) ruby2_keywords (>= 0.0.5) - mono_logger (1.1.2) - multi_json (1.17.0) - mustermann (3.0.4) - ruby2_keywords (~> 0.0.1) net-imap (0.6.7) date net-protocol @@ -278,12 +279,9 @@ GEM public_suffix (6.0.2) puma (8.0.3) nio4r (~> 2.0) + raabro (1.5.0) racc (1.8.1) rack (3.2.8) - rack-protection (4.2.1) - base64 (>= 0.1.0) - logger (>= 1.6.0) - rack (>= 3.0.0, < 4) rack-session (2.1.2) base64 (>= 0.1.0) rack (>= 3.0.0) @@ -318,19 +316,9 @@ GEM redis-client (>= 0.22.0) redis-client (0.30.1) connection_pool - redis-namespace (1.11.0) - redis (>= 4) regexp_parser (2.11.2) reline (0.7.0) io-console (~> 0.5) - resque (2.7.0) - mono_logger (~> 1) - multi_json (~> 1.0) - redis-namespace (~> 1.6) - sinatra (>= 0.9.2) - resque-pool (0.7.1) - rake (>= 10.0, < 14.0) - resque (>= 1.22, < 3) rexml (3.4.4) rqrcode (3.2.0) chunky_png (~> 1.0) @@ -383,13 +371,13 @@ GEM sentry-ruby (5.26.0) bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) - sinatra (4.2.1) - logger (>= 1.6.0) - mustermann (~> 3.0) - rack (>= 3.0.0, < 4) - rack-protection (= 4.2.1) - rack-session (>= 2.0.0, < 3) - tilt (~> 2.0) + solid_queue (1.7.2) + activejob (>= 7.1) + activerecord (>= 7.1) + concurrent-ruby (>= 1.3.1) + fugit (~> 1.11) + railties (>= 7.1) + thor (>= 1.3.1) sqlite3 (2.10.2-aarch64-linux-gnu) sqlite3 (2.10.2-arm64-darwin) sqlite3 (2.10.2-x86_64-darwin) @@ -402,7 +390,6 @@ GEM thruster (0.1.27-arm64-darwin) thruster (0.1.27-x86_64-darwin) thruster (0.1.27-x86_64-linux) - tilt (2.6.1) timeout (0.6.1) tsort (0.2.0) tzinfo (2.0.6) @@ -456,13 +443,12 @@ DEPENDENCIES rails! rails_autolink redis (~> 5.4) - resque (~> 2.7.0) - resque-pool (~> 0.7.1) rqrcode rubocop-rails-omakase selenium-webdriver sentry-rails sentry-ruby + solid_queue (~> 1.7.2) sqlite3 stimulus-rails surfguard! diff --git a/Procfile b/Procfile index 20e143048..07d876c77 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,3 @@ web: bundle exec thrust bin/start-app redis: redis-server config/redis.conf -workers: FORK_PER_JOB=false INTERVAL=0.1 bundle exec resque-pool +workers: bundle exec bin/jobs diff --git a/bench/README.md b/bench/README.md index 820495245..cab9c007f 100644 --- a/bench/README.md +++ b/bench/README.md @@ -12,7 +12,8 @@ ruby bench/compare_http.rb --baseline PATH --seed SEED --paths sidebar,search -- Both drivers alternate before/after order and reset fixture storage for each run. Results default to ignored `tmp/rails-optimization/results/`; override with `--output PATH`. -Use `--image` to override `campfire-reference:app`, and `--cpus` to override server CPUs +Use `--image` to override `campfire-reference:app`, `--baseline-image` when the baseline needs +another bundle, and `--cpus` to override server CPUs `8-11`. HTTP clients use CPUs `12-15` by default (`--client-cpus`). `--help` lists options. The rendering probe checks exact response bodies, selected headers and unread payloads, diff --git a/bench/compare_http.rb b/bench/compare_http.rb index 519b016e8..9e3944c29 100644 --- a/bench/compare_http.rb +++ b/bench/compare_http.rb @@ -46,7 +46,7 @@ command.concat environment(RAILS_ENV: "production", SECRET_KEY_BASE: "isolated-benchmark-fixture-key", DISABLE_SSL: true, SKIP_TELEMETRY: true, RAILS_LOG_LEVEL: "fatal", WEB_CONCURRENCY: 1, JOB_CONCURRENCY: 1, RAILS_MAX_THREADS: 5, REDIS_URL: "redis://#{redis}:6379/0") - command.concat [ options[:image], "bundle", "exec", "puma", "-C", "config/puma.rb" ] + command.concat [ options[side == "before" ? :baseline_image : :image], "sh", "-c", "bin/rails db:prepare && exec bundle exec puma -C config/puma.rb" ] run(*command) deadline = clock + 45 until client.ready? diff --git a/bench/compare_message_hot_paths.rb b/bench/compare_message_hot_paths.rb index 225ee3e01..8825f6cf2 100644 --- a/bench/compare_message_hot_paths.rb +++ b/bench/compare_message_hot_paths.rb @@ -24,7 +24,7 @@ File.join(options[:seed], "labels.json") => "/bench-labels.json") command.concat environment(RAILS_ENV: "production", SECRET_KEY_BASE: "isolated-benchmark-fixture-key", DISABLE_SSL: true, SKIP_TELEMETRY: true, RAILS_LOG_LEVEL: "fatal", BENCH_LABELS: "/bench-labels.json") - command.concat [ options[:image], "bundle", "exec", "ruby", "-r", "/rails/config/environment.rb", "/bench/message_hot_paths.rb" ] + command.concat [ options[side == "before" ? :baseline_image : :image], "bundle", "exec", "ruby", "-r", "/rails/config/environment.rb", "/bench/message_hot_paths.rb" ] data = JSON.parse(run(*command)) runs[side] << data write_json(File.join(options[:output], "#{side}-#{iteration + 1}.json"), data) diff --git a/bench/support.rb b/bench/support.rb index c0c4089b7..8ff3478c4 100644 --- a/bench/support.rb +++ b/bench/support.rb @@ -8,7 +8,7 @@ module BenchmarkSupport WORK = File.join(ROOT, "tmp/rails-optimization") def parse_options(description, defaults) - options = { baseline: nil, seed: nil, image: "campfire-reference:app", cpus: "8-11", rounds: 2 }.merge(defaults) + options = { baseline: nil, seed: nil, image: "campfire-reference:app", baseline_image: nil, cpus: "8-11", rounds: 2 }.merge(defaults) parser = OptionParser.new do |parser| parser.banner = "Usage: ruby #{$PROGRAM_NAME} --baseline PATH --seed PATH [options]\n#{description}" options.each do |name, default| @@ -26,6 +26,7 @@ def parse_options(description, defaults) raise OptionParser::MissingArgument, "--baseline and --seed are required" unless options[:baseline] && options[:seed] raise OptionParser::InvalidArgument, "--rounds must be even and at least 2" unless options[:rounds] >= 2 && options[:rounds].even? %i[baseline seed output].each { |name| options[name] = File.expand_path(options.fetch(name)) } + options[:baseline_image] ||= options[:image] options rescue OptionParser::ParseError => error abort "#{error.message}\n#{parser}" diff --git a/bin/boot b/bin/boot index 5106115a7..c78c7a223 100755 --- a/bin/boot +++ b/bin/boot @@ -51,4 +51,5 @@ class MonitoredProcess end end +system "bin/rails db:prepare", exception: true ProcessMonitor.new("Procfile") diff --git a/bin/jobs b/bin/jobs new file mode 100755 index 000000000..dcf59f309 --- /dev/null +++ b/bin/jobs @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "solid_queue/cli" + +SolidQueue::Cli.start(ARGV) diff --git a/bin/start-app b/bin/start-app index 9a751c2f4..c1af71202 100755 --- a/bin/start-app +++ b/bin/start-app @@ -1,5 +1,4 @@ #!/bin/bash -e rm -f tmp/pids/server.pid -./bin/rails db:prepare ./bin/rails server diff --git a/config/bundler-audit.yml b/config/bundler-audit.yml index 5068d2f2d..e74b3af94 100644 --- a/config/bundler-audit.yml +++ b/config/bundler-audit.yml @@ -3,8 +3,3 @@ ignore: - CVE-THAT-DOES-NOT-APPLY - # resque-web advisories. Campfire doesn't mount resque-web. Remove when we move to resque 3. - - GHSA-5r7x-x5pm-94xj - - GHSA-m39q-9jm7-xp4q - - GHSA-r23w-h6cg-h95q - - GHSA-r5h4-xh93-f339 diff --git a/config/database.yml b/config/database.yml index 845eb70f2..07162a66c 100644 --- a/config/database.yml +++ b/config/database.yml @@ -27,13 +27,32 @@ test: primary: <<: *default database: storage/db/test.sqlite3 + queue: + <<: *default + database: storage/db/test_queue.sqlite3 + migrations_paths: db/queue_migrate performance: primary: <<: *default database: storage/db/performance.sqlite3 + queue: + <<: *default + database: storage/db/performance_queue.sqlite3 + migrations_paths: db/queue_migrate + pragmas: + wal_autocheckpoint: 1000 production: primary: <<: *default database: storage/db/production.sqlite3 + queue: + <<: *default + database: storage/db/production_queue.sqlite3 + # Three worker threads plus the poller and heartbeat, whatever the web server's thread count. + pool: 5 + migrations_paths: db/queue_migrate + # The checkpoint contender only checkpoints the primary database. + pragmas: + wal_autocheckpoint: 1000 diff --git a/config/environments/development.rb b/config/environments/development.rb index a25cb6297..67c2c3cbd 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -64,9 +64,6 @@ # Uncomment if you wish to allow Action Cable access from any origin. # config.action_cable.disable_request_forgery_protection = true - # Uncomment to test with production class job queue - # config.active_job.queue_adapter = :resque - # Highlight code that enqueued background job in logs. config.active_job.verbose_enqueue_logs = true diff --git a/config/environments/production.rb b/config/environments/production.rb index 369bb5bf0..017c72dbd 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -93,5 +93,6 @@ # Only use :id for inspections in production. config.active_record.attributes_for_inspect = [ :id ] - config.active_job.queue_adapter = :resque + config.active_job.queue_adapter = :solid_queue + config.solid_queue.connects_to = { database: { writing: :queue } } end diff --git a/config/environments/test.rb b/config/environments/test.rb index 70deabfdf..1393f4f6b 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -34,6 +34,9 @@ # Disable request forgery protection in test environment. config.action_controller.allow_forgery_protection = false + # Jobs run on the test adapter; the queue database is for tests of Solid Queue's own records. + config.solid_queue.connects_to = { database: { writing: :queue } } + # Store uploaded files on the local file system in a temporary directory. config.active_storage.service = :test diff --git a/config/initializers/resque.rb b/config/initializers/resque.rb deleted file mode 100644 index 48b9c7d4a..000000000 --- a/config/initializers/resque.rb +++ /dev/null @@ -1 +0,0 @@ -Resque.redis = ENV.fetch("REDIS_URL") { "redis://localhost:6379" } diff --git a/config/initializers/solid_queue.rb b/config/initializers/solid_queue.rb new file mode 100644 index 000000000..97c732d7e --- /dev/null +++ b/config/initializers/solid_queue.rb @@ -0,0 +1,3 @@ +# Stop before forking, as Puma does, so workers don't inherit the contender's flock. +SolidQueue.on_start { SqliteWalCheckpoint.stop } +SolidQueue.on_worker_start { SqliteWalCheckpoint.start } diff --git a/config/initializers/sqlite_wal_checkpoint.rb b/config/initializers/sqlite_wal_checkpoint.rb index aa14b8036..d858fcc1d 100644 --- a/config/initializers/sqlite_wal_checkpoint.rb +++ b/config/initializers/sqlite_wal_checkpoint.rb @@ -1,5 +1,5 @@ # Loaded from lib/rails_ext (autoload_lib ignore list) so reloads do not orphan -# the contender thread. Non-test processes start here; Puma/Resque stop before +# the contender thread. Non-test processes start here; Puma/Solid Queue stop before # fork and start again in the child. require Rails.root.join("lib/rails_ext/sqlite_wal_checkpoint") diff --git a/config/queue.yml b/config/queue.yml new file mode 100644 index 000000000..2b0212f33 --- /dev/null +++ b/config/queue.yml @@ -0,0 +1,21 @@ +default: &default + dispatchers: + - polling_interval: 1 + batch_size: 500 + workers: + - queues: "*" + threads: 3 + processes: <%= ENV.fetch("JOB_CONCURRENCY", 1) %> + polling_interval: 0.1 + +development: + <<: *default + +test: + <<: *default + +performance: + <<: *default + +production: + <<: *default diff --git a/config/recurring.yml b/config/recurring.yml new file mode 100644 index 000000000..854f415b5 --- /dev/null +++ b/config/recurring.yml @@ -0,0 +1,4 @@ +production: + clear_solid_queue_finished_jobs: + command: "SolidQueue::Job.clear_finished_in_batches(sleep_between_batches: 0.3)" + schedule: every hour at minute 12 diff --git a/config/resque-pool.yml b/config/resque-pool.yml deleted file mode 100644 index 64f569bde..000000000 --- a/config/resque-pool.yml +++ /dev/null @@ -1 +0,0 @@ -default: <%= (Concurrent.processor_count * 0.5).ceil %> diff --git a/db/queue_schema.rb b/db/queue_schema.rb new file mode 100644 index 000000000..f9a71dabb --- /dev/null +++ b/db/queue_schema.rb @@ -0,0 +1,160 @@ +ActiveRecord::Schema[7.1].define(version: 1) do + create_table "solid_queue_blocked_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.string "concurrency_key", null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.index [ "concurrency_key", "priority", "job_id" ], name: "index_solid_queue_blocked_executions_for_release" + t.index [ "expires_at", "concurrency_key" ], name: "index_solid_queue_blocked_executions_for_maintenance" + t.index [ "job_id" ], name: "index_solid_queue_blocked_executions_on_job_id", unique: true + end + + create_table "solid_queue_claimed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.bigint "process_id" + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_claimed_executions_on_job_id", unique: true + t.index [ "process_id", "job_id" ], name: "index_solid_queue_claimed_executions_on_process_id_and_job_id" + end + + create_table "solid_queue_failed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.text "error" + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_failed_executions_on_job_id", unique: true + end + + create_table "solid_queue_jobs", force: :cascade do |t| + t.string "queue_name", null: false + t.string "class_name", null: false + t.text "arguments" + t.integer "priority", default: 0, null: false + t.string "active_job_id" + t.datetime "scheduled_at" + t.datetime "finished_at" + t.string "concurrency_key" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.bigint "batch_id" + t.index [ "active_job_id" ], name: "index_solid_queue_jobs_on_active_job_id" + t.index [ "batch_id" ], name: "index_solid_queue_jobs_on_batch_id" + t.index [ "class_name" ], name: "index_solid_queue_jobs_on_class_name" + t.index [ "finished_at" ], name: "index_solid_queue_jobs_on_finished_at" + t.index [ "queue_name", "finished_at" ], name: "index_solid_queue_jobs_for_filtering" + t.index [ "scheduled_at", "finished_at" ], name: "index_solid_queue_jobs_for_alerting" + end + + create_table "solid_queue_pauses", force: :cascade do |t| + t.string "queue_name", null: false + t.datetime "created_at", null: false + t.index [ "queue_name" ], name: "index_solid_queue_pauses_on_queue_name", unique: true + end + + create_table "solid_queue_processes", force: :cascade do |t| + t.string "kind", null: false + t.datetime "last_heartbeat_at", null: false + t.bigint "supervisor_id" + t.integer "pid", null: false + t.string "hostname" + t.text "metadata" + t.datetime "created_at", null: false + t.string "name", null: false + t.index [ "last_heartbeat_at" ], name: "index_solid_queue_processes_on_last_heartbeat_at" + t.index [ "name", "supervisor_id" ], name: "index_solid_queue_processes_on_name_and_supervisor_id", unique: true + t.index [ "supervisor_id" ], name: "index_solid_queue_processes_on_supervisor_id" + end + + create_table "solid_queue_ready_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_ready_executions_on_job_id", unique: true + t.index [ "priority", "job_id" ], name: "index_solid_queue_poll_all" + t.index [ "queue_name", "priority", "job_id" ], name: "index_solid_queue_poll_by_queue" + end + + create_table "solid_queue_recurring_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "task_key", null: false + t.datetime "run_at", null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_recurring_executions_on_job_id", unique: true + t.index [ "task_key", "run_at" ], name: "index_solid_queue_recurring_executions_on_task_key_and_run_at", unique: true + end + + create_table "solid_queue_recurring_tasks", force: :cascade do |t| + t.string "key", null: false + t.string "schedule", null: false + t.string "command", limit: 2048 + t.string "class_name" + t.text "arguments" + t.string "queue_name" + t.integer "priority", default: 0 + t.boolean "static", default: true, null: false + t.text "description" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index [ "key" ], name: "index_solid_queue_recurring_tasks_on_key", unique: true + t.index [ "static" ], name: "index_solid_queue_recurring_tasks_on_static" + end + + create_table "solid_queue_scheduled_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "scheduled_at", null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_scheduled_executions_on_job_id", unique: true + t.index [ "scheduled_at", "priority", "job_id" ], name: "index_solid_queue_dispatch_all" + end + + create_table "solid_queue_semaphores", force: :cascade do |t| + t.string "key", null: false + t.integer "value", default: 1, null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index [ "expires_at" ], name: "index_solid_queue_semaphores_on_expires_at" + t.index [ "key", "value" ], name: "index_solid_queue_semaphores_on_key_and_value" + t.index [ "key" ], name: "index_solid_queue_semaphores_on_key", unique: true + end + + create_table "solid_queue_batches", force: :cascade do |t| + t.string "active_job_batch_id" + t.string "description" + t.text "on_finish" + t.text "on_success" + t.text "on_failure" + t.text "metadata" + t.integer "total_jobs", default: 0, null: false + t.integer "completed_jobs", default: 0, null: false + t.integer "failed_jobs", default: 0, null: false + t.datetime "enqueued_at" + t.datetime "finished_at" + t.datetime "failed_at" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index [ "active_job_batch_id" ], name: "index_solid_queue_batches_on_active_job_batch_id", unique: true + t.index [ "finished_at" ], name: "index_solid_queue_batches_on_finished_at" + end + + create_table "solid_queue_batch_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.bigint "batch_id", null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_batch_executions_on_job_id", unique: true + t.index [ "batch_id" ], name: "index_solid_queue_batch_executions_on_batch_id" + end + + add_foreign_key "solid_queue_batch_executions", "solid_queue_batches", column: "batch_id", on_delete: :cascade + add_foreign_key "solid_queue_batch_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_ready_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_recurring_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_scheduled_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade +end diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 5398e0c5b..92de9bc87 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -196,7 +196,7 @@ docker run --rm \ This gives you a `campfire-backup.tar.gz` in your current directory containing the database snapshot and all uploaded files. Copy it somewhere safe, ideally off the machine. -To restore, extract the archive back into a (stopped) instance's volume, and replace the live database with the snapshot: +To restore, extract the archive back into a (stopped) instance's volume, and run the restore hook, which replaces the live database with the snapshot: ```sh docker run --rm \ @@ -205,8 +205,7 @@ docker run --rm \ --volume "$PWD":/backup \ ghcr.io/basecamp/once-campfire:latest \ bash -c "tar xzf /backup/campfire-backup.tar.gz -C /rails && - cp /rails/storage/backups/production.sqlite3 /rails/storage/db/production.sqlite3 && - rm -f /rails/storage/db/production.sqlite3-wal /rails/storage/db/production.sqlite3-shm && + /hooks/post-restore && chown -R rails:rails /rails/storage" ``` diff --git a/hooks/post-restore b/hooks/post-restore index 0b9138d1e..06bd3c22c 100755 --- a/hooks/post-restore +++ b/hooks/post-restore @@ -1,12 +1,13 @@ #!/bin/bash set -euo pipefail -db_name="${RAILS_ENV:-production}.sqlite3" -snapshot="/rails/storage/backups/$db_name" -db="/rails/storage/db/$db_name" +for db_name in "${RAILS_ENV:-production}.sqlite3" "${RAILS_ENV:-production}_queue.sqlite3"; do + snapshot="/rails/storage/backups/$db_name" + db="/rails/storage/db/$db_name" -if [ -f "$snapshot" ]; then - mkdir -p "$(dirname "$db")" - cp "$snapshot" "$db" - rm -f "$db-wal" "$db-shm" -fi + if [ -f "$snapshot" ]; then + mkdir -p "$(dirname "$db")" + cp "$snapshot" "$db" + rm -f "$db-wal" "$db-shm" + fi +done diff --git a/lib/rails_ext/sqlite_wal_checkpoint.rb b/lib/rails_ext/sqlite_wal_checkpoint.rb index 7b1e35bff..b5c58050c 100644 --- a/lib/rails_ext/sqlite_wal_checkpoint.rb +++ b/lib/rails_ext/sqlite_wal_checkpoint.rb @@ -2,7 +2,7 @@ # writer and fsyncs during the request. database.yml sets wal_autocheckpoint=0; # this module copies pages off the request thread with PASSIVE checkpoints. # -# Every non-test process starts a contender. Callers that fork (Puma, Resque pool) +# Every non-test process starts a contender. Callers that fork (Puma, Solid Queue) # must stop before fork and start again in the child so the flock is never shared # across an inherited file descriptor. module SqliteWalCheckpoint diff --git a/lib/tasks/resque.rake b/lib/tasks/resque.rake deleted file mode 100644 index 6e04c7470..000000000 --- a/lib/tasks/resque.rake +++ /dev/null @@ -1,16 +0,0 @@ -task "resque:setup" do - require_relative "../../config/environment" -end - -task "resque:pool:setup" do - ActiveRecord::Base.connection.disconnect! - # Initializer may have started a contender in the pool master; drop it before - # workers fork so they do not inherit the flock. - SqliteWalCheckpoint.stop - - Resque::Pool.after_prefork do |job| - ActiveRecord::Base.establish_connection - Resque.redis.client.close - SqliteWalCheckpoint.start - end -end diff --git a/script/admin/prepare-backup b/script/admin/prepare-backup index 7701124d3..c61b728e8 100755 --- a/script/admin/prepare-backup +++ b/script/admin/prepare-backup @@ -4,22 +4,22 @@ require File.expand_path("../../config/environment", File.dirname(__FILE__)) class Backup class << self def create - dest = SQLite3::Database.new(backup_filename) - backup = SQLite3::Backup.new(dest, "main", Room.connection.raw_connection, "main") - - backup.step(-1) - backup.finish - dest.close + snapshot SolidQueue::Record + snapshot ApplicationRecord end private - def backup_filename - Rails.root.join("storage", "backups").tap(&:mkpath).join(primary_database_filename) + def snapshot(model) + dest = SQLite3::Database.new(backup_filename(model)) + backup = SQLite3::Backup.new(dest, "main", model.connection.raw_connection, "main") + + backup.step(-1) + backup.finish + dest.close end - def primary_database_filename - path = Rails.application.config.database_configuration[Rails.env]["primary"]["database"] - File.basename(path) + def backup_filename(model) + Rails.root.join("storage", "backups").tap(&:mkpath).join(File.basename(model.connection_db_config.database)) end end end diff --git a/test/script/admin/prepare_backup_test.rb b/test/script/admin/prepare_backup_test.rb new file mode 100644 index 000000000..ca3a7be9e --- /dev/null +++ b/test/script/admin/prepare_backup_test.rb @@ -0,0 +1,35 @@ +require "test_helper" + +class PrepareBackupScriptTest < ActiveSupport::TestCase + # SQLite can't back up a connection in the middle of a write transaction. + self.use_transactional_tests = false + + teardown do + SolidQueue::Job.delete_all + SolidQueue::Process.delete_all + backups.each { |path| FileUtils.rm_f(path) } + end + + test "backs up the database and its queue, with the jobs running at the time" do + running = SolidQueue::Job.enqueue(RemoveBannedContentJob.new(users(:kevin))) + worker = SolidQueue::Process.create!(kind: "Worker", name: "worker", pid: 1, hostname: "campfire", last_heartbeat_at: Time.current) + SolidQueue::ReadyExecution.claim("*", 1, worker.id) + queued = SolidQueue::Job.enqueue(Room::DestroyJob.new(rooms(:watercooler), [])) + + load Rails.root.join("script/admin/prepare-backup").to_s + + database, queue = backups.map { |path| SQLite3::Database.new(path.to_s) } + assert_equal [ [ rooms(:watercooler).id ] ], database.execute("SELECT id FROM rooms WHERE id = ?", rooms(:watercooler).id) + assert_equal [ running.id, queued.id ].sort, queue.execute("SELECT id FROM solid_queue_jobs").flatten.sort + assert_equal [ [ running.id, worker.id ] ], queue.execute("SELECT job_id, process_id FROM solid_queue_claimed_executions") + ensure + [ database, queue ].compact.each(&:close) + end + + private + def backups + [ ApplicationRecord, SolidQueue::Record ].map do |model| + Rails.root.join("storage/backups", File.basename(model.connection_db_config.database)) + end + end +end From c2a5062dd224fdf1ad6901acce09ec60d9d3e4a6 Mon Sep 17 00:00:00 2001 From: Thomas Klemm Date: Mon, 26 Oct 2026 15:01:04 +0000 Subject: [PATCH 2/2] Retry interrupted jobs that are safe to run again Solid Queue fails the job a worker was still running when the container stops or is killed, or when a restore brings back a job that was running at the backup. Every minute, retry such a job if running it again from the start does no harm: room deletion and file purges. Webhooks, Web Push, broadcasts and banned-content removal stay failed. Give up after three retries. --- app/jobs/retry_interrupted_jobs_job.rb | 31 ++++++++++++++++++ config/recurring.yml | 3 ++ test/jobs/retry_interrupted_jobs_job_test.rb | 33 ++++++++++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 app/jobs/retry_interrupted_jobs_job.rb create mode 100644 test/jobs/retry_interrupted_jobs_job_test.rb diff --git a/app/jobs/retry_interrupted_jobs_job.rb b/app/jobs/retry_interrupted_jobs_job.rb new file mode 100644 index 000000000..2c3304f38 --- /dev/null +++ b/app/jobs/retry_interrupted_jobs_job.rb @@ -0,0 +1,31 @@ +# Solid Queue fails the job a stopped or killed worker was running. Retry it where running it again is harmless. +class RetryInterruptedJobsJob < ApplicationJob + RETRYABLE = %w[ Room::DestroyJob ActiveStorage::PurgeJob ] + INTERRUPTIONS = %w[ + SolidQueue::Processes::ProcessExitError SolidQueue::Processes::ProcessPrunedError SolidQueue::Processes::ProcessMissingError + ] + MAX_RETRIES = 3 + + def perform + SolidQueue::FailedExecution.joins(:job).where(job: { class_name: RETRYABLE }).includes(:job).find_each do |execution| + retry_interrupted(execution) if execution.exception_class.in?(INTERRUPTIONS) + end + end + + private + # FailedExecution#retry resets the job's execution count, so it's counted here instead. + def retry_interrupted(execution) + job = execution.job + retries = job.arguments["executions"].to_i + return if retries >= MAX_RETRIES + + execution.with_lock do + job.arguments["executions"] = retries + 1 + job.save! + job.prepare_for_execution + execution.destroy! + end + rescue ActiveRecord::RecordNotFound + # Retried or discarded by hand in the meantime. + end +end diff --git a/config/recurring.yml b/config/recurring.yml index 854f415b5..ce3ff6083 100644 --- a/config/recurring.yml +++ b/config/recurring.yml @@ -2,3 +2,6 @@ production: clear_solid_queue_finished_jobs: command: "SolidQueue::Job.clear_finished_in_batches(sleep_between_batches: 0.3)" schedule: every hour at minute 12 + retry_interrupted_jobs: + class: RetryInterruptedJobsJob + schedule: every minute diff --git a/test/jobs/retry_interrupted_jobs_job_test.rb b/test/jobs/retry_interrupted_jobs_job_test.rb new file mode 100644 index 000000000..2d2a5b943 --- /dev/null +++ b/test/jobs/retry_interrupted_jobs_job_test.rb @@ -0,0 +1,33 @@ +require "test_helper" + +class RetryInterruptedJobsJobTest < ActiveSupport::TestCase + test "retries only interrupted jobs that are safe to run again, a few times" do + interrupted_destroy = failed Room::DestroyJob.new(rooms(:watercooler), []), interrupted + broken_destroy = failed Room::DestroyJob.new(rooms(:hq), []), RuntimeError.new("boom") + interrupted_webhook = failed Bot::WebhookJob.new(users(:bender), messages(:first)), interrupted + interrupted_removal = failed RemoveBannedContentJob.new(users(:kevin)), interrupted + exhausted_destroy = failed Room::DestroyJob.new(rooms(:pets), []), interrupted, executions: 3 + + RetryInterruptedJobsJob.perform_now + + assert interrupted_destroy.reload.ready? + assert_equal 1, interrupted_destroy.arguments["executions"] + assert_not broken_destroy.reload.ready? + assert_not interrupted_webhook.reload.ready? + assert_not interrupted_removal.reload.ready? + assert_not exhausted_destroy.reload.ready? + end + + private + def interrupted + SolidQueue::Processes::ProcessPrunedError.new(1.minute.ago) + end + + def failed(active_job, error, executions: 0) + active_job.executions = executions + SolidQueue::Job.enqueue(active_job).tap do |job| + job.ready_execution.destroy! + job.failed_with(error) + end + end +end