Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions Gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,7 @@ gem "redis", "~> 5.4"
gem "puma", "~> 8.0"

# Jobs
gem "resque", "~> 2.7.0"
gem "resque-pool", "~> 0.7.1"
gem "solid_queue", "~> 1.7.2"

# Assets
gem "propshaft", github: "rails/propshaft"
Expand Down
42 changes: 14 additions & 28 deletions Gemfile.lock
Original file line number Diff line number Diff line change
Expand Up @@ -175,13 +175,18 @@ GEM
drb (2.2.3)
erb (6.0.7)
erubi (1.13.1)
et-orbi (1.4.2)
tzinfo
faker (3.8.0)
i18n (>= 1.8.11, < 2)
ffi (1.17.2)
ffi (1.17.2-aarch64-linux-gnu)
ffi (1.17.2-arm64-darwin)
ffi (1.17.2-x86_64-darwin)
ffi (1.17.2-x86_64-linux-gnu)
fugit (1.14.0)
et-orbi (~> 1.4)
raabro (~> 1.4)
geared_pagination (1.2.0)
activesupport (>= 5.0)
addressable (>= 2.5.0)
Expand Down Expand Up @@ -237,10 +242,6 @@ GEM
prism (~> 1.5)
mocha (3.1.0)
ruby2_keywords (>= 0.0.5)
mono_logger (1.1.2)
multi_json (1.17.0)
mustermann (3.0.4)
ruby2_keywords (~> 0.0.1)
net-imap (0.6.7)
date
net-protocol
Expand Down Expand Up @@ -278,12 +279,9 @@ GEM
public_suffix (6.0.2)
puma (8.0.3)
nio4r (~> 2.0)
raabro (1.5.0)
racc (1.8.1)
rack (3.2.8)
rack-protection (4.2.1)
base64 (>= 0.1.0)
logger (>= 1.6.0)
rack (>= 3.0.0, < 4)
rack-session (2.1.2)
base64 (>= 0.1.0)
rack (>= 3.0.0)
Expand Down Expand Up @@ -318,19 +316,9 @@ GEM
redis-client (>= 0.22.0)
redis-client (0.30.1)
connection_pool
redis-namespace (1.11.0)
redis (>= 4)
regexp_parser (2.11.2)
reline (0.7.0)
io-console (~> 0.5)
resque (2.7.0)
mono_logger (~> 1)
multi_json (~> 1.0)
redis-namespace (~> 1.6)
sinatra (>= 0.9.2)
resque-pool (0.7.1)
rake (>= 10.0, < 14.0)
resque (>= 1.22, < 3)
rexml (3.4.4)
rqrcode (3.2.0)
chunky_png (~> 1.0)
Expand Down Expand Up @@ -383,13 +371,13 @@ GEM
sentry-ruby (5.26.0)
bigdecimal
concurrent-ruby (~> 1.0, >= 1.0.2)
sinatra (4.2.1)
logger (>= 1.6.0)
mustermann (~> 3.0)
rack (>= 3.0.0, < 4)
rack-protection (= 4.2.1)
rack-session (>= 2.0.0, < 3)
tilt (~> 2.0)
solid_queue (1.7.2)
activejob (>= 7.1)
activerecord (>= 7.1)
concurrent-ruby (>= 1.3.1)
fugit (~> 1.11)
railties (>= 7.1)
thor (>= 1.3.1)
sqlite3 (2.10.2-aarch64-linux-gnu)
sqlite3 (2.10.2-arm64-darwin)
sqlite3 (2.10.2-x86_64-darwin)
Expand All @@ -402,7 +390,6 @@ GEM
thruster (0.1.27-arm64-darwin)
thruster (0.1.27-x86_64-darwin)
thruster (0.1.27-x86_64-linux)
tilt (2.6.1)
timeout (0.6.1)
tsort (0.2.0)
tzinfo (2.0.6)
Expand Down Expand Up @@ -456,13 +443,12 @@ DEPENDENCIES
rails!
rails_autolink
redis (~> 5.4)
resque (~> 2.7.0)
resque-pool (~> 0.7.1)
rqrcode
rubocop-rails-omakase
selenium-webdriver
sentry-rails
sentry-ruby
solid_queue (~> 1.7.2)
sqlite3
stimulus-rails
surfguard!
Expand Down
2 changes: 1 addition & 1 deletion Procfile
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
web: bundle exec thrust bin/start-app
redis: redis-server config/redis.conf
workers: FORK_PER_JOB=false INTERVAL=0.1 bundle exec resque-pool
workers: bundle exec bin/jobs
31 changes: 31 additions & 0 deletions app/jobs/retry_interrupted_jobs_job.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
# Solid Queue fails the job a stopped or killed worker was running. Retry it where running it again is harmless.
class RetryInterruptedJobsJob < ApplicationJob
RETRYABLE = %w[ Room::DestroyJob ActiveStorage::PurgeJob ]
INTERRUPTIONS = %w[
SolidQueue::Processes::ProcessExitError SolidQueue::Processes::ProcessPrunedError SolidQueue::Processes::ProcessMissingError
]
MAX_RETRIES = 3

def perform
SolidQueue::FailedExecution.joins(:job).where(job: { class_name: RETRYABLE }).includes(:job).find_each do |execution|
retry_interrupted(execution) if execution.exception_class.in?(INTERRUPTIONS)
end
end

private
# FailedExecution#retry resets the job's execution count, so it's counted here instead.
def retry_interrupted(execution)
job = execution.job
retries = job.arguments["executions"].to_i
return if retries >= MAX_RETRIES

execution.with_lock do
job.arguments["executions"] = retries + 1
job.save!
job.prepare_for_execution
execution.destroy!
end
rescue ActiveRecord::RecordNotFound
# Retried or discarded by hand in the meantime.
end
end
3 changes: 2 additions & 1 deletion bench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ ruby bench/compare_http.rb --baseline PATH --seed SEED --paths sidebar,search --

Both drivers alternate before/after order and reset fixture storage for each run.
Results default to ignored `tmp/rails-optimization/results/`; override with `--output PATH`.
Use `--image` to override `campfire-reference:app`, and `--cpus` to override server CPUs
Use `--image` to override `campfire-reference:app`, `--baseline-image` when the baseline needs
another bundle, and `--cpus` to override server CPUs
`8-11`. HTTP clients use CPUs `12-15` by default (`--client-cpus`). `--help` lists options.

The rendering probe checks exact response bodies, selected headers and unread payloads,
Expand Down
2 changes: 1 addition & 1 deletion bench/compare_http.rb
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
command.concat environment(RAILS_ENV: "production", SECRET_KEY_BASE: "isolated-benchmark-fixture-key", DISABLE_SSL: true,
SKIP_TELEMETRY: true, RAILS_LOG_LEVEL: "fatal", WEB_CONCURRENCY: 1, JOB_CONCURRENCY: 1, RAILS_MAX_THREADS: 5,
REDIS_URL: "redis://#{redis}:6379/0")
command.concat [ options[:image], "bundle", "exec", "puma", "-C", "config/puma.rb" ]
command.concat [ options[side == "before" ? :baseline_image : :image], "sh", "-c", "bin/rails db:prepare && exec bundle exec puma -C config/puma.rb" ]
run(*command)
deadline = clock + 45
until client.ready?
Expand Down
2 changes: 1 addition & 1 deletion bench/compare_message_hot_paths.rb
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
File.join(options[:seed], "labels.json") => "/bench-labels.json")
command.concat environment(RAILS_ENV: "production", SECRET_KEY_BASE: "isolated-benchmark-fixture-key", DISABLE_SSL: true,
SKIP_TELEMETRY: true, RAILS_LOG_LEVEL: "fatal", BENCH_LABELS: "/bench-labels.json")
command.concat [ options[:image], "bundle", "exec", "ruby", "-r", "/rails/config/environment.rb", "/bench/message_hot_paths.rb" ]
command.concat [ options[side == "before" ? :baseline_image : :image], "bundle", "exec", "ruby", "-r", "/rails/config/environment.rb", "/bench/message_hot_paths.rb" ]
data = JSON.parse(run(*command))
runs[side] << data
write_json(File.join(options[:output], "#{side}-#{iteration + 1}.json"), data)
Expand Down
3 changes: 2 additions & 1 deletion bench/support.rb
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ module BenchmarkSupport
WORK = File.join(ROOT, "tmp/rails-optimization")

def parse_options(description, defaults)
options = { baseline: nil, seed: nil, image: "campfire-reference:app", cpus: "8-11", rounds: 2 }.merge(defaults)
options = { baseline: nil, seed: nil, image: "campfire-reference:app", baseline_image: nil, cpus: "8-11", rounds: 2 }.merge(defaults)
parser = OptionParser.new do |parser|
parser.banner = "Usage: ruby #{$PROGRAM_NAME} --baseline PATH --seed PATH [options]\n#{description}"
options.each do |name, default|
Expand All @@ -26,6 +26,7 @@ def parse_options(description, defaults)
raise OptionParser::MissingArgument, "--baseline and --seed are required" unless options[:baseline] && options[:seed]
raise OptionParser::InvalidArgument, "--rounds must be even and at least 2" unless options[:rounds] >= 2 && options[:rounds].even?
%i[baseline seed output].each { |name| options[name] = File.expand_path(options.fetch(name)) }
options[:baseline_image] ||= options[:image]
options
rescue OptionParser::ParseError => error
abort "#{error.message}\n#{parser}"
Expand Down
1 change: 1 addition & 0 deletions bin/boot
Original file line number Diff line number Diff line change
Expand Up @@ -51,4 +51,5 @@ class MonitoredProcess
end
end

system "bin/rails db:prepare", exception: true
ProcessMonitor.new("Procfile")
6 changes: 6 additions & 0 deletions bin/jobs
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#!/usr/bin/env ruby

require_relative "../config/environment"
require "solid_queue/cli"

SolidQueue::Cli.start(ARGV)
1 change: 0 additions & 1 deletion bin/start-app
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
#!/bin/bash -e

rm -f tmp/pids/server.pid
./bin/rails db:prepare
./bin/rails server
5 changes: 0 additions & 5 deletions config/bundler-audit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,3 @@

ignore:
- CVE-THAT-DOES-NOT-APPLY
# resque-web advisories. Campfire doesn't mount resque-web. Remove when we move to resque 3.
- GHSA-5r7x-x5pm-94xj
- GHSA-m39q-9jm7-xp4q
- GHSA-r23w-h6cg-h95q
- GHSA-r5h4-xh93-f339
19 changes: 19 additions & 0 deletions config/database.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,32 @@ test:
primary:
<<: *default
database: storage/db/test.sqlite3
queue:
<<: *default
database: storage/db/test_queue.sqlite3
migrations_paths: db/queue_migrate

performance:
primary:
<<: *default
database: storage/db/performance.sqlite3
queue:
<<: *default
database: storage/db/performance_queue.sqlite3
migrations_paths: db/queue_migrate
pragmas:
wal_autocheckpoint: 1000

production:
primary:
<<: *default
database: storage/db/production.sqlite3
queue:
<<: *default
database: storage/db/production_queue.sqlite3
# Three worker threads plus the poller and heartbeat, whatever the web server's thread count.
pool: 5
migrations_paths: db/queue_migrate
# The checkpoint contender only checkpoints the primary database.
pragmas:
wal_autocheckpoint: 1000
3 changes: 0 additions & 3 deletions config/environments/development.rb
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,6 @@
# Uncomment if you wish to allow Action Cable access from any origin.
# config.action_cable.disable_request_forgery_protection = true

# Uncomment to test with production class job queue
# config.active_job.queue_adapter = :resque

# Highlight code that enqueued background job in logs.
config.active_job.verbose_enqueue_logs = true

Expand Down
3 changes: 2 additions & 1 deletion config/environments/production.rb
Original file line number Diff line number Diff line change
Expand Up @@ -93,5 +93,6 @@
# Only use :id for inspections in production.
config.active_record.attributes_for_inspect = [ :id ]

config.active_job.queue_adapter = :resque
config.active_job.queue_adapter = :solid_queue
config.solid_queue.connects_to = { database: { writing: :queue } }
end
3 changes: 3 additions & 0 deletions config/environments/test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@
# Disable request forgery protection in test environment.
config.action_controller.allow_forgery_protection = false

# Jobs run on the test adapter; the queue database is for tests of Solid Queue's own records.
config.solid_queue.connects_to = { database: { writing: :queue } }

# Store uploaded files on the local file system in a temporary directory.
config.active_storage.service = :test

Expand Down
1 change: 0 additions & 1 deletion config/initializers/resque.rb

This file was deleted.

3 changes: 3 additions & 0 deletions config/initializers/solid_queue.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# Stop before forking, as Puma does, so workers don't inherit the contender's flock.
SolidQueue.on_start { SqliteWalCheckpoint.stop }
SolidQueue.on_worker_start { SqliteWalCheckpoint.start }
2 changes: 1 addition & 1 deletion config/initializers/sqlite_wal_checkpoint.rb
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Loaded from lib/rails_ext (autoload_lib ignore list) so reloads do not orphan
# the contender thread. Non-test processes start here; Puma/Resque stop before
# the contender thread. Non-test processes start here; Puma/Solid Queue stop before
# fork and start again in the child.
require Rails.root.join("lib/rails_ext/sqlite_wal_checkpoint")

Expand Down
21 changes: 21 additions & 0 deletions config/queue.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
default: &default
dispatchers:
- polling_interval: 1
batch_size: 500
workers:
- queues: "*"
threads: 3
processes: <%= ENV.fetch("JOB_CONCURRENCY", 1) %>
polling_interval: 0.1

development:
<<: *default

test:
<<: *default

performance:
<<: *default

production:
<<: *default
7 changes: 7 additions & 0 deletions config/recurring.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
production:
clear_solid_queue_finished_jobs:
command: "SolidQueue::Job.clear_finished_in_batches(sleep_between_batches: 0.3)"
schedule: every hour at minute 12
retry_interrupted_jobs:
class: RetryInterruptedJobsJob
schedule: every minute
1 change: 0 additions & 1 deletion config/resque-pool.yml

This file was deleted.

Loading