From 831dd096a915ffae0fc1ac569f94dff426d8d254 Mon Sep 17 00:00:00 2001 From: Thomas Klemm Date: Thu, 8 Oct 2026 11:26:28 +0000 Subject: [PATCH 1/3] Use Solid Cache for the production Rails cache Rails.cache moves from Redis to Solid Cache, in its own SQLite database that db:prepare creates from db/cache_schema.rb. Entries aren't copied, so the cache starts cold after the upgrade. The cache database keeps SQLite's own WAL checkpoints, since the checkpoint contender only checkpoints the primary database. The restore hook deletes it: a copy taken from a running volume can be torn, and the cache refills on demand. The HTTP benchmark prepares the databases before starting Puma, so a seed made before a new database or migration still boots. --- Gemfile | 3 +++ Gemfile.lock | 5 +++++ bench/compare_http.rb | 2 +- config/cache.yml | 18 ++++++++++++++++++ config/database.yml | 13 +++++++++++++ config/environments/production.rb | 3 +-- db/cache_schema.rb | 12 ++++++++++++ docs/self-hosting.md | 5 ++--- hooks/post-restore | 3 +++ 9 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 config/cache.yml create mode 100644 db/cache_schema.rb diff --git a/Gemfile b/Gemfile index 1e602d0dc..81bb4ab5e 100644 --- a/Gemfile +++ b/Gemfile @@ -11,6 +11,9 @@ gem "benchmark" gem "sqlite3" gem "redis", "~> 5.4" +# Caching +gem "solid_cache", "~> 1.0.10" + # Deployment gem "puma", "~> 8.0" diff --git a/Gemfile.lock b/Gemfile.lock index 80909cb63..410b84e10 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -390,6 +390,10 @@ GEM rack-protection (= 4.2.1) rack-session (>= 2.0.0, < 3) tilt (~> 2.0) + solid_cache (1.0.10) + activejob (>= 7.2) + activerecord (>= 7.2) + railties (>= 7.2) sqlite3 (2.10.2-aarch64-linux-gnu) sqlite3 (2.10.2-arm64-darwin) sqlite3 (2.10.2-x86_64-darwin) @@ -463,6 +467,7 @@ DEPENDENCIES selenium-webdriver sentry-rails sentry-ruby + solid_cache (~> 1.0.10) sqlite3 stimulus-rails surfguard! diff --git a/bench/compare_http.rb b/bench/compare_http.rb index 519b016e8..51595d0a5 100644 --- a/bench/compare_http.rb +++ b/bench/compare_http.rb @@ -46,7 +46,7 @@ command.concat environment(RAILS_ENV: "production", SECRET_KEY_BASE: "isolated-benchmark-fixture-key", DISABLE_SSL: true, SKIP_TELEMETRY: true, RAILS_LOG_LEVEL: "fatal", WEB_CONCURRENCY: 1, JOB_CONCURRENCY: 1, RAILS_MAX_THREADS: 5, REDIS_URL: "redis://#{redis}:6379/0") - command.concat [ options[:image], "bundle", "exec", "puma", "-C", "config/puma.rb" ] + command.concat [ options[:image], "sh", "-c", "bin/rails db:prepare && exec bundle exec puma -C config/puma.rb" ] run(*command) deadline = clock + 45 until client.ready? diff --git a/config/cache.yml b/config/cache.yml new file mode 100644 index 000000000..654816ffc --- /dev/null +++ b/config/cache.yml @@ -0,0 +1,18 @@ +default: &default + store_options: + max_size: <%= 256.megabytes %> + namespace: <%= Rails.env %> + +development: + <<: *default + +test: + <<: *default + +performance: + database: cache + <<: *default + +production: + database: cache + <<: *default diff --git a/config/database.yml b/config/database.yml index 845eb70f2..7b56910f0 100644 --- a/config/database.yml +++ b/config/database.yml @@ -32,8 +32,21 @@ performance: primary: <<: *default database: storage/db/performance.sqlite3 + cache: + <<: *default + database: storage/db/performance_cache.sqlite3 + migrations_paths: db/cache_migrate + pragmas: + wal_autocheckpoint: 1000 production: primary: <<: *default database: storage/db/production.sqlite3 + cache: + <<: *default + database: storage/db/production_cache.sqlite3 + migrations_paths: db/cache_migrate + # The checkpoint contender only checkpoints the primary database. + pragmas: + wal_autocheckpoint: 1000 diff --git a/config/environments/production.rb b/config/environments/production.rb index 369bb5bf0..5a17d3281 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -68,8 +68,7 @@ # Don't log any deprecations. config.active_support.report_deprecations = false - # Cache in memory for now - config.cache_store = :redis_cache_store + config.cache_store = :solid_cache_store # Assets are cacheable config.public_file_server.headers = { diff --git a/db/cache_schema.rb b/db/cache_schema.rb new file mode 100644 index 000000000..81a410d18 --- /dev/null +++ b/db/cache_schema.rb @@ -0,0 +1,12 @@ +ActiveRecord::Schema[7.2].define(version: 1) do + create_table "solid_cache_entries", force: :cascade do |t| + t.binary "key", limit: 1024, null: false + t.binary "value", limit: 536870912, null: false + t.datetime "created_at", null: false + t.integer "key_hash", limit: 8, null: false + t.integer "byte_size", limit: 4, null: false + t.index ["byte_size"], name: "index_solid_cache_entries_on_byte_size" + t.index ["key_hash", "byte_size"], name: "index_solid_cache_entries_on_key_hash_and_byte_size" + t.index ["key_hash"], name: "index_solid_cache_entries_on_key_hash", unique: true + end +end diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 5398e0c5b..92de9bc87 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -196,7 +196,7 @@ docker run --rm \ This gives you a `campfire-backup.tar.gz` in your current directory containing the database snapshot and all uploaded files. Copy it somewhere safe, ideally off the machine. -To restore, extract the archive back into a (stopped) instance's volume, and replace the live database with the snapshot: +To restore, extract the archive back into a (stopped) instance's volume, and run the restore hook, which replaces the live database with the snapshot: ```sh docker run --rm \ @@ -205,8 +205,7 @@ docker run --rm \ --volume "$PWD":/backup \ ghcr.io/basecamp/once-campfire:latest \ bash -c "tar xzf /backup/campfire-backup.tar.gz -C /rails && - cp /rails/storage/backups/production.sqlite3 /rails/storage/db/production.sqlite3 && - rm -f /rails/storage/db/production.sqlite3-wal /rails/storage/db/production.sqlite3-shm && + /hooks/post-restore && chown -R rails:rails /rails/storage" ``` diff --git a/hooks/post-restore b/hooks/post-restore index 0b9138d1e..58976c7bf 100755 --- a/hooks/post-restore +++ b/hooks/post-restore @@ -4,9 +4,12 @@ set -euo pipefail db_name="${RAILS_ENV:-production}.sqlite3" snapshot="/rails/storage/backups/$db_name" db="/rails/storage/db/$db_name" +cache_db="/rails/storage/db/${RAILS_ENV:-production}_cache.sqlite3" if [ -f "$snapshot" ]; then mkdir -p "$(dirname "$db")" cp "$snapshot" "$db" rm -f "$db-wal" "$db-shm" fi + +rm -f "$cache_db" "$cache_db-wal" "$cache_db-shm" From aa26dbbec38a1facb6b1f64f8211d5bfa4a2912b Mon Sep 17 00:00:00 2001 From: Thomas Klemm Date: Thu, 8 Oct 2026 11:13:41 +0000 Subject: [PATCH 2/3] Replace Resque with Solid Queue Jobs move from Resque in the container's Redis to Solid Queue in its own SQLite database, so jobs queued when the container stops are still there when it starts again. Redis doesn't persist (save "", appendonly no), so jobs still queued in Resque at the upgrade are lost, as on any restart. The queue database is backed up and restored with the primary. A room deletion or banned user's content removal still queued at the backup would otherwise never run after a restore. The two databases are copied one after the other and the app writes them in separate commits, so a job enqueued while the backup runs can be missing from it: a room deleted in that moment comes back closed and without members, its messages left. A job running at the backup comes back claimed by its process, and fails once that process is pruned as dead. bin/boot prepares the databases before starting the web server and the workers, which now start together and both need the queue's tables. Solid Queue forks its workers from a supervisor, so the supervisor stops its checkpoint contender before forking and each worker starts its own, as Puma's workers do. Workers poll every 0.1 s, as Resque did. The queue's pool is five, for the worker's three threads plus its poller and heartbeat, independent of the web server's RAILS_MAX_THREADS. The test environment gets a queue database, so the backup test can use real Solid Queue records; jobs still run on the test adapter. The benchmark drivers take --baseline-image, since a baseline from before this change needs Resque's gems, which the current image no longer has, and prepare the databases before starting Puma, so a seed made before the queue database existed still boots. --- Gemfile | 3 +- Gemfile.lock | 42 ++--- Procfile | 2 +- bench/README.md | 3 +- bench/compare_http.rb | 2 +- bench/compare_message_hot_paths.rb | 2 +- bench/support.rb | 3 +- bin/boot | 1 + bin/jobs | 6 + bin/start-app | 1 - config/bundler-audit.yml | 5 - config/database.yml | 19 +++ config/environments/development.rb | 3 - config/environments/production.rb | 3 +- config/environments/test.rb | 3 + config/initializers/resque.rb | 1 - config/initializers/solid_queue.rb | 3 + config/initializers/sqlite_wal_checkpoint.rb | 2 +- config/queue.yml | 21 +++ config/recurring.yml | 4 + config/resque-pool.yml | 1 - db/queue_schema.rb | 160 +++++++++++++++++++ hooks/post-restore | 19 +-- lib/rails_ext/sqlite_wal_checkpoint.rb | 2 +- lib/tasks/resque.rake | 16 -- script/admin/prepare-backup | 22 +-- test/script/admin/prepare_backup_test.rb | 35 ++++ 27 files changed, 299 insertions(+), 85 deletions(-) create mode 100755 bin/jobs delete mode 100644 config/initializers/resque.rb create mode 100644 config/initializers/solid_queue.rb create mode 100644 config/queue.yml create mode 100644 config/recurring.yml delete mode 100644 config/resque-pool.yml create mode 100644 db/queue_schema.rb delete mode 100644 lib/tasks/resque.rake create mode 100644 test/script/admin/prepare_backup_test.rb diff --git a/Gemfile b/Gemfile index 81bb4ab5e..cf5b307a5 100644 --- a/Gemfile +++ b/Gemfile @@ -18,8 +18,7 @@ gem "solid_cache", "~> 1.0.10" gem "puma", "~> 8.0" # Jobs -gem "resque", "~> 2.7.0" -gem "resque-pool", "~> 0.7.1" +gem "solid_queue", "~> 1.7.2" # Assets gem "propshaft", github: "rails/propshaft" diff --git a/Gemfile.lock b/Gemfile.lock index 410b84e10..dabf9e28c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -175,6 +175,8 @@ GEM drb (2.2.3) erb (6.0.7) erubi (1.13.1) + et-orbi (1.4.2) + tzinfo faker (3.8.0) i18n (>= 1.8.11, < 2) ffi (1.17.2) @@ -182,6 +184,9 @@ GEM ffi (1.17.2-arm64-darwin) ffi (1.17.2-x86_64-darwin) ffi (1.17.2-x86_64-linux-gnu) + fugit (1.14.0) + et-orbi (~> 1.4) + raabro (~> 1.4) geared_pagination (1.2.0) activesupport (>= 5.0) addressable (>= 2.5.0) @@ -237,10 +242,6 @@ GEM prism (~> 1.5) mocha (3.1.0) ruby2_keywords (>= 0.0.5) - mono_logger (1.1.2) - multi_json (1.17.0) - mustermann (3.0.4) - ruby2_keywords (~> 0.0.1) net-imap (0.6.7) date net-protocol @@ -278,12 +279,9 @@ GEM public_suffix (6.0.2) puma (8.0.3) nio4r (~> 2.0) + raabro (1.5.0) racc (1.8.1) rack (3.2.8) - rack-protection (4.2.1) - base64 (>= 0.1.0) - logger (>= 1.6.0) - rack (>= 3.0.0, < 4) rack-session (2.1.2) base64 (>= 0.1.0) rack (>= 3.0.0) @@ -318,19 +316,9 @@ GEM redis-client (>= 0.22.0) redis-client (0.30.1) connection_pool - redis-namespace (1.11.0) - redis (>= 4) regexp_parser (2.11.2) reline (0.7.0) io-console (~> 0.5) - resque (2.7.0) - mono_logger (~> 1) - multi_json (~> 1.0) - redis-namespace (~> 1.6) - sinatra (>= 0.9.2) - resque-pool (0.7.1) - rake (>= 10.0, < 14.0) - resque (>= 1.22, < 3) rexml (3.4.4) rqrcode (3.2.0) chunky_png (~> 1.0) @@ -383,17 +371,17 @@ GEM sentry-ruby (5.26.0) bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) - sinatra (4.2.1) - logger (>= 1.6.0) - mustermann (~> 3.0) - rack (>= 3.0.0, < 4) - rack-protection (= 4.2.1) - rack-session (>= 2.0.0, < 3) - tilt (~> 2.0) solid_cache (1.0.10) activejob (>= 7.2) activerecord (>= 7.2) railties (>= 7.2) + solid_queue (1.7.2) + activejob (>= 7.1) + activerecord (>= 7.1) + concurrent-ruby (>= 1.3.1) + fugit (~> 1.11) + railties (>= 7.1) + thor (>= 1.3.1) sqlite3 (2.10.2-aarch64-linux-gnu) sqlite3 (2.10.2-arm64-darwin) sqlite3 (2.10.2-x86_64-darwin) @@ -406,7 +394,6 @@ GEM thruster (0.1.27-arm64-darwin) thruster (0.1.27-x86_64-darwin) thruster (0.1.27-x86_64-linux) - tilt (2.6.1) timeout (0.6.1) tsort (0.2.0) tzinfo (2.0.6) @@ -460,14 +447,13 @@ DEPENDENCIES rails! rails_autolink redis (~> 5.4) - resque (~> 2.7.0) - resque-pool (~> 0.7.1) rqrcode rubocop-rails-omakase selenium-webdriver sentry-rails sentry-ruby solid_cache (~> 1.0.10) + solid_queue (~> 1.7.2) sqlite3 stimulus-rails surfguard! diff --git a/Procfile b/Procfile index 20e143048..07d876c77 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,3 @@ web: bundle exec thrust bin/start-app redis: redis-server config/redis.conf -workers: FORK_PER_JOB=false INTERVAL=0.1 bundle exec resque-pool +workers: bundle exec bin/jobs diff --git a/bench/README.md b/bench/README.md index 820495245..cab9c007f 100644 --- a/bench/README.md +++ b/bench/README.md @@ -12,7 +12,8 @@ ruby bench/compare_http.rb --baseline PATH --seed SEED --paths sidebar,search -- Both drivers alternate before/after order and reset fixture storage for each run. Results default to ignored `tmp/rails-optimization/results/`; override with `--output PATH`. -Use `--image` to override `campfire-reference:app`, and `--cpus` to override server CPUs +Use `--image` to override `campfire-reference:app`, `--baseline-image` when the baseline needs +another bundle, and `--cpus` to override server CPUs `8-11`. HTTP clients use CPUs `12-15` by default (`--client-cpus`). `--help` lists options. The rendering probe checks exact response bodies, selected headers and unread payloads, diff --git a/bench/compare_http.rb b/bench/compare_http.rb index 51595d0a5..9e3944c29 100644 --- a/bench/compare_http.rb +++ b/bench/compare_http.rb @@ -46,7 +46,7 @@ command.concat environment(RAILS_ENV: "production", SECRET_KEY_BASE: "isolated-benchmark-fixture-key", DISABLE_SSL: true, SKIP_TELEMETRY: true, RAILS_LOG_LEVEL: "fatal", WEB_CONCURRENCY: 1, JOB_CONCURRENCY: 1, RAILS_MAX_THREADS: 5, REDIS_URL: "redis://#{redis}:6379/0") - command.concat [ options[:image], "sh", "-c", "bin/rails db:prepare && exec bundle exec puma -C config/puma.rb" ] + command.concat [ options[side == "before" ? :baseline_image : :image], "sh", "-c", "bin/rails db:prepare && exec bundle exec puma -C config/puma.rb" ] run(*command) deadline = clock + 45 until client.ready? diff --git a/bench/compare_message_hot_paths.rb b/bench/compare_message_hot_paths.rb index 225ee3e01..8825f6cf2 100644 --- a/bench/compare_message_hot_paths.rb +++ b/bench/compare_message_hot_paths.rb @@ -24,7 +24,7 @@ File.join(options[:seed], "labels.json") => "/bench-labels.json") command.concat environment(RAILS_ENV: "production", SECRET_KEY_BASE: "isolated-benchmark-fixture-key", DISABLE_SSL: true, SKIP_TELEMETRY: true, RAILS_LOG_LEVEL: "fatal", BENCH_LABELS: "/bench-labels.json") - command.concat [ options[:image], "bundle", "exec", "ruby", "-r", "/rails/config/environment.rb", "/bench/message_hot_paths.rb" ] + command.concat [ options[side == "before" ? :baseline_image : :image], "bundle", "exec", "ruby", "-r", "/rails/config/environment.rb", "/bench/message_hot_paths.rb" ] data = JSON.parse(run(*command)) runs[side] << data write_json(File.join(options[:output], "#{side}-#{iteration + 1}.json"), data) diff --git a/bench/support.rb b/bench/support.rb index c0c4089b7..8ff3478c4 100644 --- a/bench/support.rb +++ b/bench/support.rb @@ -8,7 +8,7 @@ module BenchmarkSupport WORK = File.join(ROOT, "tmp/rails-optimization") def parse_options(description, defaults) - options = { baseline: nil, seed: nil, image: "campfire-reference:app", cpus: "8-11", rounds: 2 }.merge(defaults) + options = { baseline: nil, seed: nil, image: "campfire-reference:app", baseline_image: nil, cpus: "8-11", rounds: 2 }.merge(defaults) parser = OptionParser.new do |parser| parser.banner = "Usage: ruby #{$PROGRAM_NAME} --baseline PATH --seed PATH [options]\n#{description}" options.each do |name, default| @@ -26,6 +26,7 @@ def parse_options(description, defaults) raise OptionParser::MissingArgument, "--baseline and --seed are required" unless options[:baseline] && options[:seed] raise OptionParser::InvalidArgument, "--rounds must be even and at least 2" unless options[:rounds] >= 2 && options[:rounds].even? %i[baseline seed output].each { |name| options[name] = File.expand_path(options.fetch(name)) } + options[:baseline_image] ||= options[:image] options rescue OptionParser::ParseError => error abort "#{error.message}\n#{parser}" diff --git a/bin/boot b/bin/boot index 5106115a7..c78c7a223 100755 --- a/bin/boot +++ b/bin/boot @@ -51,4 +51,5 @@ class MonitoredProcess end end +system "bin/rails db:prepare", exception: true ProcessMonitor.new("Procfile") diff --git a/bin/jobs b/bin/jobs new file mode 100755 index 000000000..dcf59f309 --- /dev/null +++ b/bin/jobs @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "solid_queue/cli" + +SolidQueue::Cli.start(ARGV) diff --git a/bin/start-app b/bin/start-app index 9a751c2f4..c1af71202 100755 --- a/bin/start-app +++ b/bin/start-app @@ -1,5 +1,4 @@ #!/bin/bash -e rm -f tmp/pids/server.pid -./bin/rails db:prepare ./bin/rails server diff --git a/config/bundler-audit.yml b/config/bundler-audit.yml index 5068d2f2d..e74b3af94 100644 --- a/config/bundler-audit.yml +++ b/config/bundler-audit.yml @@ -3,8 +3,3 @@ ignore: - CVE-THAT-DOES-NOT-APPLY - # resque-web advisories. Campfire doesn't mount resque-web. Remove when we move to resque 3. - - GHSA-5r7x-x5pm-94xj - - GHSA-m39q-9jm7-xp4q - - GHSA-r23w-h6cg-h95q - - GHSA-r5h4-xh93-f339 diff --git a/config/database.yml b/config/database.yml index 7b56910f0..e8a805551 100644 --- a/config/database.yml +++ b/config/database.yml @@ -27,6 +27,10 @@ test: primary: <<: *default database: storage/db/test.sqlite3 + queue: + <<: *default + database: storage/db/test_queue.sqlite3 + migrations_paths: db/queue_migrate performance: primary: @@ -38,6 +42,12 @@ performance: migrations_paths: db/cache_migrate pragmas: wal_autocheckpoint: 1000 + queue: + <<: *default + database: storage/db/performance_queue.sqlite3 + migrations_paths: db/queue_migrate + pragmas: + wal_autocheckpoint: 1000 production: primary: @@ -50,3 +60,12 @@ production: # The checkpoint contender only checkpoints the primary database. pragmas: wal_autocheckpoint: 1000 + queue: + <<: *default + database: storage/db/production_queue.sqlite3 + # Three worker threads plus the poller and heartbeat, whatever the web server's thread count. + pool: 5 + migrations_paths: db/queue_migrate + # The checkpoint contender only checkpoints the primary database. + pragmas: + wal_autocheckpoint: 1000 diff --git a/config/environments/development.rb b/config/environments/development.rb index a25cb6297..67c2c3cbd 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -64,9 +64,6 @@ # Uncomment if you wish to allow Action Cable access from any origin. # config.action_cable.disable_request_forgery_protection = true - # Uncomment to test with production class job queue - # config.active_job.queue_adapter = :resque - # Highlight code that enqueued background job in logs. config.active_job.verbose_enqueue_logs = true diff --git a/config/environments/production.rb b/config/environments/production.rb index 5a17d3281..79d697f61 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -92,5 +92,6 @@ # Only use :id for inspections in production. config.active_record.attributes_for_inspect = [ :id ] - config.active_job.queue_adapter = :resque + config.active_job.queue_adapter = :solid_queue + config.solid_queue.connects_to = { database: { writing: :queue } } end diff --git a/config/environments/test.rb b/config/environments/test.rb index 70deabfdf..1393f4f6b 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -34,6 +34,9 @@ # Disable request forgery protection in test environment. config.action_controller.allow_forgery_protection = false + # Jobs run on the test adapter; the queue database is for tests of Solid Queue's own records. + config.solid_queue.connects_to = { database: { writing: :queue } } + # Store uploaded files on the local file system in a temporary directory. config.active_storage.service = :test diff --git a/config/initializers/resque.rb b/config/initializers/resque.rb deleted file mode 100644 index 48b9c7d4a..000000000 --- a/config/initializers/resque.rb +++ /dev/null @@ -1 +0,0 @@ -Resque.redis = ENV.fetch("REDIS_URL") { "redis://localhost:6379" } diff --git a/config/initializers/solid_queue.rb b/config/initializers/solid_queue.rb new file mode 100644 index 000000000..97c732d7e --- /dev/null +++ b/config/initializers/solid_queue.rb @@ -0,0 +1,3 @@ +# Stop before forking, as Puma does, so workers don't inherit the contender's flock. +SolidQueue.on_start { SqliteWalCheckpoint.stop } +SolidQueue.on_worker_start { SqliteWalCheckpoint.start } diff --git a/config/initializers/sqlite_wal_checkpoint.rb b/config/initializers/sqlite_wal_checkpoint.rb index aa14b8036..d858fcc1d 100644 --- a/config/initializers/sqlite_wal_checkpoint.rb +++ b/config/initializers/sqlite_wal_checkpoint.rb @@ -1,5 +1,5 @@ # Loaded from lib/rails_ext (autoload_lib ignore list) so reloads do not orphan -# the contender thread. Non-test processes start here; Puma/Resque stop before +# the contender thread. Non-test processes start here; Puma/Solid Queue stop before # fork and start again in the child. require Rails.root.join("lib/rails_ext/sqlite_wal_checkpoint") diff --git a/config/queue.yml b/config/queue.yml new file mode 100644 index 000000000..2b0212f33 --- /dev/null +++ b/config/queue.yml @@ -0,0 +1,21 @@ +default: &default + dispatchers: + - polling_interval: 1 + batch_size: 500 + workers: + - queues: "*" + threads: 3 + processes: <%= ENV.fetch("JOB_CONCURRENCY", 1) %> + polling_interval: 0.1 + +development: + <<: *default + +test: + <<: *default + +performance: + <<: *default + +production: + <<: *default diff --git a/config/recurring.yml b/config/recurring.yml new file mode 100644 index 000000000..854f415b5 --- /dev/null +++ b/config/recurring.yml @@ -0,0 +1,4 @@ +production: + clear_solid_queue_finished_jobs: + command: "SolidQueue::Job.clear_finished_in_batches(sleep_between_batches: 0.3)" + schedule: every hour at minute 12 diff --git a/config/resque-pool.yml b/config/resque-pool.yml deleted file mode 100644 index 64f569bde..000000000 --- a/config/resque-pool.yml +++ /dev/null @@ -1 +0,0 @@ -default: <%= (Concurrent.processor_count * 0.5).ceil %> diff --git a/db/queue_schema.rb b/db/queue_schema.rb new file mode 100644 index 000000000..f9a71dabb --- /dev/null +++ b/db/queue_schema.rb @@ -0,0 +1,160 @@ +ActiveRecord::Schema[7.1].define(version: 1) do + create_table "solid_queue_blocked_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.string "concurrency_key", null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.index [ "concurrency_key", "priority", "job_id" ], name: "index_solid_queue_blocked_executions_for_release" + t.index [ "expires_at", "concurrency_key" ], name: "index_solid_queue_blocked_executions_for_maintenance" + t.index [ "job_id" ], name: "index_solid_queue_blocked_executions_on_job_id", unique: true + end + + create_table "solid_queue_claimed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.bigint "process_id" + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_claimed_executions_on_job_id", unique: true + t.index [ "process_id", "job_id" ], name: "index_solid_queue_claimed_executions_on_process_id_and_job_id" + end + + create_table "solid_queue_failed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.text "error" + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_failed_executions_on_job_id", unique: true + end + + create_table "solid_queue_jobs", force: :cascade do |t| + t.string "queue_name", null: false + t.string "class_name", null: false + t.text "arguments" + t.integer "priority", default: 0, null: false + t.string "active_job_id" + t.datetime "scheduled_at" + t.datetime "finished_at" + t.string "concurrency_key" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.bigint "batch_id" + t.index [ "active_job_id" ], name: "index_solid_queue_jobs_on_active_job_id" + t.index [ "batch_id" ], name: "index_solid_queue_jobs_on_batch_id" + t.index [ "class_name" ], name: "index_solid_queue_jobs_on_class_name" + t.index [ "finished_at" ], name: "index_solid_queue_jobs_on_finished_at" + t.index [ "queue_name", "finished_at" ], name: "index_solid_queue_jobs_for_filtering" + t.index [ "scheduled_at", "finished_at" ], name: "index_solid_queue_jobs_for_alerting" + end + + create_table "solid_queue_pauses", force: :cascade do |t| + t.string "queue_name", null: false + t.datetime "created_at", null: false + t.index [ "queue_name" ], name: "index_solid_queue_pauses_on_queue_name", unique: true + end + + create_table "solid_queue_processes", force: :cascade do |t| + t.string "kind", null: false + t.datetime "last_heartbeat_at", null: false + t.bigint "supervisor_id" + t.integer "pid", null: false + t.string "hostname" + t.text "metadata" + t.datetime "created_at", null: false + t.string "name", null: false + t.index [ "last_heartbeat_at" ], name: "index_solid_queue_processes_on_last_heartbeat_at" + t.index [ "name", "supervisor_id" ], name: "index_solid_queue_processes_on_name_and_supervisor_id", unique: true + t.index [ "supervisor_id" ], name: "index_solid_queue_processes_on_supervisor_id" + end + + create_table "solid_queue_ready_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_ready_executions_on_job_id", unique: true + t.index [ "priority", "job_id" ], name: "index_solid_queue_poll_all" + t.index [ "queue_name", "priority", "job_id" ], name: "index_solid_queue_poll_by_queue" + end + + create_table "solid_queue_recurring_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "task_key", null: false + t.datetime "run_at", null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_recurring_executions_on_job_id", unique: true + t.index [ "task_key", "run_at" ], name: "index_solid_queue_recurring_executions_on_task_key_and_run_at", unique: true + end + + create_table "solid_queue_recurring_tasks", force: :cascade do |t| + t.string "key", null: false + t.string "schedule", null: false + t.string "command", limit: 2048 + t.string "class_name" + t.text "arguments" + t.string "queue_name" + t.integer "priority", default: 0 + t.boolean "static", default: true, null: false + t.text "description" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index [ "key" ], name: "index_solid_queue_recurring_tasks_on_key", unique: true + t.index [ "static" ], name: "index_solid_queue_recurring_tasks_on_static" + end + + create_table "solid_queue_scheduled_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "scheduled_at", null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_scheduled_executions_on_job_id", unique: true + t.index [ "scheduled_at", "priority", "job_id" ], name: "index_solid_queue_dispatch_all" + end + + create_table "solid_queue_semaphores", force: :cascade do |t| + t.string "key", null: false + t.integer "value", default: 1, null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index [ "expires_at" ], name: "index_solid_queue_semaphores_on_expires_at" + t.index [ "key", "value" ], name: "index_solid_queue_semaphores_on_key_and_value" + t.index [ "key" ], name: "index_solid_queue_semaphores_on_key", unique: true + end + + create_table "solid_queue_batches", force: :cascade do |t| + t.string "active_job_batch_id" + t.string "description" + t.text "on_finish" + t.text "on_success" + t.text "on_failure" + t.text "metadata" + t.integer "total_jobs", default: 0, null: false + t.integer "completed_jobs", default: 0, null: false + t.integer "failed_jobs", default: 0, null: false + t.datetime "enqueued_at" + t.datetime "finished_at" + t.datetime "failed_at" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index [ "active_job_batch_id" ], name: "index_solid_queue_batches_on_active_job_batch_id", unique: true + t.index [ "finished_at" ], name: "index_solid_queue_batches_on_finished_at" + end + + create_table "solid_queue_batch_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.bigint "batch_id", null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_batch_executions_on_job_id", unique: true + t.index [ "batch_id" ], name: "index_solid_queue_batch_executions_on_batch_id" + end + + add_foreign_key "solid_queue_batch_executions", "solid_queue_batches", column: "batch_id", on_delete: :cascade + add_foreign_key "solid_queue_batch_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_ready_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_recurring_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_scheduled_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade +end diff --git a/hooks/post-restore b/hooks/post-restore index 58976c7bf..c106deb5b 100755 --- a/hooks/post-restore +++ b/hooks/post-restore @@ -1,15 +1,16 @@ #!/bin/bash set -euo pipefail -db_name="${RAILS_ENV:-production}.sqlite3" -snapshot="/rails/storage/backups/$db_name" -db="/rails/storage/db/$db_name" -cache_db="/rails/storage/db/${RAILS_ENV:-production}_cache.sqlite3" +for db_name in "${RAILS_ENV:-production}.sqlite3" "${RAILS_ENV:-production}_queue.sqlite3"; do + snapshot="/rails/storage/backups/$db_name" + db="/rails/storage/db/$db_name" -if [ -f "$snapshot" ]; then - mkdir -p "$(dirname "$db")" - cp "$snapshot" "$db" - rm -f "$db-wal" "$db-shm" -fi + if [ -f "$snapshot" ]; then + mkdir -p "$(dirname "$db")" + cp "$snapshot" "$db" + rm -f "$db-wal" "$db-shm" + fi +done +cache_db="/rails/storage/db/${RAILS_ENV:-production}_cache.sqlite3" rm -f "$cache_db" "$cache_db-wal" "$cache_db-shm" diff --git a/lib/rails_ext/sqlite_wal_checkpoint.rb b/lib/rails_ext/sqlite_wal_checkpoint.rb index 7b1e35bff..b5c58050c 100644 --- a/lib/rails_ext/sqlite_wal_checkpoint.rb +++ b/lib/rails_ext/sqlite_wal_checkpoint.rb @@ -2,7 +2,7 @@ # writer and fsyncs during the request. database.yml sets wal_autocheckpoint=0; # this module copies pages off the request thread with PASSIVE checkpoints. # -# Every non-test process starts a contender. Callers that fork (Puma, Resque pool) +# Every non-test process starts a contender. Callers that fork (Puma, Solid Queue) # must stop before fork and start again in the child so the flock is never shared # across an inherited file descriptor. module SqliteWalCheckpoint diff --git a/lib/tasks/resque.rake b/lib/tasks/resque.rake deleted file mode 100644 index 6e04c7470..000000000 --- a/lib/tasks/resque.rake +++ /dev/null @@ -1,16 +0,0 @@ -task "resque:setup" do - require_relative "../../config/environment" -end - -task "resque:pool:setup" do - ActiveRecord::Base.connection.disconnect! - # Initializer may have started a contender in the pool master; drop it before - # workers fork so they do not inherit the flock. - SqliteWalCheckpoint.stop - - Resque::Pool.after_prefork do |job| - ActiveRecord::Base.establish_connection - Resque.redis.client.close - SqliteWalCheckpoint.start - end -end diff --git a/script/admin/prepare-backup b/script/admin/prepare-backup index 7701124d3..c61b728e8 100755 --- a/script/admin/prepare-backup +++ b/script/admin/prepare-backup @@ -4,22 +4,22 @@ require File.expand_path("../../config/environment", File.dirname(__FILE__)) class Backup class << self def create - dest = SQLite3::Database.new(backup_filename) - backup = SQLite3::Backup.new(dest, "main", Room.connection.raw_connection, "main") - - backup.step(-1) - backup.finish - dest.close + snapshot SolidQueue::Record + snapshot ApplicationRecord end private - def backup_filename - Rails.root.join("storage", "backups").tap(&:mkpath).join(primary_database_filename) + def snapshot(model) + dest = SQLite3::Database.new(backup_filename(model)) + backup = SQLite3::Backup.new(dest, "main", model.connection.raw_connection, "main") + + backup.step(-1) + backup.finish + dest.close end - def primary_database_filename - path = Rails.application.config.database_configuration[Rails.env]["primary"]["database"] - File.basename(path) + def backup_filename(model) + Rails.root.join("storage", "backups").tap(&:mkpath).join(File.basename(model.connection_db_config.database)) end end end diff --git a/test/script/admin/prepare_backup_test.rb b/test/script/admin/prepare_backup_test.rb new file mode 100644 index 000000000..ca3a7be9e --- /dev/null +++ b/test/script/admin/prepare_backup_test.rb @@ -0,0 +1,35 @@ +require "test_helper" + +class PrepareBackupScriptTest < ActiveSupport::TestCase + # SQLite can't back up a connection in the middle of a write transaction. + self.use_transactional_tests = false + + teardown do + SolidQueue::Job.delete_all + SolidQueue::Process.delete_all + backups.each { |path| FileUtils.rm_f(path) } + end + + test "backs up the database and its queue, with the jobs running at the time" do + running = SolidQueue::Job.enqueue(RemoveBannedContentJob.new(users(:kevin))) + worker = SolidQueue::Process.create!(kind: "Worker", name: "worker", pid: 1, hostname: "campfire", last_heartbeat_at: Time.current) + SolidQueue::ReadyExecution.claim("*", 1, worker.id) + queued = SolidQueue::Job.enqueue(Room::DestroyJob.new(rooms(:watercooler), [])) + + load Rails.root.join("script/admin/prepare-backup").to_s + + database, queue = backups.map { |path| SQLite3::Database.new(path.to_s) } + assert_equal [ [ rooms(:watercooler).id ] ], database.execute("SELECT id FROM rooms WHERE id = ?", rooms(:watercooler).id) + assert_equal [ running.id, queued.id ].sort, queue.execute("SELECT id FROM solid_queue_jobs").flatten.sort + assert_equal [ [ running.id, worker.id ] ], queue.execute("SELECT job_id, process_id FROM solid_queue_claimed_executions") + ensure + [ database, queue ].compact.each(&:close) + end + + private + def backups + [ ApplicationRecord, SolidQueue::Record ].map do |model| + Rails.root.join("storage/backups", File.basename(model.connection_db_config.database)) + end + end +end From 4540f4764c3cd7794048b92d36b97c752825a54f Mon Sep 17 00:00:00 2001 From: Thomas Klemm Date: Thu, 8 Oct 2026 13:42:52 +0000 Subject: [PATCH 3/3] Replace Redis with Solid Cable With the cache and the jobs on Solid Cache and Solid Queue, Action Cable was the last thing using Redis. It moves to Solid Cable, in its own SQLite database, and Redis goes: the server process, the redis and kredis gems, the Debian package, bin/setup's container and CI's service. Development uses Solid Cable too, rather than the async adapter, because bin/dev runs several Puma workers and a broadcast has to reach sockets held by the others. Development caching uses the memory store. Broadcasts are transient, so the restore hook deletes the cable database with the cache's and db:prepare recreates both. --- .github/workflows/ci.yml | 16 ---------------- Dockerfile | 2 +- Gemfile | 3 +-- Gemfile.lock | 16 ++++++---------- Procfile | 1 - app/models/room.rb | 2 +- bin/setup | 22 ---------------------- config/cable.yml | 13 ++++++++----- config/database.yml | 19 +++++++++++++++++++ config/environments/development.rb | 2 +- config/redis.conf | 4 ---- db/cable_schema.rb | 10 ++++++++++ docs/development.md | 4 +--- hooks/post-restore | 6 ++++-- 14 files changed, 52 insertions(+), 68 deletions(-) delete mode 100644 config/redis.conf create mode 100644 db/cable_schema.rb diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70cdb6b8c..bd730503f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,12 +94,6 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - services: - redis: - image: redis # zizmor: ignore[unpinned-images] -- version tag is fine for service containers - ports: - - 6379:6379 - options: --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5 steps: - name: Install packages run: sudo apt-get update && sudo apt-get install --no-install-recommends -y libsqlite3-0 sqlite3 libvips curl ffmpeg @@ -111,8 +105,6 @@ jobs: - name: Set up Ruby uses: ruby/setup-ruby@f9c4492e86def3e644ab32a0ef185eda5b0c20d0 # v1.342.0 - env: - REDIS_URL: redis://localhost:6379/0 with: ruby-version: .ruby-version bundler-cache: true @@ -127,12 +119,6 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - services: - redis: - image: redis # zizmor: ignore[unpinned-images] -- version tag is fine for service containers - ports: - - 6379:6379 - options: --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5 steps: - name: Install packages run: sudo apt-get update && sudo apt-get install --no-install-recommends -y libsqlite3-0 sqlite3 libvips curl ffmpeg @@ -144,8 +130,6 @@ jobs: - name: Set up Ruby uses: ruby/setup-ruby@f9c4492e86def3e644ab32a0ef185eda5b0c20d0 # v1.342.0 - env: - REDIS_URL: redis://localhost:6379/0 with: ruby-version: .ruby-version bundler-cache: true diff --git a/Dockerfile b/Dockerfile index be20f07bd..a2aecdf87 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ WORKDIR /rails # Install base packages RUN apt-get update -qq && \ - apt-get install --no-install-recommends -y curl libsqlite3-0 libvips libjemalloc2 ffmpeg redis && \ + apt-get install --no-install-recommends -y curl libsqlite3-0 libvips libjemalloc2 ffmpeg && \ ln -s /usr/lib/$(uname -m)-linux-gnu/libjemalloc.so.2 /usr/local/lib/libjemalloc.so && \ rm -rf /var/lib/apt/lists /var/cache/apt/archive diff --git a/Gemfile b/Gemfile index cf5b307a5..aeba98e88 100644 --- a/Gemfile +++ b/Gemfile @@ -9,7 +9,6 @@ gem "benchmark" # Drivers gem "sqlite3" -gem "redis", "~> 5.4" # Caching gem "solid_cache", "~> 1.0.10" @@ -27,6 +26,7 @@ gem "importmap-rails", github: "rails/importmap-rails" # Hotwire gem "turbo-rails", github: "hotwired/turbo-rails" gem "stimulus-rails" +gem "solid_cable", "~> 4.1" # Rich text gem "lexxy", "~> 0.9.24" @@ -46,7 +46,6 @@ gem "rails_autolink" gem "geared_pagination" gem "jbuilder" gem "surfguard", github: "basecamp/surfguard" # The SSRF address policy behind RestrictedHTTP -gem "kredis" gem "platform_agent" gem "thruster" diff --git a/Gemfile.lock b/Gemfile.lock index dabf9e28c..1430aacc0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -214,10 +214,6 @@ GEM json (2.21.2) jwt (3.2.0) base64 - kredis (1.8.0) - activemodel (>= 6.0.0) - activesupport (>= 6.0.0) - redis (>= 4.2, < 6) language_server-protocol (3.17.0.5) lexxy (0.9.24) rails (>= 8.0.2) @@ -312,10 +308,6 @@ GEM prism (>= 1.6.0) rbs (>= 4.0.0) tsort - redis (5.4.1) - redis-client (>= 0.22.0) - redis-client (0.30.1) - connection_pool regexp_parser (2.11.2) reline (0.7.0) io-console (~> 0.5) @@ -371,6 +363,11 @@ GEM sentry-ruby (5.26.0) bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) + solid_cable (4.1.0) + actioncable (>= 7.2) + activejob (>= 7.2) + activerecord (>= 7.2) + railties (>= 7.2) solid_cache (1.0.10) activejob (>= 7.2) activerecord (>= 7.2) @@ -437,7 +434,6 @@ DEPENDENCIES image_processing (>= 1.2) importmap-rails! jbuilder - kredis lexxy (~> 0.9.24) mocha ostruct @@ -446,12 +442,12 @@ DEPENDENCIES puma (~> 8.0) rails! rails_autolink - redis (~> 5.4) rqrcode rubocop-rails-omakase selenium-webdriver sentry-rails sentry-ruby + solid_cable (~> 4.1) solid_cache (~> 1.0.10) solid_queue (~> 1.7.2) sqlite3 diff --git a/Procfile b/Procfile index 07d876c77..4eab710d3 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,2 @@ web: bundle exec thrust bin/start-app -redis: redis-server config/redis.conf workers: bundle exec bin/jobs diff --git a/app/models/room.rb b/app/models/room.rb index 5ce909796..871b54c2d 100644 --- a/app/models/room.rb +++ b/app/models/room.rb @@ -61,7 +61,7 @@ def destroy_later # Deleting the memberships skips the reset that revoking one does, so the former members get it here, all at once, # before the messages go: their connections stop receiving the room's streams. It's done in the job, not the - # request, because it costs a Redis round trip per member. + # request, because it costs a broadcast per member. def reset_remote_connections_of(user_ids) User.where(id: user_ids).find_each(&:reset_remote_connections) end diff --git a/bin/setup b/bin/setup index e0f5bcec7..73492dd5b 100755 --- a/bin/setup +++ b/bin/setup @@ -8,9 +8,6 @@ app_root="$( )" export PATH="$app_root/bin:$PATH" -REDIS_PORT=6379 -REDIS_HOST=localhost - if [ "$RAILS_ENV" = "production" ]; then echo "RAILS_ENV is production; bailing out" exit 1 @@ -45,10 +42,6 @@ step() { return $exit_code } -redis_running() { - nc -z "$REDIS_HOST" "$REDIS_PORT" 2>/dev/null -} - echo gum style --foreground 214 " ) " gum style --foreground 208 " ) \\ campfire" @@ -88,21 +81,6 @@ if [[ $* == *--reset* ]]; then fi step "Preparing the database" rails db:prepare -# Start Redis if not running -if ! redis_running; then - if command -v docker &>/dev/null; then - if docker ps -aq -f name=campfire-redis | grep -q .; then - step "Starting Redis" docker start campfire-redis - else - step "Setting up Redis" docker run -d --name campfire-redis -p "$REDIS_PORT:$REDIS_PORT" redis:7 - fi - else - echo "Couldn't start Redis" - echo "Install either docker or redis and then run this command again" - exit 1 - fi -fi - # Install GitHub Actions linting tools for tool in actionlint shellcheck zizmor; do if ! command -v "$tool" &> /dev/null; then diff --git a/config/cable.yml b/config/cable.yml index 95e54f7cf..4b1b05a5e 100644 --- a/config/cable.yml +++ b/config/cable.yml @@ -1,18 +1,21 @@ default: &default - adapter: redis - url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379" } %> + adapter: solid_cable + connects_to: + database: + writing: cable + polling_interval: 0.1.seconds + message_retention: 1.day + writer_batch_size: 4 + writer_batch_delay: 0.001.seconds development: <<: *default - channel_prefix: campfire_development test: adapter: test performance: <<: *default - channel_prefix: campfire_performance production: <<: *default - channel_prefix: campfire_production diff --git a/config/database.yml b/config/database.yml index e8a805551..2180b0ddf 100644 --- a/config/database.yml +++ b/config/database.yml @@ -19,6 +19,12 @@ development: primary: <<: *default database: storage/db/development.sqlite3 + cable: + <<: *default + database: storage/db/development_cable.sqlite3 + migrations_paths: db/cable_migrate + pragmas: + wal_autocheckpoint: 1000 # Warning: The database defined as "test" will be erased and # re-generated from your development database when you run "rake". @@ -36,6 +42,12 @@ performance: primary: <<: *default database: storage/db/performance.sqlite3 + cable: + <<: *default + database: storage/db/performance_cable.sqlite3 + migrations_paths: db/cable_migrate + pragmas: + wal_autocheckpoint: 1000 cache: <<: *default database: storage/db/performance_cache.sqlite3 @@ -53,6 +65,13 @@ production: primary: <<: *default database: storage/db/production.sqlite3 + cable: + <<: *default + database: storage/db/production_cable.sqlite3 + migrations_paths: db/cable_migrate + # The checkpoint contender only checkpoints the primary database. + pragmas: + wal_autocheckpoint: 1000 cache: <<: *default database: storage/db/production_cache.sqlite3 diff --git a/config/environments/development.rb b/config/environments/development.rb index 67c2c3cbd..7b95989dc 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -21,7 +21,7 @@ config.action_controller.perform_caching = true config.action_controller.enable_fragment_cache_logging = true - config.cache_store = :redis_cache_store + config.cache_store = :memory_store config.public_file_server.headers = { "cache-control" => "public, max-age=#{2.days.to_i}" } diff --git a/config/redis.conf b/config/redis.conf deleted file mode 100644 index 804d865c6..000000000 --- a/config/redis.conf +++ /dev/null @@ -1,4 +0,0 @@ -port 6379 -daemonize no -appendonly no -save "" diff --git a/db/cable_schema.rb b/db/cable_schema.rb new file mode 100644 index 000000000..9aeefca25 --- /dev/null +++ b/db/cable_schema.rb @@ -0,0 +1,10 @@ +ActiveRecord::Schema[7.1].define(version: 1) do + create_table "solid_cable_messages", force: :cascade do |t| + t.binary "channel", limit: 1024, null: false + t.binary "payload", limit: 536870912, null: false + t.datetime "created_at", null: false + t.integer "channel_hash", limit: 8, null: false + t.index ["channel_hash"], name: "index_solid_cable_messages_on_channel_hash" + t.index ["created_at"], name: "index_solid_cable_messages_on_created_at" + end +end diff --git a/docs/development.md b/docs/development.md index c53e24360..a15f18b3f 100644 --- a/docs/development.md +++ b/docs/development.md @@ -8,7 +8,7 @@ First, get everything installed and configured with: bin/setup ``` -This installs the system packages Campfire needs (SQLite, ffmpeg), the right Ruby version (via [mise](https://mise.jdx.dev)), and the app's gems; prepares the database; and starts Redis (in a Docker container called `campfire-redis`, if it isn't already running locally). +This installs the system packages Campfire needs (SQLite, ffmpeg), the right Ruby version (via [mise](https://mise.jdx.dev)), and the app's gems, and prepares the databases. If you want to start over at any point, run: @@ -28,8 +28,6 @@ You'll be able to access the app at http://localhost:3000. On first run you'll be guided through creating your admin account, and you can sign in with that account from then on. -Note that Campfire needs Redis (for Action Cable, caching, and background jobs), so if you've restarted your machine or stopped the container, `docker start campfire-redis` will bring it back. - ### Web Push notifications Campfire uses VAPID (Voluntary Application Server Identification) keys to send browser push notifications. For notifications to work in development you'll need to generate a key pair and set these environment variables: diff --git a/hooks/post-restore b/hooks/post-restore index c106deb5b..4eb235098 100755 --- a/hooks/post-restore +++ b/hooks/post-restore @@ -12,5 +12,7 @@ for db_name in "${RAILS_ENV:-production}.sqlite3" "${RAILS_ENV:-production}_queu fi done -cache_db="/rails/storage/db/${RAILS_ENV:-production}_cache.sqlite3" -rm -f "$cache_db" "$cache_db-wal" "$cache_db-shm" +for disposable in cache cable; do + db="/rails/storage/db/${RAILS_ENV:-production}_$disposable.sqlite3" + rm -f "$db" "$db-wal" "$db-shm" +done