diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb57ce4..28af586 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,27 @@ jobs: bundler-cache: true - run: bundle exec rubocop + docs: + name: Docs + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + - uses: ruby/setup-ruby@762794c140bbeda0f1224786aa33b4b46783a6c1 # v1.326.0 + with: + ruby-version: "3.4" + bundler-cache: true + - run: bundle exec rake docs:check + # A warning on each page whose sources this pull request changed while the page did not. + - if: github.event_name == 'pull_request' + run: bundle exec rake docs:stale + env: + BASE: origin/${{ github.base_ref }} + # No converters installed here, on purpose. The three hotcell gems claim to be testable with no toolchain and # no container, and a job that installed libvips would stop that claim from being checked. hotcell: diff --git a/AGENTS.md b/AGENTS.md index 396a922..493e3db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,12 +1,39 @@ -# HotCell +# Hot Cell -This project includes a set of core gems used to isolate untrusted code or handling untrusted input: -hotcell-core, hotcell-server, and hotcell-client. +This repository holds the core gems that isolate untrusted code and untrusted input (`hotcell-core`, +`hotcell-server`, and `hotcell-client`), the Active Storage gems built on them +(`activestorage-hotcell-client` and `activestorage-hotcell-server`), and `yabeda-hotcell`. -It also includes a set of gems that build on the HotCell gems to provide drop-in replacement for -Active Storage analyzers, transformers, and previewers: activestorage-hotcell-client and -activestorage-hotcell-server. +## Find your way around -## How to develop this project. +The docs are in sibling sections, and each section's index lists its pages with a one-line description, so +you can pick the page you need without opening the others: -@CONTRIBUTING.md +- [`docs/index.md`](docs/index.md): the reference manual, which describes behavior. +- [`docs/design/index.md`](docs/design/index.md): the design rationale. +- [`docs/contributing/index.md`](docs/contributing/index.md): working on the gems. + +Each page's frontmatter names the code it describes, under `sources`. + +The numbered invariants in [`docs/design/invariants.md`](docs/design/invariants.md) and the numbered +experiments in [`docs/contributing/experiments.md`](docs/contributing/experiments.md) are cited by number from code, +tests, and docs. Never renumber them. + +## Keep the docs current + +@docs/contributing/docs.md + +## Write the docs + +- The README is the introduction for people: what Hot Cell is, and a quick start. Keep it short, and link + to a reference page for details instead of repeating them. +- The pages in `docs/` are a reference manual, written for agents first and readable by people. Write them + in [Google developer documentation style](https://developers.google.com/style): second person, present + tense, active voice, sentence-case headings, numbered lists for steps, and tables for settings. +- Keep one topic on each page, and keep each fact on one page. Link to it from everywhere else. +- The pages in `docs/design/` record rationale, not behavior. Change them when a decision changes. Record a + decision that was argued in [`adr/`](docs/contributing/decisions.md). + +## Working on the gems + +@docs/contributing/guide.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 39e3540..a221aff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,12 @@ This changelog covers these gems, which release together on the same version: A `Tooling` section records changes to the checks and scripts in `bin/` and `examples/`, which ship in no gem but are what an operator runs against their own image. +## next / unreleased + +### Documentation + +* The README is a short introduction and quick start, and the details are in a reference manual with one topic per page, starting at [docs/index.md](docs/index.md). `docs/DESIGN.md`, `docs/DEPLOYMENT.md`, `docs/TUNING.md` and `docs/LOGS.md` are gone. Their content is in [docs/design/](docs/design/index.md), [docs/container.md](docs/container.md), [docs/cell-settings.md](docs/cell-settings.md), [docs/client-api.md](docs/client-api.md), [docs/tuning.md](docs/tuning.md), [docs/scratch.md](docs/scratch.md) and [docs/observability.md](docs/observability.md). + ## v0.6.0 / 2026-10-01 ### Upgrading @@ -113,7 +119,7 @@ Some actions that application developers should consider taking when upgrading f Some actions that application developers should consider taking when upgrading from an earlier version: * Log `stderr` from the `perform.hot_cell` event in the client Rails application. This improves observability and provides forensic evidence about cell crashes. -* Set `MAGICK_MEMORY_LIMIT`, `MAGICK_MAP_LIMIT` and `MAGICK_DISK_LIMIT` in a cell image that installs ImageMagick. See [docs/IMAGEMAGICK.md](docs/IMAGEMAGICK.md). +* Set `MAGICK_MEMORY_LIMIT`, `MAGICK_MAP_LIMIT` and `MAGICK_DISK_LIMIT` in a cell image that installs ImageMagick. See [docs/imagemagick.md](docs/imagemagick.md). ### HotCell::Client @@ -192,7 +198,7 @@ Some actions that application developers should consider taking when upgrading f #### Added -* `request`, `request.abandoned`, `worker.crashed`, `worker.killed` and `worker.undispatchable` carry `hotcell.op`, the operation the line is about. A cell runs several operations at once and they do not share limits, so `worker.killed cause=fsize` on a host serving three PDF operations named none of them, and no join was available elsewhere: the response carries no operation, and `hotcell_killed` is tagged `cell` and `cause` only. The worker parses the name out of the request; the supervisor, which never reads one, learns it from the worker's report and holds it, because a killed worker cannot write its own `worker.killed`. Where the name is not known the field is `null` rather than an earlier request's name. See [docs/LOGS.md](docs/LOGS.md). +* `request`, `request.abandoned`, `worker.crashed`, `worker.killed` and `worker.undispatchable` carry `hotcell.op`, the operation the line is about. A cell runs several operations at once and they do not share limits, so `worker.killed cause=fsize` on a host serving three PDF operations named none of them, and no join was available elsewhere: the response carries no operation, and `hotcell_killed` is tagged `cell` and `cause` only. The worker parses the name out of the request; the supervisor, which never reads one, learns it from the worker's report and holds it, because a killed worker cannot write its own `worker.killed`. Where the name is not known the field is `null` rather than an earlier request's name. See [docs/observability.md](docs/observability.md#cell-log). * `worker.undispatchable` is the one line where the supervisor reads a request, since the worker died before the dispatch write and nothing else has read it. It peeks rather than reads, taking neither the bytes nor the caller's descriptors off the connection, and never waits for a request that has not arrived. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f693ebb..cc4f352 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,310 +1,4 @@ -# Contributing to HotCell +# Contributing to Hot Cell -This is a guide to working on the gems themselves. If you are building *on* HotCell — writing operations for -your own application — start with [README.md](README.md) instead. - - - - - -- [Setting up](#setting-up) -- [Running the tests](#running-the-tests) -- [The layout](#the-layout) -- [How a cell gets exercised](#how-a-cell-gets-exercised) - * [`rake test:devcell` — does the development configuration work?](#rake-testdevcell--does-the-development-configuration-work) - * [`bin/conformance IMAGE` — does this image support hotcell?](#binconformance-image--does-this-image-support-hotcell) - * [`bin/load IMAGE` — how does it behave under pressure?](#binload-image--how-does-it-behave-under-pressure) -- [Rules that are not obvious](#rules-that-are-not-obvious) -- [Style](#style) -- [Writing it down](#writing-it-down) - * [Accepted risk](#accepted-risk) - * [The changelog](#the-changelog) -- [CI](#ci) -- [Making a release](#making-a-release) - - - -## Setting up - -``` -bundle install -``` - -That is enough for the hotcell gems and for the example cell. It is deliberately enough: **the -hotcell gems need no container and no tool installed**, and `rake test:hotcell` is what keeps that honest — -CI runs it on a machine with nothing on it. Fixture operations stand in for the work, so the protocol, the -fork, the descriptor passing, the limits and the reap are all exercised in milliseconds. - -Two things need more, and only when you touch them: - -| To run | You need | -| --- | --- | -| `rake test:activestorage` | libvips, mutool, ffmpeg, ffprobe, ImageMagick, pdftoppm | -| `bin/conformance`, `bin/load` | Docker, and Linux | - -## Running the tests - -``` -rake # everything, then rubocop -rake test:hotcell # everything that needs no tool installed -rake test:activestorage # everything that needs the converters installed -rake test:devcell # boot the real development cell and run the example battery -rake test:gem:hotcell-core # one gem's suite -rake rubocop # style, one configuration for every gem -``` - -One file, or one test, from inside a gem directory: - -``` -cd hotcell-server -rake test TEST=test/slot_test.rb -rake test TEST=test/slot_test.rb TESTOPTS="-n/reuse/" -``` - -The container checks are scripts rather than rake tasks, because they need Docker and a built image: - -``` -bin/example-image # install the cell scaffold and build what it wrote -bin/conformance IMAGE # does this image support hotcell? -bin/load IMAGE [SCENARIO] [SECONDS] [THREADS] -``` - -## The layout - -``` -hotcell-core/ the wire protocol, descriptors, failures — both sides depend on it -hotcell-client/ the application side, and the installer that scaffolds a cell -hotcell-server/ the supervisor, workers, slots, limits, and exe/hotcell -activestorage-hotcell-server/ the media operations -activestorage-hotcell-client/ the Rails transformer, analyzers and previewers -yabeda-hotcell/ Yabeda metrics for the application side - -examples/ one cell's worth of sample operations and the battery that drives them -bin/ the container checks: example-image, conformance, load -docs/ the design document, the deployment guide, the tuning guide -adr/ decisions that were argued rather than obvious -``` - -## How a cell gets exercised - -There are two sets of operations in this repository, and reaching for the wrong one is the easiest mistake -to make here. - -**Fixture operations** — `HotCell::Fixtures` in `hotcell-server/lib/hot_cell/test_operations.rb`, named -`test.*`. These are for the minitest suites: forty-odd operations covering every edge the protocol has, from -`test.mojibake` to `test.early_idle`. They ship in the gem rather than sitting in its own test directory, -because `hotcell-client`'s suite boots a real cell and needs an inventory to point it at. If you are writing -a test for a gem, this is what you want. - -**Example operations** — `examples/operations`, named `example.*`. These are for exercising a *whole cell* -from the outside, where the suite does not own the process: a development cell booted from `exe/hotcell`, or -somebody's container image. They double as worked examples of how to write an operation, so they stay few, -readable and pure Ruby with no external toolchain — which is also what lets them run on macOS, where the -converters and Docker are absent. - -`echo` is also meant to be copied into a real cell and kept there. It is the only cheap way to prove the -work socket answers, because `describe` and `metrics` cross the control socket instead. See "Rails -healthcheck" in the [README](README.md#rails-healthcheck). - -| op | behavior | what it proves | -| --- | --- | --- | -| `echo` | read the input fd, write it back to the output fd | descriptor passing round-trips; a throughput baseline | -| `sleep` | block for `seconds` | the deadline kills and answers; head-of-line behavior under load | -| `greedy` | allocate `megabytes` | the `memory` clamp → `killed: memory`, where enforceable | -| `overflow` | write `megabytes` to the output | the `file_size` clamp → `killed: fsize` | -| `crash` | raise, or die by signal | worker death → a clean verdict and continued service | -| `spawn` | start a grandchild that would outlive the worker | the group kill leaves nothing behind | -| `probe` | whether a pid is alive, asked from inside the cell | how `spawn` is watched, in any pid namespace | -| `isolation` | interfaces, root writability, scratch `noexec`, a tool's environment | the container-only checks, from the only place that can make them | - -`examples/lib` holds the thin `HotCell::Client` classes that call them and the **battery** — one list of -checks built from those clients, so every consumer asks a cell the same questions. Consumers drive through -the client classes rather than a raw `Connection`, because that is what an application does, and the wrapper -costs little enough that load numbers still measure the cell. - -Three consumers run that battery. - -### `rake test:devcell` — does the development configuration work? - -Boots the real `exe/hotcell` as a plain host process on the example operations, exactly the way you run it -yourself, runs the battery, then sends `SIGTERM` and asserts exit 0 with no socket left behind. It is part -of `rake test:hotcell`, so it runs with the rest of the suite, on Linux and macOS both. - -It is Docker-free on purpose. See "Cells run uncontainerized in development" below. - -### `bin/conformance IMAGE` — does this image support hotcell? - -The real use case is somebody building their own cell image and wanting to know whether it works. It boots -`IMAGE` with the accessory's real flags — `network: none`, `cap-drop ALL`, a read-only root, a `noexec` -tmpfs — with `HOTCELL_DIR` pointed somewhere non-default, and asserts that the server starts, that the socket -appears where `HOTCELL_DIR` said it would, that the battery passes, and that the isolation holds. It exits -non-zero on the first failure, so it gates. - -The driver runs in a **second container** over a shared volume, so a descriptor crossing the container -boundary is part of every check. That container is a stock Ruby image rather than the image under test, so a -minimal cell image never has to carry the client's dependencies. - -The example operations are mounted over `/hotcell/operations`, shadowing whatever the image baked in. That is -deliberate: what is under test is the image's runtime — its Ruby, its gems, its user, and the flags — rather -than the work it does. - -CI runs it against what `bin/example-image` builds. That script runs the installer and builds exactly what it -wrote, so the scaffold an application is actually given is the thing CI proves. Nothing derives from a -published base image, so the installed scaffold is the only kind of cell image there is. - -### `bin/load IMAGE` — how does it behave under pressure? - -The same operations at volume against a containerized cell. It reports throughput, latency split into -`queued_ms` against `perform_ms`, and the verdict breakdown, which is what tells saturation apart from -slowness. Scenarios: `echo` for a baseline, `sleep` for queueing, `greedy` and `overflow` for the resource -kills, `crash` for worker death, `spawn` for orphans, and `mix` for a weighted blend. - -The cell's knobs pass through the environment, so finding a knee needs no edit: - -``` -EXAMPLE_CONCURRENCY=4 EXAMPLE_QUEUE_SIZE=16 bin/load hotcell:example echo 30 16 -``` - -Heavy runs are manual. A short bounded run works as a CI gate if one is ever wanted. - -## Rules that are not obvious - -**Nothing loads libvips into a test process.** libvips creates its thread pool the first time it processes an -image, and that pool does not survive `fork`: a child forked afterwards waits forever for a worker thread -that does not exist. The Active Storage suites boot real cells by forking, so the operations load inside the -cell, the fixtures are generated by CLI tools, and `Cell.boot` refuses to fork a process that has libvips -loaded. `fork_safety_test.rb` holds both halves of that, because a test that only showed the good case would -not establish that the hazard is real. - -The same rule is what `before_fork` and `before_worker_boot` exist for. A hook that evaluates an image in the -supervisor is a silent hang on every request afterwards, not a crash — see the comments on -`HotCell::Operation.before_fork`. - -**Cells run uncontainerized in development**, on every platform, so there is one thing to document and one -thing to debug. The reason is macOS and it cannot be engineered around: Docker Desktop runs containers in a -Linux VM, a file descriptor is an index into one kernel's table, and `sendmsg` has nothing meaningful to hand -across two kernels. Native `AF_UNIX` and `SCM_RIGHTS` on macOS are fine; it is the host-to-VM boundary that -cannot carry a descriptor. That is why `rake test:devcell` exists and why it uses no Docker, and what it -leaves out — the isolation — is what `bin/conformance` covers on Linux. - -**macOS has no finite `RLIMIT_DATA`.** `Process.setrlimit` rejects one with `EINVAL`, so a cell there runs -with its memory clamp unenforced and says so once, on `$stderr`. Tests that assert the clamp skip themselves -where it cannot be set; a new one that asserts memory behaviour has to do the same. Every other limit is -strict on both platforms. - -**Testing a control.** Security controls here fail silently, so a test that would still pass with -the control removed is worse than no test: it reads as assurance. Every control is covered by a test -that observes the behavior the control produces, rather than by an assertion that the control is -written down — `unsetenv_others: true` is proved by setting a variable, running a tool, and finding -that the tool never saw it. Where a control has no reachable trigger and so cannot be tested, it -says so where it lives. - -**A verdict that cannot be taken back needs a reason.** `permanent` means an application may write a failure -down against a blob and serve it from a cache forever, so anything unclassified is transient by default. -`hotcell-core/lib/hot_cell/codes.rb` carries the argument for each code; a new code goes in that table, and a -new kill reason goes in `PERMANENT_BY_CAUSE`, or it silently becomes the wrong kind of permanent. - -## Style - -``` -rake rubocop -``` - -One configuration for every gem, because the style is one style. It runs in CI as its own job. - -## Writing it down - -[docs/DESIGN.md](docs/DESIGN.md) holds the threat model, the numbered invariants, and the facts that were -established by experiment rather than by reasoning — the fork hazard, what `RLIMIT_DATA` charges, what -`/proc` gives a sibling away. Read it before changing anything a limit or an isolation claim rests on. It -deliberately does not describe behavior: that is the code's job, and a duplicate description rots. - -The invariants are numbered, and code comments across three gems cite them by number. Do not renumber them. - -[adr/](adr/README.md) holds the decisions that were argued rather than obvious, so the argument does not have -to be rerun. Write one when a decision cost you a real debate; do not edit one afterwards — a later decision -that changes it gets its own record. - -Comments in this codebase carry the reasoning, not the mechanics. A comment that restates the line below it -is noise; a comment that says why the obvious version does not work is the whole value. Match what is -already there. - -### Accepted risk - -A comment opening with **`Accepted risk.`** marks a decision rather than a claim. Somebody weighed that -exposure with the risk in view and chose to carry it, and the comment states the premise the choice rests -on. Grep for the phrase to find them all. - -Two kinds of comment sit near risky code and they are read differently. A **safety claim** says how -something works — "`cap-drop ALL` leaves no capability that overrides a mode". It can be wrong, and a -review should try to falsify it. An **accepted risk** is a preference, and restating the exposure it names -is not a finding. - -**So an accepted risk may only be re-opened by attacking its premise, never by restating its risk.** If the -premise still holds, there is nothing to report, whatever the exposure looks like. If you can show the -premise is false, report the premise: that is what changed, and it is what makes the decision worth taking -again. [ADR 0003](adr/0003-remove-the-persistent-slot-home.md) is the shape of a good one — the warm slot -home was an accepted risk whose premise, "the exposure is a write rather than a read", turned out to be -false, because what a tool reads from `$HOME` is configuration and configuration is executable. - -This applies to security reviews in particular, including ones run by agents. Reporting a priced risk back -as a new finding costs a maintainer the same argument every time. - -### The changelog - -Every pull request that changes behavior should update `CHANGELOG.md` in the same pull request. One -reader is a busy developer who is scanning the changelog to determine if they care. Another reader -is an agent trying to determine if the application needs any changes to accommodate the new version. - -Add to the `next / unreleased` section, under the gem the change is in — or under `Tooling` for the -checks and scripts in `bin/` and `examples/`, which ship in no gem — and under `Breaking`, -`Security`, `Added`, `Fixed`, `Improved` or `Changed`. `Improved` is for a change that makes existing -behavior better rather than fixing a defect — a performance win, or a limit that no longer applies. -Keep it short, one sentence is preferred that explains what changed. Lead with why the change matters to -the reader: what is now faster, safer, simpler or possible. Describe the change as it ships. Leave out -known defects and planned follow-ups; each gets its own entry when it is fixed, so no entry goes stale or -depends on another. If it's a breaking change, it can run longer to explain how to tell if an -application will be impacted, and how to update the application in that case. - -Skip the entry only for a change no consumer can observe: a test, a refactor, a comment, a CI tweak. If you -are unsure, write one. - -This applies to agents. If you opened the pull request, the entry is part of it. - -When a change gives an application something to do — a field to log, a variable to set, a value to -recompute — add it to an `Upgrading` section at the top of the release. Each item is two sentences. The -first is the action, in the imperative. The second is the rationale, or a link to the doc that holds it. - -## CI - -| Job | Runs | -| --- | --- | -| `hotcell (ruby …)` | `rake test:hotcell` on 3.3, 3.4, 4.0 and head, on a machine with no tools | -| `hotcell (macos, ruby 3.4)` | the same, on macOS — advisory, because the runners are slow and bill about ten times the Linux rate | -| `activestorage (ruby …)` | `rake test:activestorage` with the converters installed | -| `container conformance` | `bin/example-image` then `bin/conformance` | -| `Style` | `rubocop` | -| `GitHub Actions audit` | actionlint and zizmor | - -Actions are pinned to SHAs and the workflow has no default permissions. `zizmor` will tell you if a change -breaks either. - -## Making a release - -The gems release together on one version, and `VERSION` at the repository root is what sets it. - -- Prechecks - - [ ] make sure CI is green - - [ ] `bundle exec rake` — the full suite and rubocop - - [ ] update `CHANGELOG.md`: retitle `next / unreleased` with the version and the date - - [ ] `bundle exec rake version:bump[1.2.3]` — writes `VERSION` and every gem's version constant - - [ ] commit, and tag as `v1.2.3` -- Release - - [ ] `bundle exec rake gems` — builds every gem into `pkg/`, which it empties first - - [ ] `git push && git push --tags` — **before** the gems: every gemspec's `changelog_uri` and - `source_code_uri` name the tag, and they 404 until it is on GitHub - - [ ] `for f in pkg/*.gem ; do gem push $f ; done` - - [ ] create a release at https://github.com/basecamp/hotcell/releases -- Post-release - - [ ] `bundle exec rake version:bump[1.3.0.dev]`, and open a new `next / unreleased` section in - `CHANGELOG.md`, so that `master` never reports a version that was released +See [Working on the gems](docs/contributing/guide.md) in the [Contributing to Hot Cell](docs/contributing/index.md) +section of the docs. diff --git a/README.md b/README.md index 3fbe1b3..2023875 100644 --- a/README.md +++ b/README.md @@ -1,52 +1,24 @@ ![hotcell-logo](docs/hotcell-logo.png) -# HotCell - -Securely run untrusted code on untrusted inputs. HotCell lets you move that work out of your application -and into an unprivileged sibling container: no network, no credentials, and nothing on its filesystem -worth stealing. - -Inputs and outputs travel as file descriptors over a UNIX socket on a shared volume. Each call is a -remote procedure call ("RPC"): your application calls an ordinary Ruby method, the arguments are -forwarded to the container where the work runs in a forked worker process with strict limits -applied, and the results are returned or written to the output file. - - - - - -- [Status](#status) -- ["Why would I use this?"](#why-would-i-use-this) -- [Extensibility](#extensibility) -- [The gems](#the-gems) -- [The moving parts](#the-moving-parts) -- [How a request works](#how-a-request-works) -- [Usage](#usage) - * [Using the Active Storage operations](#using-the-active-storage-operations) - * [Using custom operations](#using-custom-operations) -- [Observability](#observability) - * [Recommended alerts](#recommended-alerts) - * [Cell logs](#cell-logs) - * [Application logs](#application-logs) - * [Metrics collection](#metrics-collection) - * [Per-call telemetry](#per-call-telemetry) - * [Container healthcheck](#container-healthcheck) - * [Rails healthcheck](#rails-healthcheck) -- [Development](#development) -- [Design](#design) - - +# Hot Cell + +Securely run untrusted code on untrusted inputs. Hot Cell moves that work out of your application and into +an unprivileged sibling container: no network, no credentials, and nothing on its filesystem worth +stealing. + +Inputs and outputs travel as file descriptors over a UNIX socket on a shared volume. Each call is a remote +procedure call (RPC): your application calls an ordinary Ruby method. Hot Cell forwards the arguments to the container, runs the work in a forked +worker process under strict limits, and returns the result or writes it to the output file. ## Status -This is still pre-release software! It may break in interesting ways. Use with caution until there's a v1.0 release. +This is pre-release software. It may break in interesting ways. Use it with caution until the v1.0 +release. -The `activestorage-hotcell-client` gem needs Rails 8.2, which is unreleased: variant processing can only -be swapped out from [rails/rails#58384](https://github.com/rails/rails/pull/58384) -([`5ea765e5`](https://github.com/rails/rails/commit/5ea765e5b00085a22f5cbe863c0d2ac765428242)) onward. Track -`rails/rails` `main` until it ships — see [Using the Active Storage operations](#using-the-active-storage-operations). +The Active Storage gems need Rails 8.2, which is unreleased. Track `rails/rails` `main` until it ships. +See [Active Storage operations](docs/active-storage.md). -## "Why would I use this?" +## Why would I use this? Your Rails application accepts uploads, so somewhere in it there's a line like this: @@ -54,183 +26,78 @@ Your Rails application accepts uploads, so somewhere in it there's a line like t blob.variant(resize_to_limit: [ 800, 600 ]).processed ``` -If you think about what that line actually does, it's scary. An attacker just handed you a crafted -file, and Rails is about to hand it to libvips -- a few hundred thousand lines of C whose entire job -is to guess at file formats it has never seen before and delegate handling to ANOTHER -format-specific library that you may not even be aware of. - -libvips, ImageMagick, ffmpeg and LibreOffice all have long histories of memory-safety bugs, and -every one of them is running by default in the same container that holds your database credentials, -your session secret, and a route to every network service the app uses. The potential blast radius -is large. +An attacker just handed you a crafted file, and Rails is about to hand it to libvips: a few hundred +thousand lines of C whose whole job is to guess at file formats and delegate them to other format-specific +libraries that you may not know about. libvips, ImageMagick, ffmpeg, and LibreOffice all have long +histories of memory-safety bugs, and by default each one runs in the container that holds your database +credentials, your session secret, and a route to every network service that the app uses. -HotCell gives that work its own container and its own forked process, holding nothing an attacker -wants. Take libvips and ffmpeg out of your application image entirely and keep them isolated in the -Cell. Code execution in that process buys an attacker a read-only input descriptor, a write-only -output descriptor, and the scratch of whatever else the cell is converting. The blast radius is much -smaller than if that attack succeeded in your application code. +Hot Cell gives that work its own container and its own forked process, holding nothing an attacker wants. +Remove libvips and ffmpeg from your application image and keep them in the cell. Code execution there +gets an attacker a read-only input descriptor, a write-only output descriptor, and the scratch of whatever +else the cell is converting. The blast radius is much smaller than if that attack succeeded in your +application code. -If you're a Rails developer, this project also ships drop-in replacements for the Active Storage -analyzers, transformers, and previewers so that using HotCell only requires configuration changes, -not code changes. +For Rails, Hot Cell ships drop-in replacements for the Active Storage analyzers, transformers, and +previewers, so adopting it takes configuration changes, not code changes. In our environment, Hot Cell adds +about 8 milliseconds per call and one more container on each host. We think that's a very good trade for +the improved security posture. -In our environment, using HotCell adds about 8 milliseconds per call, and one more container to -deploy on each host. We think this is a very good trade for the improved security posture. +Hot Cell isn't limited to media conversion: -## Extensibility +- You can configure several cells on each host. +- A call can pass several input and output files. +- Custom operations have a simple `#perform` API, like Active Job's. +- Cell limits are configurable: memory, wall-clock time, disk usage, and more. +- You can bring your own container image, checked by the included conformance test. +- You can set how often workers are forked again, to trade isolation for performance. -HotCell was designed to be flexible and configurable beyond just the media conversion use case: - -- multiple cells can be configured per host -- multiple input and output files are supported -- custom operations have a simple Active-Job-like `#perform` API -- cell limits are configurable: memory, wall clock time, disk usage, and more -- bring your own container by using the included conformance test -- customize how often workers are re-forked, for performance optimization - -So you could try using HotCell for handling ZIP files, or for compute workloads that might be CPU -hogs. Whatever is putting your trusted core application at risk, move it out! +So you could use Hot Cell for ZIP files, or for compute that might hog the CPU. If something puts your +trusted application at risk, move it into a cell. ## The gems | Gem | Runs in | Contains | | --- | --- | --- | -| `hotcell-core` | both sides | The wire protocol, descriptor passing, payload validation, the error taxonomy. | -| `hotcell-client` | the application | `HotCell::Client`, cell registration, routing, classification, instrumentation. | -| `hotcell-server` | the cell | The supervisor, the worker, `HotCell::Operation`, the container image. | -| `activestorage-hotcell-client` | the application | The transformer, analyzer, and previewers Rails is configured with. | -| `activestorage-hotcell-server` | the cell | The `transformers.image.*`, `analyzers.image.*`, `analyzers.media.ffprobe`, and `previewers.*` operations. | -| `yabeda-hotcell` | the application | Yabeda metrics for every call and for each local cell's counters. | - -They are in one repository because they are being developed together today. We may split out the -Active Storage gems into another repository at a later date. - -## The moving parts - -A **cell** is one deployment of the hot side: a supervisor, the workers it forks, and the operations -they serve, reachable through two unix sockets in one directory. A cell is the unit of isolation and -the unit of capacity. An application may register several cells by calling `HotCell.register` for -each one. - -The **supervisor** is pid 1 in the cell. It accepts connections, queues them, dispatches each to a -worker, enforces the wall-clock deadline from outside, kills and reaps. It never reads a request and -never evaluates a byte of image data -- it hands the accepted connection itself to a worker over -`SCM_RIGHTS` without ever calling `recvmsg`, so the caller's descriptors are still queued on it when -the worker reads them. - -A **worker** is a child the supervisor forks before a request needs it: one per slot at boot, and a -replacement as soon as the supervisor reaps one that served. Every untrusted byte is touched there and -nowhere else. It applies the cell's resource limits before touching the socket, serves -`max_requests_per_worker` requests, and exits without running finalizers. - -A **slot** is the numbered workspace a worker borrows. It holds one directory per request, which is -that request's `$HOME`, with the request's staged files under `scratch` inside it. The whole thing is -created when the request starts and removed before the caller hears the answer, so nothing a tool -writes reaches the next request on that slot. - -An **operation** is the unit of work a cell offers: a subclass of `HotCell::Operation` with a -routing name, its own `limits`, and a `perform(inputs, outputs, **payload)` that declares the -payload keys it wants as keyword arguments. By default, both sides derive the same name from the -class path, and the cell-side `Operation` suffix is stripped. So `ExtractTextOperation` in the cell -and `ExtractText` in the application both answer to `extract_text`. The set of operations a cell -carries is its **inventory** -- logged at boot, advertised on the control socket. - -A **client** is the application-side mirror of an operation: a subclass of `HotCell::Client` that -names the cell with `hotcell` and the operation with `operation`, and exposes -`perform_in_hotcell`. That is a blocking call -- it sends the request and waits for the cell's -answer, up to the `timeout` the cell was registered with. - -A **tool** is a program an operation runs in a subprocess (e.g., `mutool`, `ffmpeg`) via `run_tool`, -with a fully written environment and bounded capture of its output. The worker waits for it, and the -tool sees only the environment its operation wrote for it. Not every operation uses a tool, for -example the Vips operations call `libvips` directly from the Ruby worker process. - -The **payload** is a `Hash` of options, riding the request as its one JSON object and arriving in -`perform` as keyword arguments; the **result** is the `Hash` an operation returns, riding the -response the same way. Neither carries file contents. - -**Inputs** and **outputs** are the open file descriptors a caller passes -- inputs read-only, -outputs write-only. They are the only way file contents enter or leave a cell. Asking one for its -`path` materializes a temporary file on the worker's scratch that a tool can take. An input's bytes -are copied there on the first ask, and an output's file is sent back through the descriptor when -`perform` returns. An operation may read and write directly to the descriptors for efficiency. - -A failure carries a **code** -- `unreadable`, `invalid`, `unsupported`, `failed`, `capacity`, -`unavailable`, `timeout`, `protocol`, or `killed` with a cause -- and each code is **permanent** or -**transient**. Permanent means the input will fail this way every time, so the caller may record -that verdict against it. Everything uncertain is transient, meaning it might succeed on a retry. - -## How a request works - -```mermaid -sequenceDiagram - participant App as app process
(cold side, privileged) - participant Supervisor as supervisor, pid 1
(hot side, unprivileged) - participant Worker as worker
(forked ahead of dispatch) - - App->>Supervisor: one sendmsg -- JSON request + N descriptors - note over Supervisor: never reads the request
queues it, or answers capacity - Supervisor->>Worker: passes the connection itself over SCM_RIGHTS - note over Worker: applies the cell's limits before touching the socket
reads the request, narrows to the operation's limits - Worker->>Worker: perform(inputs, outputs, **payload)
an input copies to scratch when asked for a path - Worker->>App: posts the outputs, flushes, answers with one JSON line - note over Supervisor,Worker: a worker past its deadline is killed as a process group,
and the supervisor answers killed - Worker->>Worker: exits, or waits for the next dispatch -``` +| `hotcell-core` | both sides | The wire protocol, descriptor passing, payload validation, and the error taxonomy. | +| `hotcell-client` | the application | `HotCell::Client`, cell registration, routing, classification, and instrumentation. | +| `hotcell-server` | the cell | The supervisor, the worker, `HotCell::Operation`, and the container image. | +| `activestorage-hotcell-client` | the application | The transformers, analyzers, and previewers that Rails is configured with. | +| `activestorage-hotcell-server` | the cell | The Active Storage operations. | +| `yabeda-hotcell` | the application | Yabeda metrics for every call and for each local cell. | -1. The application calls `perform_in_hotcell inputs, outputs, payload` on a client class. The client wraps - each IO as an `Input` or `Output`, verifies its access mode -- inputs read-only, outputs write-only -- - validates the payload, and connects to the cell's `work.sock`. -2. One `sendmsg` carries one JSON line and every descriptor. -3. The supervisor accepts and dispatches, or answers `capacity` when the queue is full. -4. The worker narrows to the operation's limits, clamped to the cell's, before reading any untrusted byte, - and re-runs `before_worker_boot` when the operation differs from the last one it served. -5. `perform` runs on a fresh operation instance. An `Input` copies itself onto the slot's scratch the first - time the operation asks for its `path`; an operation that reads the descriptor directly never pays for - the copy. Outputs are posted back through their descriptors and flushed before success is reported. -6. One JSON line answers: `ok` with the result and the timing, or a failure with its code. -7. The supervisor enforces the deadline from outside, because a thread inside a C extension cannot be - interrupted from within. A worker past its deadline is killed as a process group, and the supervisor -- - the only survivor holding the connection -- answers `killed` with the cause. -8. The client raises the exception class registered for that side of the permanent split, and publishes a - `perform.hot_cell` notification either way. +The gems are in one repository because they're developed together. The Active Storage gems may move to +another repository later. -## Usage +## Using the Active Storage operations -The first section covers using HotCell's Active Storage operations straight out of the box. The -second section covers writing and using your own custom operations in HotCell. +This section runs Active Storage's variants, analysis, and previews in a cell. Your application code +doesn't change, but you deploy a second container beside the application: a Kamal accessory, or your +infrastructure's equivalent sidecar. -### Using the Active Storage operations +The examples use libvips, mutool, and ffmpeg. If your application uses `variant_processor = :magick`, +use the `Magick` classes instead of the `Vips` classes. See +[Active Storage operations](docs/active-storage.md#classes). -The two `activestorage-hotcell-*` gems run Active Storage's variants, analysis, and previews in a -cell instead of in the application. You application code doesn't need to change, though you will -need to deploy a Kamal accessory (or whatever flavor of sidecar container your infrastructure -supports). +### 1. Install -#### How to get started - -**Install.** Add the client gem to the application: +Add the client gem to the application: ```ruby # Gemfile -- the application gem "activestorage-hotcell-client" ``` -The Active Storage gems are what need the unreleased Rails 8.2 (see [Status](#status)); the -`hotcell-*` gems themselves do not require Rails. - -Then run `bin/rails hotcell:install` which creates: - -- `hotcell/Gemfile` for what the operations need, -- `hotcell/Dockerfile` that builds the cell's image, -- `hotcell/config.rb` for the cell's own settings, and -- `hotcell/operations/` directory of Ruby files the cell loads at boot. +Run `bin/rails hotcell:install`. It creates a `hotcell/` directory in the application root that holds +everything about the cell, separate from the client configuration and the application code: -Everything about the cell lives in this `hotcell/` directory in the application root, separate from the -client configuration and the application code. +- `hotcell/Gemfile`: the gems that the operations need. +- `hotcell/Dockerfile`: the recipe for the cell's image. +- `hotcell/config.rb`: the cell's own settings. +- `hotcell/operations/`: the Ruby files that the cell loads at boot. -Add the server gem to the cell's `Gemfile`, and load the operations that match the classes the -application will name -- requiring an operation's file is what serves it: +Add the server gem to the cell's `Gemfile`, and require the operations that match the classes that the +application uses. Requiring an operation's file is what makes the cell serve it. ```ruby # hotcell/Gemfile -- the cell @@ -246,27 +113,27 @@ require "active_storage/hot_cell/server/previewers/pdf/mutool" require "active_storage/hot_cell/server/previewers/video/ffmpeg" ``` -💡 This section's examples use libvips, mutool, and ffmpeg; `Transformers::Image::Magick` and -`Analyzers::Image::Magick` use ImageMagick instead, and `Previewers::Pdf::Poppler` uses Poppler. If -your application is currently using `variant_processor = :magick` then to retain compatibility you -should replace references to "vips" or `Vips` with "magick" or `Magick` in this section. +### 2. Configure the application -**Configure the application.** Register the cell in an initializer, with an application exception -class for each side of the permanent split: +Register the cell in an initializer, with one of your exception classes for permanent failures and one +for transient failures: ```ruby # config/initializers/hotcell.rb -# These environment variables are set in your deployment configuration HotCell.root = ENV["HOTCELL_ROOT"] # unset means every cell is off HotCell.group = ENV["HOTCELL_GROUP"] # the gid shared between app and cell -# Quick health check at boot. Warns about a cell that is unreachable, slower than this client waits, in the wrong group, or on another hotcell version. +HotCell.register "active_storage", + permanent: MyApp::UnprocessableUpload, + transient: MyApp::ConversionTemporarilyUnavailable + +# Warns at boot about a cell that is unreachable, slower than this client waits, in the wrong group, or on another hotcell version. Rails.application.config.after_initialize { HotCell.describe_cells } ``` -`HotCell.root` names the directory that holds one subdirectory of sockets per cell, so this cell's -sockets live at `$HOTCELL_ROOT/active_storage`. Note that omitting `HOTCELL_ROOT` off makes every -variant, analysis, and preview raise `HotCell::CellNotConfigured` rather than fall back in process. +The cell's sockets are in `$HOTCELL_ROOT/active_storage`. If `HOTCELL_ROOT` is unset, every variant, +analysis, and preview raises `HotCell::CellNotConfigured` rather than falling back to the application. See +[Client API](docs/client-api.md) and [Response codes](docs/codes.md). Then tell Rails which classes to use: @@ -280,67 +147,45 @@ config.active_storage.previewers = [ ActiveStorage::HotCell::Client::Previewers: ActiveStorage::HotCell::Client::Previewers::Video::FFmpeg ] ``` -For every class you name here, the cell must load the matching operation and the cell's image must -have installed the underlying library or tool. +For every class that you name, the cell must load the matching operation, and the cell's image must +install the library or tool. You can mix these classes with Rails' own. See +[Active Storage operations](docs/active-storage.md). -Rails' own classes mix freely with these in the `analyzers` and `previewers` arrays, so you can -choose to offload only specific operations to HotCell: +### 3. Run it in development -```ruby -# PDF previews handled by HotCell, video previews still in the application -config.active_storage.previewers = [ ActiveStorage::HotCell::Client::Previewers::Pdf::Mutool, - ActiveStorage::Previewer::VideoPreviewer ] -``` +Run the cell as a plain process, managed by foreman beside the Rails server. A containerized cell works +only on Linux, because descriptor passing doesn't cross the VM that runs containers on macOS. The resource +limits and the deadline apply either way. The cell keeps its own bundle, from the same `hotcell/Gemfile` +that the image build copies, so the two sides stay separate in development as they are in production. -**Run it in development.** A cell can run in development either as a container or as a plain, -uncontainerized process. The container route works only on Linux (on macOS the containers run in a -VM, and descriptor passing will not work), so we recommend the plain process, managed by foreman -beside the Rails server. The resource limits and the deadline apply either way. The cell keeps its -own bundle -- the same `hotcell/Gemfile` the image build copies -- so the two sides stay separate in -development the way they are in production. - -Add an entry for your cell to `Procfile.dev`: +Add the cell to `Procfile.dev`: ```procfile web: HOTCELL_ROOT=$PWD/tmp/hotcell-sockets bin/rails server cell: BUNDLE_GEMFILE=$PWD/hotcell/Gemfile HOTCELL_CONFIG=$PWD/hotcell/config.rb HOTCELL_OPERATIONS=$PWD/hotcell/operations HOTCELL_DIR=$PWD/tmp/hotcell-sockets/active_storage bundle exec hotcell --development ``` -Then `bin/dev` boots both, and the app finds the sockets under `tmp/hotcell-sockets`. At boot the cell -empties its `TMPDIR` of every entry its uid owns, and told none it empties the system temporary directory. -On a developer's machine that is `/tmp`, or on macOS the per-user `TMPDIR` every shell sets, both shared -with everything else the developer runs. With `--development` the cell never sweeps the directory it is -given: its scratch is `hotcell-` beneath it. `HOTCELL_WORKSPACE` -defaults under the scratch; pointed elsewhere, its parent is swept too. +`bin/dev` then boots both, and the app finds the sockets under `tmp/hotcell-sockets`. Keep +`--development`: without it, the cell empties the system temporary directory at boot. See +[Development mode](docs/cell-settings.md#development-mode). -#### Configure the cell and operation limits +### 4. Set the cell's limits -`hotcell/config.rb` loads when the cell boots, before any operation. Declare the cell's limits -there: +Declare the cell's limits in `hotcell/config.rb`: ```ruby # hotcell/config.rb HotCell.limits concurrency: 4, queue_size: 8, deadline: 60, memory: 1536 * 1024**2 ``` -Each operation declares its own `limits`, clamped to the cell's. To change an operation's default -limits, set them from an operations file: - -```ruby -# hotcell/operations/zz_limits.rb -require "active_storage/hot_cell/server/transformers/image/vips" -ActiveStorage::HotCell::Server::Transformers::Image::Vips.limits file_size: 256 * 1024**2 -``` - -[docs/DEPLOYMENT.md](docs/DEPLOYMENT.md) explains every setting and how to size the numbers against -the container's own flags, and [docs/TUNING.md](docs/TUNING.md) covers measuring your own workload. - -#### Configure and deploy the HotCell container +Each operation declares its own limits, and the cell clamps them to its own. See +[Cell settings](docs/cell-settings.md) for every setting and [Tuning](docs/tuning.md) for how to choose +the numbers. -There is no published base image. The installed `Dockerfile` is only a base recipe, and you should -customize it for your application. +### 5. Build and deploy the cell -Install the system packages your operations require: +Hot Cell publishes no base image. Customize the installed `Dockerfile` for your application, starting with +the system packages that your operations need: ```dockerfile # hotcell/Dockerfile (excerpt) @@ -349,14 +194,11 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* ``` -Match the `OMP_NUM_THREADS` the installed `Dockerfile` sets to the container's `cpus` below. OpenMP -reads the host's core count, which a `cpus` quota does not lower, so an unbounded libvips or ImageMagick -asks a large host for one 8MB thread stack per core and dies on its own memory limit -- see -[Bound the OpenMP thread pools](docs/DEPLOYMENT.md#bound-the-openmp-thread-pools). +Match the `OMP_NUM_THREADS` that the `Dockerfile` sets to the container's `cpus`. Without that bound, a +cell on a large host dies. See [Bound the OpenMP thread pools](docs/container.md#bound-the-openmp-thread-pools). -Build the image from the `hotcell/` directory and deploy it as a second container beside the -application, on the same host, sharing one volume that holds the sockets. With Kamal, that is one -accessory per cell: +Build the image from the `hotcell/` directory, and deploy it as a second container on the same host as the +application, sharing one volume for the sockets. With Kamal, that's one accessory for each cell: ```yaml # config/deploy.yml -- the cell @@ -387,7 +229,7 @@ accessories: HOTCELL_DIR: /run/hotcell/cell # where this cell writes its two sockets ``` -And the application's half, mounting the same volume: +The application mounts the same volume: ```yaml # config/deploy.yml -- the app @@ -410,272 +252,80 @@ env: HOTCELL_GROUP: 10001 # must match group-add above ``` -The two mount paths differ, and only the volume name has to match: a cell always writes its sockets -to `HOTCELL_DIR`, and the app resolves a cell's name under `HOTCELL_ROOT`. Give each cell its own -volume -- two accessories sharing one would write `work.sock` over each other. +The two mount paths differ, and only the volume name must match: a cell writes its sockets to +`HOTCELL_DIR`, and the app finds a cell by name under `HOTCELL_ROOT`. Give each cell its own volume. -Without Kamal, the same two containers need `--volume hotcell-sockets:/run/hotcell/cell` and the -security flags above on the cell, and `--volume hotcell-sockets:/run/hotcell/active_storage`, -`--group-add 10001` and `HOTCELL_ROOT=/run/hotcell` on the app. +[Container](docs/container.md) explains every flag and how to check a deployed accessory. +[Scratch](docs/scratch.md) covers moving scratch off the tmpfs. If your image installs ImageMagick, set +its limits too. See [ImageMagick](docs/imagemagick.md). -Once a file type's processing has moved into the cell, remove its packages (for example `libvips`) -from the application image -- that removal is the security win. Remove them only after the cell -handles the type: Rails' own previewers and analyzers look for their tool in `accept?`, so a package -removed too early turns that processing off without an error. +### 6. Remove the packages from the application image -[docs/DEPLOYMENT.md](docs/DEPLOYMENT.md) covers all of this in detail: every container flag, how to -size the numbers, the shared group, bringing your own container, and where scratch lives. -[docs/IMAGEMAGICK.md](docs/IMAGEMAGICK.md) covers ImageMagick's own resource limits, which an image that -installs it must size from the same numbers. +After the cell handles a file type, remove that type's packages, such as libvips, from the application +image. That removal is the security improvement. -### Using custom operations +Don't remove a package before the cell handles its file type. Rails' own previewers and analyzers look for +their tool in `accept?`, so a package removed too early turns that processing off without an error. -Everything above still applies: the same `hotcell/` directory, the same limits, the same container, -the same deployment. What changes is that you write both sides of the call yourself. +## Using custom operations -#### Write an operation +The `hotcell/` directory, the limits, the container, and the deployment all stay the same. You write both +sides of the call. -The client class in the application and the matching operation class in the cell share a routing -name. The fixed signature of the `#perform` method -- inputs, outputs, payload -- is the contract -for what crosses the socket. Descriptors travel as-is (without additional copies) and the payload -travels as one JSON object. +In the application, a client names its cell and the operation's routing name: -In the application, the client class and call site might look like: - -``` ruby +```ruby class TransformImage < HotCell::Client - hotcell "images" # the cell that serves this call, by registered name - operation "images.transform" # the routing name an operation must answer to + hotcell "images" + operation "images.transform" end -# source and destination are Files the app already opened. This is a blocking call that waits for -# a response from the cell. -TransformImage.perform_in_hotcell source, destination, - format: "png", - operations: { resize_to_limit: [ 800, 600 ] } +result = TransformImage.perform_in_hotcell source, destination, format: "png" ``` - -The operation will run in the cell. Declare the routing name and the limits, hook library loading, -and `#perform`: +In the cell, an operation answers to the same name, and `perform` receives the descriptors and the +payload as keyword arguments: ```ruby -require "active_support" -require "active_support/core_ext/numeric" # for 30.seconds and 1280.megabytes - class TransformImageOperation < HotCell::Operation - operation "images.transform" # the same routing name - - # Ceilings for one request enforced by rlimits and the supervisor's clock, - # clamped to the cell's own limits. - limits deadline: 30.seconds, memory: 1280.megabytes, file_size: 48.megabytes - - before_fork { require "my_image_processor" } # once, in the supervisor - before_worker_boot { MyImageProcessor.concurrency_set 4 } # in forked worker, before it serves a request - - # The descriptors the caller passed, and the payload as keyword arguments. A missing or undeclared - # key raises, so the signature is the schema. Declare **payload instead to take the Hash whole. - def perform(inputs, outputs, format:, operations: {}) - source, = inputs - destination, = outputs - - # fd_path reads the caller's file in place, with no copy onto scratch, so an input of any size - # costs nothing against file_size. Reach for source.path only when a tool needs a distinct on-disk - # copy; that stages the bytes, and the kernel charges the write. - MyImageProcessor.source(source.fd_path) - .apply(format:, operations) - .write_to(destination.fd_path) - - # The result: one JSON object, which the caller receives as perform_in_hotcell's return value (in - # addition to the destination file descriptor) - { format: format, bytes: File.size(destination.path) } - end -end -``` - -`before_fork` runs once in the supervisor and must never evaluate an image; `before_worker_boot` -runs in the worker and is where a library gets sized. `unreadable` names the library exceptions that -mean "this input cannot be decoded", a permanent verdict. An operation that shells out calls -`run_tool "mutool", "draw", ...` and gets the exit status and bounded output back; the tool sees -only the environment the operation wrote. - -A subclass inherits the `hotcell` name but not the `operation` name. The operation name is the wire -name, so to avoid answering to the same name as its parent, it derives its own from its class path -unless one is declared. - -#### Configure the application - -The initializer does not change shape: register the cell your client classes name, with exception -classes that fit the domain. Nothing under `config.active_storage` applies. This example -additionally declares custom base classes for exceptions, which can be useful when wrapping existing -libraries. - -```ruby -HotCell.register "images", - permanent: MyApp::UnreadableImage, - transient: MyApp::ConversionTemporarilyUnavailable -``` - -Leaving `HOTCELL_ROOT` unset turns every cell off -- `enabled?` answers false, and -`perform_in_hotcell` raises `HotCell::CellNotConfigured`. There is no automatic in-process fallback: -a caller that wants one checks `enabled?` and takes its old path, which is how an application rolls -a cell out as a configuration change rather than a release. - -#### Configure the cell - -The cell's `Gemfile` names `hotcell-server` directly -- the Active Storage server gem is only needed -for the shipped operations -- plus whatever gems the operation itself uses: - -```ruby -# hotcell/Gemfile -- the cell -gem "hotcell-server" -gem "my_image_processor" -``` - -The operation file goes in `hotcell/operations/`, which the cell requires in sorted order at boot, -after `config.rb`. The `Dockerfile` installs whatever tools the operation shells out to. `config.rb` -itself does not change: the cell's limits are declared there, and the operation's own `limits` ride -its class, clamped to the cell's exactly as the shipped ones are. - -## Observability - -Use these signals to monitor HotCell: the cell's log, the counters that the cell reports on its control -socket, and the application's record of each call. Set the alerts below. The sections after the alerts -tell where each signal comes from. - -### Recommended alerts - -- **Cell availability.** Alert when the `up` gauge is 0 or absent for any cell on any host. It reads 0 - first when a deploy missed a role, when the application lacks the cell's group, or when the supervisor is - dead. On a host without `HOTCELL_ROOT`, the gauge is absent. Calls on that host raise - `HotCell::CellNotConfigured`. -- **Failed calls.** Alert on the `requests` counter by `code`. The application records `unavailable` - when the cell is down, restarting, or unreachable. Any shift away from `ok` is an early warning. -- **Queue headroom.** Alert when `queued` nears the cell's `queue_size`, when `queue_high_water` rises - toward it, or when `capacity` appears in steady state. Each means the cell is under-provisioned. - `queue_size` is configuration, not a metric. `queue_high_water` resets only at boot, so alert on its - rise. A rising `cancelled` means callers gave up waiting. -- **Scratch space.** Alert on free space on the scratch of each host. For a disk-backed scratch, use - `node_filesystem_avail_bytes` from the node exporter. For a tmpfs, compare the container's memory usage - with the tmpfs `size=`. A full scratch fails every request that needs it. A write that fails inside libvips - gets `unreadable` from the cell, a permanent verdict against the file (see - [docs/IMAGEMAGICK.md](docs/IMAGEMAGICK.md)). - [Where scratch lives](docs/DEPLOYMENT.md#where-scratch-lives) covers the layouts. -- **Cell errors.** Alert on any `ERROR` event in the cell log, such as `worker.crashed` or - `worker.unforkable`, which should never happen. Alert on a rise in the `killed` gauge by cause: a - single kill for `memory` or `fsize` is the cell rejecting a hostile file. - -[What to watch](docs/TUNING.md#what-to-watch) lists the signals for tuning a cell's limits. - -### Cell logs - -The cell writes one JSON object per event to stdout, so whatever ships your container logs ships -these too. [docs/LOGS.md](docs/LOGS.md) lists every event and field. - -### Application logs - -In a Rails application, `HotCell::LogSubscriber` writes one `info` line per call to the Rails log: - -``` - HotCell (41.2ms) {"cell":"images","operation":"active_storage.transformers.image.vips","code":"ok","perform_ms":38,"duration_ms":41.2,"bytes_in":20480,"bytes_out":8192} -``` - -For a failed call, the line adds `cause` and `stderr` when they exist. For a call interrupted by an -exception, such as the application's own request timeout, the line has the exception's class in place of -the code. To turn the line off, call `HotCell::LogSubscriber.detach_from :hot_cell` in an initializer. -Without Rails, require `hot_cell/log_subscriber`. Then call `HotCell::LogSubscriber.attach_to :hot_cell`. -Then set `ActiveSupport::LogSubscriber.logger`. - -### Metrics collection - -The `yabeda-hotcell` gem records HotCell metrics in [Yabeda](https://github.com/yabeda-rb/yabeda). Add -the gem to the application's `Gemfile`. Call `Yabeda::HotCell.install!` once at boot: - -```ruby -# Gemfile -gem "yabeda-hotcell" - -# config/initializers/hotcell.rb -Yabeda::HotCell.install! -``` - -The metrics are in the `hotcell` group. The `requests` counter counts each call by `cell`, `operation`, -`code` and `cause`. The `perform` histogram measures the time that the cell used. On each scrape, the -gem gets `cell.metrics` from each registered cell. It sets these gauges: `up`, `running`, `queued`, -`queue_high_water`, `cancelled`, `killed` (by `cause`), and `uptime_seconds`. - -The control socket answers when the work socket is saturated. The control socket is local to its host. -Thus the scraped process must be on the same host as the cell. - -### Per-call telemetry - -The application sends the `perform.hot_cell` Active Support Notification for each call, successful or -failed. Thus the notification reports a dead cell: an unreachable socket gives the code `unavailable`. -`HotCell::LogSubscriber` and `yabeda-hotcell` subscribe to it. To record other data, subscribe to it. - -### Container healthcheck - -The installed Dockerfile wires `hotcell-health` up as the Docker `HEALTHCHECK`. It probes the -supervisor's control socket from inside the container, where `network: none` does not apply. Healthy -means the supervisor answers, not that a worker is free. If you're using your own container, -remember to use this. - -### Rails healthcheck - -`hotcell-client` defines two controllers, `HotCell::HealthController` and `HotCell::DiagnosticsController`. -Add a route for each to the application's `config/routes.rb`, as the example below shows. - -`HotCell::HealthController` asks each registered cell for `describe` and `metrics` over its control socket. -It returns `OK` with a 200 when at least one cell is registered and each cell answers. Otherwise, it -returns `FAIL` with a 503. These calls take no worker. Thus you can make the endpoint public, like `/up`. - -`HotCell::DiagnosticsController` returns the result of every check as JSON, with a 503 if any check fails. -Along with `describe` and `metrics`, it sends `health.echo` and `health.reopen` over the work socket. Each -round trip takes a worker. Put this endpoint behind authentication. + operation "images.transform" + limits deadline: 30, memory: 1280 * 1024**2 -Of the cell's two sockets, only the work socket carries file descriptors. Thus `describe` and `metrics` -succeed when the application cannot use the work socket. Only the round trips test the work socket. A -cell without the shared group passes `health.echo` and fails `health.reopen` with `EACCES`. Add -`require "hot_cell/health_operations"` to one of the cell's operation files. Without it, the cell -answers `unsupported` for both operations. + before_fork { require "my_image_processor" } -Set the diagnostics controller's superclass in an initializer. Then add the routes: - -```ruby -# config/initializers/hotcell.rb -HotCell.diagnostics_controller_parent = "Admin::BaseController" - -# config/routes.rb -get "up/hotcell" => "hot_cell/health#show", as: :hotcell_health_check - -constraints subdomain: "admin" do - get "hotcell" => "hot_cell/diagnostics#show", as: :hotcell_diagnostics + def perform(inputs, outputs, format:) + MyImageProcessor.convert inputs.first.fd_path, outputs.first.fd_path, format: format + { format: format } + end end ``` -If your authentication is a concern, subclass the controller instead: +Register the `images` cell in the application's initializer, as in +[Configure the application](#2-configure-the-application). The cell's `Gemfile` names `hotcell-server` +directly, plus whatever gems the operation uses. Put the operation's file in `hotcell/operations/`, and +install the tools that it runs in the `Dockerfile`. `config.rb` doesn't change: the operation's own +`limits` come with its class, clamped to the cell's. -```ruby -# app/controllers/hotcell_diagnostics_controller.rb -class HotcellDiagnosticsController < HotCell::DiagnosticsController - include StaffOnly -end - -# config/routes.rb -get "up/hotcell/diagnostics" => "hotcell_diagnostics#show" -``` +[Operation API](docs/operation-api.md) and [Client API](docs/client-api.md) cover the rest. -From a console, `HotCell.diagnose(work: true).as_json` returns the same checks. +## Running Hot Cell in production -## Development +Set these alerts. [Observability](docs/observability.md) explains each signal. -Working on the gems themselves is documented in [CONTRIBUTING.md](CONTRIBUTING.md): how the suites are split, -how a cell is exercised natively and in a container, and the rules that are not obvious from the code. +- **Cell availability:** the `up` gauge is 0 or absent for any cell on any host. +- **Failed calls:** the `requests` counter shifts away from `ok`, especially toward `unavailable`. +- **Queue headroom:** `queued` nears `queue_size`, `queue_high_water` rises, or `capacity` appears in + steady state. +- **Scratch space:** free space on each host's scratch runs low. +- **Cell errors:** any `ERROR` event in the cell log, or a rise in the `killed` gauge. -## Design +## Documentation -[docs/DESIGN.md](docs/DESIGN.md) holds what the code cannot tell you: the threat model, the invariants the -design exists to hold, why descriptors rather than a shared volume, and the facts that were measured rather -than reasoned about. Behavior is the code's to describe, and it does. +The docs have these sections: -[adr/](adr/README.md) describes some decisions that we arrived at during development. +- [Reference manual](docs/index.md): every part of Hot Cell, one topic per page, for agents and for + readers who want the details. +- [Design](docs/design/index.md): the intent behind Hot Cell's design, including the threat model and the + invariants that the design exists to hold. +- [Contributing to Hot Cell](docs/contributing/index.md): working on the gems, keeping the docs current, + the experiments that the design rests on, and the decision records. diff --git a/Rakefile b/Rakefile index 62c7ca9..fc6b45e 100644 --- a/Rakefile +++ b/Rakefile @@ -105,17 +105,4 @@ task :rubocop do sh "rubocop" end -task default: [ :test, :rubocop ] - -# markdown-toc is optional: a machine without it skips regeneration rather than failing. -desc "Regenerate the tables of contents in README.md, CONTRIBUTING.md and docs/DEPLOYMENT.md" -task :toc do - require "mkmf" - if find_executable0("markdown-toc") - sh "markdown-toc --maxdepth=3 -i README.md" - sh "markdown-toc --maxdepth=3 -i CONTRIBUTING.md" - sh "markdown-toc --maxdepth=3 -i docs/DEPLOYMENT.md" - else - puts "WARN: cannot find markdown-toc, skipping. Install it with 'npm install -g markdown-toc'." - end -end +task default: [ :test, :rubocop, "docs:check" ] diff --git a/activestorage-hotcell-server/test/docs_test.rb b/activestorage-hotcell-server/test/docs_test.rb new file mode 100644 index 0000000..19c4ad6 --- /dev/null +++ b/activestorage-hotcell-server/test/docs_test.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require "test_helper" + +# The reference pages describe behavior, and this fails when the limits table there stops matching the code. +class DocsTest < ActiveStorageHotCellTest + DOCS = File.expand_path("../../docs", __dir__) + OPERATIONS = File.expand_path("../lib/active_storage/hot_cell/server", __dir__) + + def test_the_limits_table_carries_each_shipped_operations_limits + assert_equal declared_limits, documented_limits + end + + private + # In a child process, because requiring the operations loads libvips, and libvips in this process would + # deadlock every cell the rest of the suite forks. + def declared_limits + script = <<~RUBY + require "json" + Dir[#{File.join(OPERATIONS, "{transformers,analyzers,previewers}", "**", "*.rb").inspect}].each { |file| require file } + concrete = HotCell::Registry.operations.reject(&:abstract_operation?) + puts JSON.generate(concrete.to_h { |operation| [ operation.operation_name, operation.limits.to_h ] }) + RUBY + output = IO.popen([ RbConfig.ruby, "-I", File.expand_path("../lib", __dir__), "-e", script ], &:read) + assert_predicate $?, :success? + + JSON.parse(output).transform_values { |limits| limits.transform_keys(&:to_sym) } + end + + def documented_limits + lines = File.readlines(File.join(DOCS, "active-storage.md"), chomp: true) + lines = lines.drop_while { |line| line != "## Limits" } + header = lines.index { |line| line.start_with?("| Routing name |") } + + lines.drop(header + 2).take_while { |line| line.start_with?("|") }.to_h do |line| + name, deadline, memory, file_size, open_files = line.delete_prefix("|").delete_suffix("|").split(" | ").map(&:strip) + + [ name.delete("`"), { deadline: Float(deadline), memory: megabytes(memory), file_size: megabytes(file_size), + open_files: Integer(open_files) } ] + end + end + + def megabytes(text) + Integer(text.delete_suffix("MB")) * 1024**2 + end +end diff --git a/adr/README.md b/adr/README.md index 4405b59..3178ece 100644 --- a/adr/README.md +++ b/adr/README.md @@ -1,14 +1,4 @@ # Architecture decision records -Decisions that were argued rather than obvious, kept so the argument does not have to be rerun. Each record -states what was decided, what it was decided against, and what it costs. - -One file per decision, numbered in the order they were accepted, named `NNNN-what-was-decided.md`. A record -is written when the decision is made and is not edited afterwards. A later decision that changes an earlier -one gets its own record and marks the earlier one superseded. - -| # | Decision | Status | -| --- | --- | --- | -| [0001](0001-reuse-workers-across-requests.md) | A worker may serve more than one request, and serves one by default | Accepted | -| [0002](0002-keep-the-warm-slot-home.md) | Keep the warm slot home, as an accepted trade-off | Superseded by [0003](0003-remove-the-persistent-slot-home.md) | -| [0003](0003-remove-the-persistent-slot-home.md) | Remove the persistent slot home; a request's `$HOME` is made and removed with it | Accepted | +The index of these records, and how to write one, is in +[Architecture decision records](../docs/contributing/decisions.md). diff --git a/bin/conformance b/bin/conformance index 20f6269..b0a4c5f 100755 --- a/bin/conformance +++ b/bin/conformance @@ -57,11 +57,11 @@ trap cleanup EXIT # A named volume inherits the ownership of the image's directory, which is why the base image creates # /run/hotcell owned by the cell's user. A bind mount would keep the host directory's ownership, and the -# cell then could not create its own socket. See docs/DEPLOYMENT.md. +# cell then could not create its own socket. See docs/container.md. docker volume create "$VOLUME" >/dev/null # The security flags the accessory sets, added one at a time so a negative job can leave exactly one out. -# `keep NAME arg...` appends the flag unless it is the one named in DROP. No --ulimit stack: docs/DEPLOYMENT.md +# `keep NAME arg...` appends the flag unless it is the one named in DROP. No --ulimit stack: docs/container.md # forbids it — a lowered stack turns an overflow into a SIGSEGV the cell reports as killed/crashed, a transient # verdict the caller retries against a limit that will fail it again, so the helper would measure a shape # operators are told not to deploy. diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md deleted file mode 100644 index ddc830c..0000000 --- a/docs/DEPLOYMENT.md +++ /dev/null @@ -1,669 +0,0 @@ -# Deploying a cell - -A cell is a second container beside the application, on the same host, sharing one volume that holds its -UNIX sockets. The README carries the complete Kamal configuration for both halves, under -[Configure and deploy the HotCell container](../README.md#configure-and-deploy-the-hotcell-container). - -This document explains that configuration: how the image is built, what every flag and setting does, -how the numbers constrain each other, and one alternative worth knowing about. The examples use Kamal; -any orchestrator that can set the same `docker run` flags will do. - -[docs/TUNING.md](TUNING.md) is a companion document. It covers measuring your way to values for your -own workload, where this one covers what the values mean. - - - - - -- [Building an image](#building-an-image) - * [Using the installed Dockerfile](#using-the-installed-dockerfile) - * [Bringing your own container](#bringing-your-own-container) -- [Container settings](#container-settings) - * [Performance](#performance) - * [Security](#security) - * [General](#general) - * [Bound the OpenMP thread pools](#bound-the-openmp-thread-pools) - * [Sizing the numbers](#sizing-the-numbers) -- [Cell settings](#cell-settings) - * [Performance](#performance-1) - * [Security](#security-1) - * [Changing a shipped operation's limits](#changing-a-shipped-operations-limits) - * [Settings that trade one for the other](#settings-that-trade-one-for-the-other) -- [Application settings](#application-settings) - * [The socket's file mode](#the-sockets-file-mode) - * [The group both sides share](#the-group-both-sides-share) -- [Making the numbers agree](#making-the-numbers-agree) -- [Where scratch lives](#where-scratch-lives) - * [Keeping the tmpfs](#keeping-the-tmpfs) - * [A named volume](#a-named-volume) - * [A host-mounted filesystem](#a-host-mounted-filesystem) - * [What changes in the numbers](#what-changes-in-the-numbers) - - - -## Building an image - -### Using the installed Dockerfile - -There is no published base image to derive from. `bin/rails hotcell:install` writes a `hotcell/` -directory into the application holding a complete `Dockerfile`, the cell's `Gemfile`, its -`config.rb`, and an `operations/` directory. That Dockerfile is the whole recipe, and it is yours to -customize. Build it from its own directory: - -``` -docker build -t your-image:latest hotcell/ -``` - -The README's [Configure and deploy the HotCell -container](../README.md#configure-and-deploy-the-hotcell-container) walks through the mechanical -edits, but here are some additional tips: - -**Every gem is inside the blast radius.** Keep the cell's `Gemfile` short. A larger supervisor heap also -costs every request — see "Sizing the numbers". - -**Load only the operations your image has tools for.** A cell advertises what it loaded, on `describe`, -and a client checks that inventory at boot to catch a cell that does not carry the operation it wants. An -operation whose tool is missing makes that check pass and fails at the first request instead. So requiring -an operation you did not install a tool for is worse than not requiring it. - -**The cell's image must create its own socket mount point**, owned by the cell's user — `/run/hotcell/cell` -and not only `/run/hotcell`. Docker creates a missing last level as root, and the cell then cannot create a -socket in it. The installed Dockerfile does this. The application's image needs nothing at its own mount -point. - -**Keep the app's and the cell's lockfiles in step.** They resolve hotcell separately. -`HotCell.describe_cells` warns at boot when the cell's `hotcell-server` version differs from the app's -`hotcell-client` version. - - -### Bringing your own container - -If you prefer to use your own base image to run HotCell, that's fine! - -#### Conformance checks - -`bin/conformance IMAGE` checks whether an image supports hotcell. It boots the image with the flags -under "Container settings", mounts the example operations over `/hotcell/operations`, and drives a -battery of checks from a second container over a shared volume: descriptor round-trips, each kill -verdict, refusal at capacity, and the isolation flags. It exits non-zero on the first failed check. - -It then runs the same battery again against your image booted without one security flag at a time, and -requires each run to fail at that flag's own check. That is what makes the isolation results a test of -the flags rather than a restatement of them: a check that cannot see a flag would pass a cell running -without it. A runtime that force-mounts `noexec` onto every tmpfs — Docker Desktop does — cannot host -the `noexec` negative, and that one run reports `SKIP` with the reason. - -``` -docker build -t my-cell:test hotcell/ -bin/conformance my-cell:test -``` - -The mount shadows the operations your image carries. What the checks cover is the image's runtime — its -Ruby, its gems, and its socket behaviour — not the work your operations do. - -`bin/example-image` builds an image to try it against. CI runs both on every push. - -#### Additional requirements - -Three things conformance cannot prove for you. - -**`kernel.yama.ptrace_scope >= 1` on the host.** No container flag can supply it, and it is what stops one -worker reading another request's memory through `/proc//mem`. A cell refuses to boot below it: - -``` -kernel.yama.ptrace_scope is 0 on this host, so one worker can read another request's memory -through /proc//mem. No container flag can set it. Set it to 1 or higher and boot again. -``` - -A host that does not expose the file at all logs `cell.ptrace_scope_unknown` and boots anyway. - -**The security flags it cannot observe.** It reads six from inside the cell: `network: none` (the only -interface is `lo`), `read-only` (the root filesystem is mounted `ro`), the tmpfs `noexec` flag, `cap-drop` -(the bounding capability set is empty), `no-new-privileges`, and the uid. A flag the cell cannot read fails -the check rather than passing it. It does not observe the tmpfs `nosuid` and `nodev` flags, `pids-limit`, -or the resource limits. Those are yours to get right, and "Verifying your accessory" is how. - -**Your application's group membership.** Conformance does check the shared group, because it runs the cell -as `10001` against files a different user owns: `health.reopen` proves a cell can open an input by name, -and `example.tamper` proves it can do nothing else with either file. What it cannot reach is the -application's own side — its driver owns the files as root, which needs no membership to set a group. That -one fails as `EPERM` in the application rather than in the cell. - -#### Verifying your accessory - -`bin/conformance` supplies its own docker flags, so it proves the image works when the flags are right. -It doesn't read your Kamal configuration, and it passes against an image you are about to deploy with -`cap-drop` missing. - -To check your accessory before you deploy it, print the command Kamal will run. `kamal config` does not -answer this. It prints merged configuration, not the command, so a `--network` emitted twice does not -appear there at all. - -``` -bundle exec ruby -rkamal -e ' - config = Kamal::Configuration.create_from( - config_file: Pathname.new("config/deploy.yml"), destination: "production", version: "check") - puts Kamal::Commands::Accessory.new(config, name: :images).run.flatten.join(" ") -' -``` - -Read `--network` off that line. It must appear once, as `--network none`. Two of them is the `options:` -mistake described under "Container settings", and Docker rejects the container at boot with exit -status 125. - -To check a deployed cell, read the flags on the running container: - -``` -docker inspect --format '{{json .HostConfig}}' | jq '{ - NetworkMode, ReadonlyRootfs, CapDrop, SecurityOpt, PidsLimit, Memory, MemorySwap, Tmpfs, Binds -}' -``` - - -## Container settings - -These are `docker run` flags. Under Kamal they go in an accessory's `options:`, and Kamal supplies no -value of its own for any of them. - -`network` is the one exception, and it is the flag this design depends on most. It is an accessory key of -its own, a sibling of `image:` and `roles:`. Kamal always emits a `--network` of its own, `kamal` by -default. An entry under `options:` adds a second `--network` rather than replacing the first, and Docker -refuses the container: - -``` -docker: conflicting options: cannot attach both user-defined and non-user-defined network-modes -``` - -The complete accessory, both halves, is in the README under -[Configure and deploy the HotCell container](../README.md#configure-and-deploy-the-hotcell-container). -Copy it from there. The tables below say what each flag does and how to size it. - -Give each cell its own volume. Two accessories sharing one would write `work.sock` over each other. - -### Performance - -**These have no defaults.** Docker applies no limit to a flag you omit, so a cell without them can take -the whole host. Tune each one to your workload and your hardware. The values below are the worked example -from the README's accessory, not recommendations. - -| Flag | Example | Tune it from | -| --- | --- | --- | -| `cpus` | `2` | The share of the host this cell may use. Start the cell's `concurrency` at twice this number, and match the image's `OMP_NUM_THREADS` to it — see "Bound the OpenMP thread pools". | -| `memory` | `2g` | The cgroup limit, counting every worker and the tmpfs. Size it from `concurrency × peak RSS` plus the tmpfs. Keep it above the cell's `memory`. On a disk-backed scratch there is no tmpfs term — see "Where scratch lives". | -| `memory-swap` | `2g` | Set it equal to `memory`. Omit it and Docker allows twice `memory` in swap, so the memory limit no longer holds. | -| `tmpfs` size | `size=512m` | Scratch for all concurrent workers together. It pairs with `file_size × concurrency`. Moving scratch onto disk decouples it from `memory` — see "Where scratch lives". | -| `ulimit: stack` | leave it unset | A container inherits the Docker daemon's value, normally 8MB, and that is where to leave it. Lowering it buys a worker about 24MB more room at 2MB, and nothing at all for an operation that shells out. It costs far more than it buys: a thread that overflows the smaller stack dies on `SIGSEGV`, and the cell reports that as `killed`/`crashed`, which is transient — so the request is retried against a limit that will fail it again for as long as the caller's job keeps trying, and nothing in the verdict points at the setting that caused it. Raise the cell's `memory` instead. It belongs here rather than in `config.rb` because glibc reads it at exec, before `Process.setrlimit` could run. | - -### Security - -**Use these values.** They are what a cell is for. Omit one and the protection is gone, while the cell -serves requests exactly as before. - -| Flag | Recommended | What it does | -| --- | --- | --- | -| `network` | `none` | Removes every network interface. A tool that is persuaded to fetch a URL cannot reach anything. Set it as an accessory key, not under `options:`. | -| `read-only` | `true` | Makes the root filesystem read-only. A cell writes only to `/tmp` and to the socket volume. | -| `tmpfs` flags | `nosuid,nodev,noexec` | `noexec` stops a dropped binary running from `/tmp`, which is where a request's files live. It does not cover the socket volume, and it does not stop `ruby payload.rb`. On a disk-backed scratch Docker sets none of them — see "Where scratch lives". | -| `cap-drop` | `ALL` | Removes every Linux capability. | -| `security-opt` | `no-new-privileges:true` | Prevents a setuid binary from regaining what `cap-drop` removed. | -| `user` | `10001:10001` | Runs the cell with no home directory and no shell. Without user namespace remapping this is a host uid, and it is inside the ordinary range, so pick one your hosts do not give to a person. | -| `pids-limit` | `512` | Bounds the number of processes in the cell. It must clear `concurrency` plus the threads and subprocesses one toolchain starts, so check it when you raise `concurrency`. | - -### General - -Environment variables. The image sets all of them unless the table says otherwise, so set one only to -override it. - -| Variable | Default | What it does | -| --- | --- | --- | -| `HOTCELL_DIR` | `/run/hotcell/cell` | Where the cell creates `work.sock` and `control.sock`. The app must use the same directory. | -| `HOTCELL_OPERATIONS` | `/hotcell/operations` | The directory the cell loads at boot, in sorted order. | -| `HOTCELL_CONFIG` | `/hotcell/config.rb` | Loaded before the operations, if the file exists. | -| `HOTCELL_WORKSPACE` | a directory under `TMPDIR` | Where each request's directory is made and removed. On the default accessory this is the tmpfs. Must be absolute. Emptied at boot — see "Where scratch lives". | -| `TMPDIR` | unset, so `/tmp` | The scratch, emptied at boot of every entry the cell's uid owns. On the accessory `/tmp` is the cell's own mount. Under `hotcell --development` it is only the scratch's parent and is never swept — see "Where scratch lives". | -| `HOTCELL_HEALTH_TIMEOUT` | `5` | Seconds `hotcell-health` waits for an answer before it reports unhealthy. | -| `HOME` | `/tmp` | Bundler needs one, and the cell's user has no home directory. A worker replaces it with a directory made for the request and removed with it. | -| `OMP_NUM_THREADS` | `2` | The OpenMP pool size libvips and ImageMagick use. Match it to `cpus`. See "Bound the OpenMP thread pools". | -| `OMP_THREAD_LIMIT` | `8` | The ceiling on that pool, including a library that raises the count itself. | -| `MAGICK_MEMORY_LIMIT`, `MAGICK_MAP_LIMIT`, `MAGICK_DISK_LIMIT` | unset | ImageMagick's pixel cache limits. The scaffold installs no ImageMagick; an image that does sets these from the container's `memory`, the scratch and `concurrency`. See [docs/IMAGEMAGICK.md](IMAGEMAGICK.md). | - -### Bound the OpenMP thread pools - -**Set `OMP_NUM_THREADS` and `OMP_THREAD_LIMIT` in the image.** The installed `Dockerfile` sets both, and -on a large host neither is optional: without them a cell dies there. `hotcell:install` leaves an existing -`Dockerfile` untouched, so add both by hand to a cell installed before this and rebuild its image. - -OpenMP sizes its thread pool from the host's core count, and a `cpus:` limit is a CFS quota rather than an -affinity mask, so it does not lower that count — on a 98-core host libvips and ImageMagick ask for 98 -threads. Each thread stack is 8MB of private anonymous memory, which the cell's `memory` charges as -`RLIMIT_DATA`: under a 1280MB limit, 96 of those stacks fit and 104 do not. Past that line `pthread_create` returns `EAGAIN`, which libgomp treats as unrecoverable — it writes - -``` -libgomp: Thread creation failed: Resource temporarily unavailable -``` - -to stderr and calls `exit(1)`. The worker dies before answering, so the caller gets a transient failure and -the job retries it against a host that fails the same way. This killed 285 Basecamp workers in production -on 2026-08-31 and forced a rollback. - -That line reaches a log now: the supervisor captures what a worker writes to fd 2 and attaches its tail to -the `worker.killed` that reports its death, as `hotcell.stderr`, and to the failure the caller receives. See -`docs/LOGS.md`. - -**Size `OMP_NUM_THREADS` from the container's `cpus`:** the number follows the allocation, not the example -above. It is a thread count, so round a fractional quota down, and up to 1 below that. `OMP_THREAD_LIMIT` is the backstop against a library that raises the count itself by calling -`omp_set_num_threads`, which is what ImageMagick does for `MAGICK_THREAD_LIMIT`. - -**The cell forwards the bound to the tools it execs.** A tool sees the environment its operation wrote for -it rather than the worker's own, which is invariant 9, so the image's variables alone would bound -in-process libvips and nothing else. `Operation#run_tool` and mini_magick both carry the pair from the -cell's environment. - -**A deploy to staging or beta cannot catch a regression here.** The failure exists only at production's -core count — which is how it reached production. So the guard is a test: -`hotcell-client/test/install_test.rb` holds the scaffold's variables, and an image you customize needs its -own. - -To verify a built image, write a GIF inside it and count `/proc/self/task` during the write. GIF is the -cheapest probe: it is the only common output format that quantizes, and quantization is where the threads -appear. Unbounded, the count tracks the visible cores; bounded, it stays at the limit. - -### Sizing the numbers - -These numbers are arithmetic against the container flags under "Container settings". Against the -`cpus: 2`, `memory: 2g`, `size=512m` accessory there: `concurrency: 4` because a request spends much of -its life off the CPU, so twice `cpus` is where to start; `file_size: 48MB` because that bounds what one -worker writes; and `memory: 1536MB` because that is the measured working value for `RLIMIT_DATA`. - -**Size the cell to its most demanding operation.** An operation's own `limits` are clamped to the cell's, -so the cell's numbers only ever take away. Read the `limits` that each operation you carry declares, and -set the cell above the highest of them. The shipped video previewer asks for `deadline: 120` and -`file_size: 128MB`. A cell configured with the 30 seconds and 48MB above kills every video preview, and -nothing else, which is a hard failure to place. - -Some guidelines for reading those numbers: - -**`memory` does not multiply by `concurrency`.** It is an address-space charge on one worker. About 620MB -of it is reserved and never touched, and about 450MB of that is Ruby's own reservation, which -`RLIMIT_DATA` charges in full. Subtract 450MB before you read `memory` as the amount an input may consume. -The cgroup limit is what bounds real memory across the cell. - -**An input is charged only when an operation asks for its path.** A descriptor that an operation reads in -place costs no tmpfs and no `file_size`, so a multi-gigabyte upload can be analyzed under a small -`file_size`. An operation that needs a filename copies the input onto scratch first, and the kernel -charges that copy exactly as it charges an output. Size `file_size` from the largest thing your operations -write, and count a staged input as one of them. - -**A large supervisor makes every request slower.** A worker's fixed cost is copy-on-write settling, and it -is proportional to the supervisor's resident heap. A `before_fork` that requires more than the cell's own -operations need is paid on every request for the life of the deployment. - -## Cell settings - -One call, read at boot. Every value has a default, so set only what you are changing. - -```ruby -HotCell.limits concurrency: 4, queue_size: 8, queue_wait: 10, deadline: 30, - max_requests_per_worker: 1, control_deadline: 5, - memory: 1536 * 1024**2, file_size: 48 * 1024**2, open_files: 256 -``` - -### Performance - -| Setting | Default | What it does | -| --- | --- | --- | -| `concurrency` | `4` | Workers that run at once, and the number of slots. Start at twice `cpus`. See "Settings that trade one for the other". | -| `queue_size` | `8` | Connections that may wait for a worker. When `running + queued` reaches `concurrency + queue_size`, the cell answers `capacity`. Use `0` to refuse instead of queueing. | -| `queue_wait` | `10` | Seconds a queued connection may wait before the cell answers `capacity`. This makes a saturated cell answer with a verdict instead of holding the caller until its own timeout. | -| `control_deadline` | `5` | Seconds a control connection may take to send its request. | -| `sweep_interval` | `10` | How often, in seconds, the supervisor looks for the directories that killed requests left behind. When it finds one, it forks a sweeper process to delete them. The sweeper runs under `deadline` like a worker, so the supervisor and the requests in flight never wait on the deletion. | -| `max_requests_per_worker` | `1` | Requests one worker serves before the cell discards it. `1` forks per request. `:unlimited` keeps a worker for the life of the cell. See "Settings that trade one for the other". | - -### Security - -| Setting | Default | What it does | -| --- | --- | --- | -| `deadline` | `60` | Maximum wall-clock seconds for one request. The supervisor kills the worker's process group and answers `killed: deadline`. This is the only bound on a wedged or deliberately slow input. There is no CPU limit; `HotCell::Limits` gives the reason. | -| `memory` | `1536MB` | `RLIMIT_DATA` per worker, and the bound on a decompression bomb. A breach gives `killed: memory`. The floor is 1GiB, and a cell below it does not boot. | -| `file_size` | `64MB` | `RLIMIT_FSIZE` per worker. A breach raises `SIGXFSZ` and gives `killed: fsize`. | -| `open_files` | `256` | `RLIMIT_NOFILE` per worker. | - -These four are sized like performance settings and exist as limits on a hostile input. - -An operation can declare `limits deadline:, memory:, file_size:, open_files:` of its own. Those values -narrow the limit for that request. They never widen it, because the cell's numbers are the ceiling. - -### Changing a shipped operation's limits - -An operation's `limits` is a class-level declaration that accumulates: naming one limit changes that one -and keeps the rest. So an operator can give a shipped operation a different budget without editing the -gem, by redeclaring the one number after the operation loads: - -```ruby -# hotcell/operations/zz_limits.rb -require "active_storage/hot_cell/server/transformers/image/vips" - -ActiveStorage::HotCell::Server::Transformers::Image::Vips.limits file_size: 128 * 1024**2 -``` - -The transformer's `deadline`, `memory` and `open_files` are unchanged. Naming a limit to `nil` withdraws -it, which hands that one back to the cell's number. - -Three things about that file. - -**Require the operation first.** The class has to exist before it can be redeclared. `config.rb` loads -before any operations file, and operations load in sorted order, so a redeclaration that relies on load -order alone belongs in an operations file that sorts last — hence the `zz_` prefix. The `require` at the -top removes that dependency: with it, the same two lines work from any operations file, or from -`config.rb`. Prefer an operations file anyway, so every declaration about operations lives in one place. - -**A subclass works the same way, and copies at the moment it declares.** One that declares -`limits deadline: 5` takes every other value from its parent, so a narrowing subclass writes the number it -narrows and nothing else. It takes them once: `limits` resolves to the first ancestor that declared any, -and once a class has stored its own it stops looking up. So redeclaring the parent afterwards does not -reach a child that has already declared — redeclare the child too. A subclass that never declares follows -its parent live. - -**The cell's numbers are still the ceiling.** A redeclaration can only ask; the cell clamps it the way it -clamps the original, so the effective value is the smaller of the two. To receive 128MB the cell's own -`file_size` has to allow at least 128MB. A cell at 64MB gives that redeclaration 64MB — a change, but not -the one asked for. - -For reference, the shipped Active Storage operations declare: - -| Operation | `deadline` | `memory` | `file_size` | `open_files` | -| --- | --- | --- | --- | --- | -| `transformers.image.vips`, `transformers.image.magick` | 30 | 1280MB | 48MB | 256 | -| `analyzers.image.vips`, `analyzers.image.magick` | 10 | 1024MB | 48MB | 64 | -| `analyzers.media.ffprobe` | 30 | 1024MB | 48MB | 128 | -| `previewers.pdf.mutool`, `previewers.pdf.poppler` | 30 | 1024MB | 48MB | 128 | -| `previewers.video.ffmpeg` | 120 | 1536MB | 128MB | 128 | - -The 48MB is arithmetic from the example accessory — four workers, two staged files each, on a 512MB -tmpfs — and not a property of any operation. An operation that reads its input through the descriptor -stages only its output, so it needs half of what that arithmetic assumed. - -### Settings that trade one for the other - -**`max_requests_per_worker` above `1`** lets one request reach another. A worker holds each of its -requests in the same address space, so an input that runs code can read and change every later request -that worker serves. [ADR 0001](../adr/0001-reuse-workers-across-requests.md) records the measurements and -the trade. - -At `:unlimited` it also gives up the deadline. A worker reports itself idle when it finishes, and the -supervisor cannot tell a true report from a false one, so a worker that lies stops being timed. At every -finite setting it is still retired and killed on a grace period; at `:unlimited` it is retired by nothing, -and the cell cannot end it. Prefer a finite number, however large. - -**`concurrency`** sets how many requests hold bytes in the cell at once. Files are not isolated between -concurrent workers, so this value is also the width of that exposure. -[docs/DESIGN.md](DESIGN.md) gives the measurements. - -## Application settings - -Per registered cell. They set how the application responds to what a cell answers. - -```ruby -HotCell.root = ENV["HOTCELL_ROOT"] # unset turns every cell off -HotCell.group = ENV["HOTCELL_GROUP"] # the cell's gid; unset where both sides are one user - -HotCell.register "images", - timeout: 30, - permanent: ActiveStorage::PreviewError, - transient: MyApp::ConversionTemporarilyUnavailable, - on_contract_skew: ->(error, cell) { Sentry.capture_exception(error) } -``` - -| Setting | Default | What it does | -| --- | --- | --- | -| `HotCell.root` | — | The parent directory that cell names resolve under. When it is unset, every cell is off and callers run in process. | -| `HotCell.group` | — | The group both sides hold, so a cell can open a caller's file by name. Set it to the cell's gid. Unset is for an installation whose two sides already run as one user, which is how development runs. See "The group both sides share". | -| `dir:` | from `root` | An explicit socket directory for one cell. Give a lambda to make a change of path a configuration change instead of a deploy. | -| `timeout:` | `30` | Seconds this caller waits for an answer to a work request. It must clear the cell's `answer_within`. A positive, finite number: `register` raises `ConfigurationError` on anything else, including `nil`, which stops the boot rather than leaving the wait for a cell to decide. | -| `control_timeout:` | `5` | Seconds this caller waits for `describe` or `metrics`. The supervisor answers both inline, with no fork and no queue, so this is short on purpose and must not be raised toward `timeout`. It is what bounds app boot when a cell accepts connections and never answers, and what lets a health check report a wedged cell instead of waiting out a work timeout. Bounded like `timeout:`, and refused the same way. | -| `permanent:`, `transient:` | the gem's classes | The exception classes the application raises for each side of the split. `transient` must not descend from `permanent`, and the client refuses to start if it does. | -| `on_contract_skew:` | — | Called when a cell answers `protocol`, so a protocol version mismatch is visible to an application that rescues broadly. | - -### The socket's file mode - -The cell creates both sockets `0660`, owned by its own user and group. A Unix socket needs write -permission to connect, so the shared group in the next section is what lets your application reach a cell -at all. It is not optional. - -Without it every call fails with `EACCES` from `connect`, and `HotCell.describe_cells` says so at boot, -before any traffic depends on it. - -### The group both sides share - -Set `HotCell.group` to the cell's gid, and put the application in that group: - -```yaml -# config/deploy.yml — the app -servers: - web: - options: - group-add: 10001 -``` - -**Why it is needed.** Two reasons. - -The sockets are `0660` and owned by the cell's user and group, so without the group your application -cannot connect to a cell at all. - -And an operation that hands a tool a filename does not copy the input. It re-opens the descriptor as -`/dev/fd/N`. That is a fresh open, and the kernel rechecks it against the **cell's** user rather than the -caller's. The two sides do not share a uid, so a mode `0600` file the application owns gives `EACCES`. An -Active Storage tempfile is exactly that. - -Every shipped Active Storage operation hands a tool a filename, so this applies to all of them. - -**What the client does with it.** It puts each descriptor in the group and sets the mode on the way out: -`0640` for an input, `0620` for an output. It does this through the open file rather than a path, so it -names nothing. - -**It sets those and does not put them back.** The file you hand over keeps the cell's group and the new -mode after the call returns, so a `0600` file of your own comes back readable by that group. Active -Storage hands over tempfiles and unlinks them, so this is invisible there. If you write your own client, -hand over files you are willing to share with the cell's group, and do not pass one whose permissions -something else depends on. - -**What that buys, and what it does not.** A cell may read an input and write an output. It may not write -an input or read an output, and it cannot widen either, because it does not own these files and -`cap-drop ALL` leaves it no capability that overrides a mode. The application must own them and be in the -group, which is what lets it set the group at all. - -**Do not share a uid instead.** It is the obvious shortcut and it costs two things. The cell would own the -files, so it could set any mode it liked and the one-way rule would stop holding. And a cell that escaped -its container would land on the application's own user on the host, where it can read the environment of -the application's processes. Keep the uids apart and share only the group. - -**How it fails.** Immediately, on every call, with `EACCES` from `connect`. There is no partly working -state to misread as healthy. - -**What warns first.** `HotCell.describe_cells` checks two things at boot, and warns rather than raising, -like every other boot check here. - -- **Whether this process holds `HotCell.group`.** A missing `group-add` is then visible before any traffic - depends on it. This check is local and needs no cell, so it reports even when every cell is down. -- **Whether the cell runs in that group.** `describe` reports the groups a cell holds, and the client - compares them. The number lives in your deploy file and the cell's gid is baked into an image built - somewhere else, so nothing else would catch a cell image that moved its gid. A cell too old to report - them says nothing. - -Neither check trusts what the cell says. The process answering runs untrusted content, so its description -is read inside a rescue: an answer this client cannot use is logged and ignored, and the cell is treated as -one that answered nothing. What that catches is a misdeployment, not a compromised cell — a cell that has been taken over can answer a perfectly well-formed -description that is simply false, and nothing on this side can tell. - -What the rescue covers is what a cell *says*. It does not cover a cell that never answers at boot: -`control_timeout` bounds the answer and not the connection, so a listener that stops accepting until its -backlog fills can hold the connect. That is [#20](https://github.com/basecamp/hotcell/issues/20). - -And it covers what raises at boot, not what a value does later. A description is handed back whole, so a -cell can put a string in it that is not valid UTF-8, or a number that parses to an infinity. Neither raises -here. The whole returned description stays untrusted: validate it for whatever you do with it — serializing, -matching, storing or rendering are all places these values raise. - -## Making the numbers agree - -Nothing checks these for you. - -- The client's `timeout` must be more than the cell's `answer_within`, which is - `queue_wait + deadline + 1`. Below it, a saturated cell reaches the caller as a transport failure - instead of as `capacity` or `killed`. The client warns at boot, and `describe` reports the number. -- The cell's `memory` must be less than the container's `memory`. At equal values the cgroup fires - first, and a cgroup kill is a `SIGKILL` with no diagnostic. -- `file_size × concurrency` must be no more than the scratch. Above it, concurrent workers fill the - scratch and requests fail with `ENOSPC` instead of with a limit verdict. On the default accessory the - scratch is the tmpfs, and its `size=` is the number to fit. -- `concurrency × (MAGICK_DISK_LIMIT + everything else one worker writes on scratch)` must be no more - than the scratch. Above it, concurrent ImageMagick processes fill the scratch before any of them - refuses a frame, and `file_size` cannot prevent that, because it bounds each cache file and not their - sum. See [docs/IMAGEMAGICK.md](IMAGEMAGICK.md). - -Three things are fixed and cannot be configured: the one-second grace between the signal to a worker and -the kill of its process group, the absence of an `RLIMIT_CPU`, and the socket file mode. - -## Where scratch lives - -The configuration documented above uses RAM-backed tmpfs for scratch space. This couples three -configuration params: an operation's `FSIZE` (max file size write), and the `tmpfs` and `memory` -container configs. Increasing `FSIZE` require increasing the other two, which may end up putting -memory pressure on the host node. - -Scratch is `/tmp`: staged inputs, staged outputs, and every intermediate a tool writes. Those pages -are charged to the container's cgroup and, with `memory-swap` equal to `memory`, they cannot be -reclaimed or swapped — so the tmpfs `size=` and the container's `memory` grow together, byte for -byte. Every time `file_size` goes up to admit a larger conversion, `memory` pays for it. - -Moving scratch onto disk breaks that coupling. Files written there pass through the page cache, and the -kernel writes those pages back and drops them under pressure, so a large file on scratch can no longer -OOM the cell and `memory` sizes from `concurrency × peak RSS` alone, kept above the cell's own `memory` -rlimit. - -Three layouts, trading the same three things: - -| Layout | Backed by | Size cap | `nosuid,nodev,noexec` | Provisioning | -| --- | --- | --- | --- | --- | -| tmpfs | RAM, charged to the cgroup | the `size=` option | yes | none | -| named volume | disk, under `/var/lib/docker/volumes` | none | none | none | -| host mount | disk | the filesystem's size | from the host mount | per host | - -Both disk layouts give up speed, and that is rarely the bottleneck: on NVMe, for spool-and-process -pipelines, the descriptor design already keeps the biggest bytes off scratch entirely. Both also outlive -the container, so cleanup stops being the mount's job and becomes the cell's. At boot it empties -`TMPDIR` and the workspace's parent of every entry its uid owns, `lost+found` excepted, so rebooting -the accessory clears a scratch a killed tool filled. A removal that fails is logged as `scratch.unswept` -and does not stop the boot. The cell refuses to boot if either directory is missing, is reached through a -symlink its uid owns, or holds `HOTCELL_DIR`. - -The rule is the same in every layout, so a cell's `TMPDIR` must be a directory nothing else uses. In the -container `/tmp` is the cell's own; on a developer's machine it is everyone's, so an uncontainerized cell -is booted with `hotcell --development`, which never sweeps the directory it is given. Its scratch is a -directory of its own beneath it, named for `HOTCELL_DIR`. An explicit `HOTCELL_WORKSPACE` has its parent -swept wherever it points, flag or not. - -### Keeping the tmpfs - -Right when scratch is small and stays small. It is RAM-fast, it vanishes when the container stops, and one -flag carries the size cap and all three security flags. It is the only layout where Docker sets those -flags for you. - -### A named volume - -The way onto disk that needs nothing of the host. Replace the `tmpfs:` option with a named volume on the -same path: - -```yaml -# config/deploy.yml — the cell, changed from the README's accessory -accessories: - active_storage: - volumes: - - hotcell-sockets:/run/hotcell/cell # unchanged - - hotcell-scratch:/tmp # scratch, on disk - - options: - memory: 2g # no tmpfs term: concurrency × peak RSS - memory-swap: 2g # equal to memory - # tmpfs: gone. /tmp is the named volume above. -``` - -Docker creates the volume on first boot and initializes it from the image's own `/tmp` — the content and -the permissions both, which is `1777` on the Debian base — so the cell's uid can write to it with no host -work at all. No `mkdir`, no `chown`, no fstab line. That is the same rule the socket volume already -depends on, and the installed `Dockerfile` records it. - -What you give up is the cap and the flags, and Docker can set neither on a named volume. A runaway write -is then bounded only by the `file_size` rlimit per file and by deadline × disk throughput, and a fill -lands on whatever filesystem holds `/var/lib/docker` — usually the root filesystem. The `isolation` -operation reports `scratch_noexec: false` against a cell configured this way, which is the truth. - -### A host-mounted filesystem - -The way onto disk that keeps the cap and the flags. Give the accessory an absolute host path, which Kamal -passes to `docker run -v` verbatim as a bind mount: - -```yaml -# config/deploy.yml — the cell, changed from the README's accessory -accessories: - active_storage: - volumes: - - hotcell-sockets:/run/hotcell/cell # unchanged - - /var/lib/hotcell-scratch:/tmp # scratch, from the host - - options: - memory: 2g # no tmpfs term: concurrency × peak RSS - memory-swap: 2g # equal to memory - # tmpfs: gone. /tmp is the bind mount above, and its nosuid,nodev,noexec - # must come from the host mount. -``` - -**Give it a dedicated filesystem** — a partition, an LV, or a loopback image file. Its size is then the -cap, so a fill stays inside scratch and reaches the caller as `Errno::ENOSPC`, which the cell classifies -`failed`: transient, retried, never recorded against a blob. The flags ride the host mount options, and a -bind mount carries its source's flags into the container, so `scratch_noexec` still reports true. - -A plain host directory is not worth the trouble. It gives up the cap and the flags exactly as a named -volume does, and adds the `chown` work that a named volume avoids. - -**Chown the source.** A bind mount keeps the host directory's ownership rather than taking the image's, -and the cell runs as `10001:10001`. Docker creates a missing source owned by root, and the cell then fails -every request with `EACCES`. Create and chown it before the first boot. As a loopback image — no -repartitioning, and sized from `file_size × concurrency` plus headroom: - -``` -fallocate -l 8G /var/lib/hotcell-scratch.img -mkfs.ext4 /var/lib/hotcell-scratch.img -mkdir -p /var/lib/hotcell-scratch -echo '/var/lib/hotcell-scratch.img /var/lib/hotcell-scratch ext4 loop,nosuid,nodev,noexec 0 0' >> /etc/fstab -mount /var/lib/hotcell-scratch -chown 10001:10001 /var/lib/hotcell-scratch -``` - -**Do not use the host's `/tmp` as the source.** On most systemd distributions it is itself a tmpfs, which -puts scratch back in RAM — still charged to the cell's cgroup, because the writer pays — and -`systemd-tmpfiles` reaps old files out from under long requests. Check the source before you use it: - -``` -findmnt -T /var/lib/hotcell-scratch -o TARGET,SOURCE,FSTYPE,OPTIONS -``` - -`FSTYPE` must be a disk filesystem, and `OPTIONS` shows which of the three flags the cell will actually -get. - -### What changes in the numbers - -The `file_size × concurrency` arithmetic under "Making the numbers agree" does not disappear. On either -disk layout `memory` loses its tmpfs term, and the number that has to fit becomes the host filesystem's -size — or, on a named volume, nothing, because there is no cap to fit inside. - -Mind one interaction. If you also raise or remove `file_size` so that large conversions succeed, a cap is -the only bound left on what a runaway write consumes; without one the bound is deadline × disk throughput. -An uncapped layout with an uncapped `file_size` is the one combination with no bound at all, and a capped -filesystem is what makes a generous `file_size` safe to run. - -Two things do not change on any layout: - -- `HOTCELL_WORKSPACE` keeps its default. It lives under `Dir.tmpdir`, and the server treats scratch as a - plain directory without ever checking the filesystem type. -- A full scratch still reaches the caller as `failed`, which is transient, exactly as a full tmpfs did. diff --git a/docs/DESIGN.md b/docs/DESIGN.md deleted file mode 100644 index 4037d29..0000000 --- a/docs/DESIGN.md +++ /dev/null @@ -1,392 +0,0 @@ -# HotCell — design - -The code is the specification. This document holds the parts of the design that reading the code cannot -give you: why the work moves out of the application process at all, what a cell is defended against, the -invariants the whole thing exists to hold, and the facts that were measured rather than reasoned about. - -It deliberately does not describe the gems' APIs. That description used to live here, it went stale as -soon as the code moved, and worse, its errors propagated back into the code's own comments. Behavior -belongs in the code and its tests; [CONTRIBUTING.md](../CONTRIBUTING.md) covers working on them, and -[docs/DEPLOYMENT.md](DEPLOYMENT.md) covers running one. - -## What this is - -HotCell moves untrusted work out of a privileged application process into an unprivileged, -resource-capped, network-less sibling container. The application reaches in through a narrow interface. -The work never reaches out. - -A hot cell is a shielded chamber for handling highly radioactive material. The operator stays outside and -manipulates the contents remotely, and nothing leaves except by controlled transfer. The vocabulary -follows: **hot** is untrusted, **cold** is trusted, and material is **posted in** and **posted out**. -Everything else uses the ordinary word. - -## The problem - -Firejail and similar sandboxes do not work inside a container, because containers and sandboxing use the -same kernel features. An application deployed in a container therefore runs media libraries unconfined. A -vulnerability in libvips, `soffice`, `mutool`, `ffmpeg`, or ImageMagick, reached through an uploaded file, -executes with the application's secrets, its database credentials, its code, and its network. - -Two properties of `/proc` shape the response. Scrubbing a tool's environment does not help, because a -file-read primitive can retarget `/proc//environ` at a same-UID process and read the environment -that the scrubbed invocation was supposed to hide. And moving secrets out of the environment does not help -either, because the same primitive reads arbitrary files, including a credentials file such as -`config/master.key`. - -That leaves two remedies: a bubblewrap or nsjail wrapper inside the application image, or a separate -conversion service holding no secrets. HotCell is the second. The second also avoids an open question the -first carries, which is whether an in-image wrapper survives the container runtime it is deployed under. -That is the verification firejail does not pass. - -## Threat model - -Assume arbitrary code execution inside a cell, or merely an arbitrary file read, reached through a -malicious input file. The design must make that outcome cheap. - -In scope: reading application code or secrets, reaching the database, reaching the network, reading -another request's memory or environment, consuming the host's CPU or disk, escalating through a path the -application later opens, and reaching a different cell's toolchain. - -Out of scope: a kernel container escape, a malicious operation implementation, and denial of service -against a cell. - -Out of scope means the design does not promise to stop it, not that it is harmless. A malicious operation -— most plausibly a supply-chain compromise in a gem an operation depends on — is in fact contained by -everything here, because an operation runs in the same cell under the same limits as the library it -calls. It is excluded because it is not what this design is *for*, and because treating it as in scope -would drag operation review and dependency policy into a document about a transport. - -The minimum requirement is a PID and mount namespace exposing only the input and output, with no `/proc` -and no application filesystem. HotCell meets the namespace requirement by being a separate container, and -**exceeds the input/output part**: descriptors mean there is no path in the cell to expose or to traverse, -rather than a narrowed set of bind mounts. The `/proc` requirement is treated separately under "Worker -isolation", because it is the one part a container boundary does not give us for free. - -## The invariants - -These are the design properties the whole thing exists to hold. They are not a test plan. Most are -verifiable by reading the code, and a test that restates what an obvious ten-line method plainly does buys -nothing but maintenance. Test the ones where inspection is not enough — where the behaviour is the -kernel's rather than ours, where it is silent when broken, or where a plausible refactor would quietly -remove it. - -**These numbers are load-bearing.** Code and tests across three gems cite them by number. Do not renumber -them, and mark one withdrawn rather than removing it. - -1. ~~No application framework loads in a cell.~~ **Withdrawn.** Nothing enforces this and nothing should. - The whole design is that the container provides the guarantee structurally, so policing what code runs - inside it both duplicates a control that already holds and implies it does not. Under `network: none` a - loaded `ActiveRecord` cannot reach a database, and untouched pages are not copied, so the - copy-on-write argument for warning about it was not real either. Keeping a cell's gem graph small is - still worth doing — a smaller graph is a smaller thing to audit — but that is a budget, not a rule - about what an operation may require. -2. A cell holds no application credentials, in its environment **or on its filesystem**. -3. The supervisor never evaluates image data, and a worker forked from it produces correct output. -4. Descriptor access modes are one-way: an input cannot be written, an output cannot be read. -5. No filesystem path chosen by a hot side is ever opened by the cold side. -6. An operation cannot exceed its cell's limits, whatever it declares. -7. A cell cannot reach another cell's socket. -8. A worker cannot read another **request's** memory. Conditional on two things: - `kernel.yama.ptrace_scope >= 1`, a host setting no container flag can supply, and - `max_requests_per_worker: 1`. Not files — see "Worker isolation". -9. A tool subprocess sees only the environment its operation wrote for it. - -Invariant 2 is the awkward one. "A cell holds no application credentials" is a negative over a whole -filesystem and cannot be asserted directly. It stays on the list anyway, because it is the property the -whole design exists to hold. What the tests can reach is narrower: the built image contains no application -source tree, and `bin/conformance` boots the hardened container and checks its flags directly. - -A cell could go further and enforce it, by reading an allowlist of environment variable names and refusing -to boot on anything outside it. That is deferred, not rejected. It is a boot check that can be added at -any time without touching anything else here, and it addresses an operator mistake rather than the threat -this design is about. - -## Worker isolation - -Workers in a cell are siblings under one UID in one PID namespace, so another worker's `/proc` entries are -same-UID reads. Measured, not assumed — items 7 and 8 below: - -| Target | At `ptrace_scope = 1` | Why | -| --- | --- | --- | -| `/proc//mem` | `EACCES` | Needs `PTRACE_MODE_ATTACH`, which Yama restricts to descendants. | -| `/proc//environ` | **Readable** | Needs only `PTRACE_MODE_READ`, which Yama does not restrict. | -| `/proc//fd/N` | **Readable** | Same `PTRACE_MODE_READ`. So are the scratch files themselves, by directory listing. | - -So invariant 8 has three parts and they have different answers. Only the first is about `ptrace_scope`. - -**Request memory** is protected by `kernel.yama.ptrace_scope >= 1`. That is a host sysctl a container -cannot set, so it is a deployment precondition. At `ptrace_scope = 0` the guarantee is gone, and the cell -therefore **refuses to boot** rather than logging a warning and serving anyway. A host sysctl is invisible -to the image, it silently voids the guarantee, and a warning in a log is how a dead control stays dead. - -**Above `max_requests_per_worker: 1` this invariant is about workers, not requests.** A worker serving -several requests in turn holds each of them in the same address space, so an input that achieves code -execution can read and tamper with every later request that worker handles — no race to win, and covering -requests that were never concurrent with it. That is a deliberate setting with a measured payoff; see -[ADR 0001](../adr/0001-reuse-workers-across-requests.md). It is the only place in this design where the -isolation between two requests is a configuration value. - -**Nothing on disk carries from one request to the next.** A slot holds one directory per request, which is -that request's `$HOME` and also where its inputs and outputs are staged, and it is created when the request -starts and removed before the caller hears the answer. It used to survive across worker processes, to keep -an expensive per-user profile warm. That was a hole rather than a trade: what a tool reads from `$HOME` is -configuration, and for these toolchains configuration is executable — ImageMagick runs the command lines in -`delegates.xml` and applies the rights in `policy.xml`, both read from `$HOME/.config/ImageMagick`. So one -input that achieved code execution could reconfigure every later request on that slot, which is precisely -the bound `max_requests_per_worker: 1` exists to hold. [ADR 0003](../adr/0003-remove-the-persistent-slot-home.md) -records the reversal. - -The directory carries a fresh unpredictable name for every request, and that is what makes the removal a -guarantee rather than an intention. A tool that reaches code execution runs as the user that owns the tree, -so it can `chmod 0500` its own configuration directory and the slot directory around it, and both the -worker's delete and the supervisor's rename then fail. Under a stable name the next request was handed the -tree that had just refused to go. A name no earlier request has held is not a name an earlier request could -have prepared. A mode is also not a permission the process lost, so a cleanup that fails on one is retried -after putting the mode back, and what a cleanup that still fails costs is disk rather than isolation. - -**That bounds what a finished request left behind, and not what a live process is doing.** Every worker runs -as the same uid, and `0700` is that uid's own mode, so a concurrent sibling can write into a home as soon as -it exists — and so can a `setsid` descendant of a request that has already answered, which process groups do -not contain. The slot directory itself is a name a worker can rename aside and replace, and a pathname -`chmod` follows what it finds. Those are the residuals below, and a fresh name does not close them: it -closes the offline route, where nothing of the attacker's is still running. - -**Files are not isolated between concurrent workers, and cannot be.** Every worker runs as the same uid in -one mount namespace, so a worker that reads another worker's scratch directory — by listing it, or through -`/proc//fd/N` — gets that request's input and output bytes. Unlinking the scratch file does not -close it, because the descriptor is still reachable through the sibling's `/proc`. The two fixes that -would work need `CAP_SETUID` for a per-worker uid or `CAP_SYS_ADMIN` for a per-request mount namespace, -and `cap-drop ALL` removes both. This is the same shape as the environment: a real residual, stated rather -than papered over. - -What bounds it is the size of the window and the value of the contents. Only requests actually in flight -have bytes inside a cell, a cell holds no credentials, and a cell carries one toolchain. So the exposure -is "the other conversions happening right now in this cell", which is why per-toolchain cells and a sober -`concurrency` are containment decisions and not just scheduling ones. - -**A compromised worker can steal the cell's sockets.** It unlinks `work.sock` and binds its own, and every -later request arrives at its listener with the caller's descriptors already attached. It may then read -those inputs, write those outputs, and answer `ok`. This works because the socket directory has to be -writable by the user the supervisor runs as, and workers run as that user. - -The worker that did it exits as designed and its listener does not, because a child it forked calls -`setsid` and leaves the process group the reap sweep kills. So the reach is every request the cell -serves from then on, rather than only the ones in flight: the bound above covers one worker reading -another's files, and it does not cover this. - -**Nothing stops this today.** Every prevention available needs something the deployment does not have. A -tighter directory mode is undone by the owner, and the sticky bit grants the owner what it withholds from -others. The immutable flag needs `CAP_LINUX_IMMUTABLE` and a uid per worker needs `CAP_SETUID`, and -`cap-drop ALL` removes both. An abstract-namespace socket has no name to unlink and cannot be reached -across `network: none`. - -Detection was tried and withdrawn. The supervisor can record each socket's inode and re-check it, and an -attacker defeats that by hard-linking the original aside, serving from an impostor, and renaming the -original back before the next check: the inode then matches and the theft leaves no trace. Measured. The -check also cost more than it bought, because a supervisor that stops on a changed inode will unlink its -successor's sockets during an overlapping restart. - -[Landlock](https://github.com/basecamp/hotcell/issues/13) is the one prevention that fits an unprivileged -container: a worker gives up write access to the socket directory right after the fork, irreversibly and -with no capability. Until then this is a known gap, and the containment is the same as for a compromise -generally — a cell holds no credentials, carries one toolchain, and is replaced rather than repaired. - -This corrects an overclaim worth being explicit about, because it is easy to make: fork-per-request buys -**memory** isolation, not file isolation. The argument for descriptors below is about the boundary between -the application and the cell, and it does not extend to workers inside one cell. - -**Environment** is not protected by `ptrace_scope` at all, and cannot be fixed inside the worker. A forked -process's `/proc/self/environ` is the exec-time environment of the process it was forked from, so a worker -calling `ENV.delete` changes nothing about what a sibling reads. Two controls replace it. Invariant 2 -keeps anything worth stealing out of a cell's environment in the first place. And invariant 9 requires -tools to be spawned with `unsetenv_others: true` and an explicitly written environment, because a tool is -`exec`ed and therefore does get a fresh `/proc//environ` — the one thing in this picture that is -actually under our control. - -`hidepid=2` on `/proc` would remove the sysctl dependency by hiding sibling processes entirely, and it is -not available: Docker rejects Podman's `--security-opt proc-opts=`, and remounting `/proc` inside the -container needs `CAP_SYS_ADMIN`, which `cap-drop ALL` removes. Revisit only if the runtime changes. - -This is the third `/proc` surprise in this design, after `/proc/self/fd/N` laundering a read-only -descriptor and the environ retargeting that motivated the whole project. `/proc` is where these -assumptions go to die, and it deserves the attention. - -## Established by experiment - -Each of these was measured, not reasoned about. A specification cannot derive them, getting them wrong -produces failures that are hard to diagnose, and several constrain the architecture rather than the -implementation. - -1. **libvips cannot survive `fork` once it has evaluated an image.** After `require` and - `Vips.concurrency_set` the process has three threads and forks children that work; the first image - evaluation takes it to five, and from then on **every** forked child deadlocks in `futex_do_wait`, - permanently. Reproducible. This is the whole reason for the `before_fork` and `before_worker_boot` - split, and the reason `before_fork` may require and configure but must never evaluate. -2. **Two descriptors over `SCM_RIGHTS` work end to end**, with `Vips::Source.new_from_descriptor` and - `Vips::Target.new_to_descriptor`, and the kernel enforces the access modes: writing an input or reading - an output raises `Errno::EBADF`. -3. **Reopening `/proc/self/fd/N` defeats a read-only descriptor**, because it is a fresh `open` rechecked - against the inode and does not inherit the original flags. Never use it to turn a descriptor into a - filename. Copy instead. -4. **An empty Docker named volume takes its ownership from the image of whichever container mounts it**, - even if an earlier container already mounted it, provided it is still empty. So accessory and app boot - order does not matter. A bind mount instead takes the host directory's ownership, which is why local - development needs the directory created first. -5. **Kamal 2.11 hard-codes `--network kamal` for app roles.** Only accessories accept `network`, and only - as an accessory key. Under `options:` it is additive rather than overriding — Kamal emits its own - `--network` first — and Docker then refuses the container. Accessories can target `roles: [web, jobs]`, - and are not updated by a deploy. -6. **A worker killed by a resource limit produces a bare end of stream**, which is why the supervisor must - hold the connection and report `killed`. A `memory` breach does this too, roughly a third of the time: - libvips 8.18 dereferences null on its own out-of-memory path and takes `SIGSEGV` at - `vips_image_decode` *after* printing the correct diagnostic, and GLib's non-nullable `g_malloc` aborts. - So the reap-and-report path carries `memory` as well as `fsize`. -7. **A sibling process's `/proc//mem` is `EACCES` at `ptrace_scope = 1`, but its - `/proc//environ` is readable.** Verified with two same-UID siblings forked from one parent: the - environ read returned the victim's canary. Yama restricts `PTRACE_MODE_ATTACH`, which `mem` needs, and - does not restrict `PTRACE_MODE_READ`, which `environ` needs. -8. **A forked process cannot change what its own `/proc/self/environ` shows.** That view is the exec-time - environment, so `ENV.delete` in a worker is invisible to a reader. Only an `exec`ed child gets a fresh - one, which is why `unsetenv_others` on the tool spawn is the control. -9. **Docker cannot mount `/proc` with `hidepid`.** `--security-opt proc-opts=hidepid=2` is a Podman - feature; Docker rejects it outright, and remounting inside the container needs `CAP_SYS_ADMIN`. -10. **LibreOffice corrupts itself when two instances share a `$HOME` profile.** That is the origin of - slots. A hardened conversion measures at roughly 613ms, which sizes a soffice cell's deadline - concretely. -11. **`RLIMIT_AS` is unusable and `RLIMIT_DATA` is expensive.** For a real variant whose peak RSS is 45MB, - `RLIMIT_AS` must be at least 1536MB to succeed reliably and fails nondeterministically for a 400MB - band below that; `RLIMIT_DATA` works at 704MB. `RLIMIT_DATA` charges private writable anonymous - mappings and ignores `PROT_NONE` reservations, read-only private file mappings, and `MAP_SHARED` - entirely. It does charge thread stacks, so shrinking `RLIMIT_STACK` at container entry is worth real - headroom — and `RLIMIT_STACK` cannot be changed after `exec`, because glibc snapshots it at init. -12. **Ruby reserves about 450MB of `RLIMIT_DATA` at boot and never touches it.** A single ~404MB writable - anonymous region, introduced in 3.3 and unchanged in 3.4 and 4.0, unaffected by the GC and malloc - environment knobs. It is why the `memory` floor is what it is, and why `memory` cannot be read as how - much a bomb may consume. -13. **A cgroup memory kill is a prompt, silent `SIGKILL` with no diagnostic**, and the kernel chose the - allocating worker rather than the supervisor in every trial, because badness is RSS-proportional. The - argument for a per-worker limit is the diagnostic, not the choice of victim. -14. **Plain `require "vips"` leaves the libheif plugins un-`dlopen`ed.** They then load lazily inside the - worker, after its limits are on, where `dlopen` fails with only a warning and the process continues - with HEIC and AVIF missing — turning a limit breach into `unreadable` for a whole format family. - `require "image_processing/vips"` or `Vips.block_untrusted true` maps them in the supervisor instead. -15. **`fork` costs about 2.8ms**, measured as `fork` + `exit!` + `wait` from a 58MB three-thread parent - with libvips required and never evaluated. It is a small part of the cell's fixed overhead, not most - of it. -16. **The cell's fixed overhead is copy-on-write settling, and it is proportional to the supervisor's - resident heap.** A worker's first pipeline run takes about 7,900 minor faults against 1,564 for a warm - process. Forking the same work from a 265MB parent instead of a 58MB one took faults to about 25,900 - and added roughly 52ms per request. So preloading generously in the supervisor makes every request - slower for the life of the deployment. `RLIMIT_AS` and libvips thread-pool size were both tested and - neither moves it. -17. **macOS has no finite `RLIMIT_DATA`.** `Process.setrlimit` rejects one with `EINVAL`, and the - inherited hard limit is already infinity, so it is not a privilege problem. A cell there runs with its - memory clamp unenforced and warns once. Every other limit is strict on both platforms. -18. **Re-opening `/dev/fd/N` is checked against the opening process's credentials and the file's mode.** - Not against the caller's. So a cell handed a descriptor for a mode `0600` file the application owns - cannot open it by name at all, and every operation that gives a tool a filename dies as `EACCES`. - Operations that read the descriptor directly are unaffected, which is why the failure looks selective. - The mode is also what enforces invariant 4 once a tool holds a filename: at `0400` even the owner is - refused `O_RDWR`, and at `0200` even the owner is refused a read. That holds only while the cell does - not own the file. Changing a mode needs ownership, and `cap-drop ALL` leaves no capability that - overrides it — so a shared group enforces the invariant and a shared uid cannot. - -## Why descriptors rather than a shared volume - -The obvious alternative is a directory mounted into both containers: the app writes an input file and -names it, the cell writes an output file and names it. It is simpler, and it has one real advantage this -design gives up. Recorded here because the reasoning is not obvious and will otherwise be relitigated. - -**It removes a class of bug rather than a bug, and that is the deciding reason.** With a volume the app -must open a path the cell can write to. The cell creates that path as a symlink to -`/rails/config/master.key`, which resolves in the *app's* mount namespace; the app reads it and publishes -it as a public image variant. `O_NOFOLLOW` closes that, but only for the final component, so an -intermediate directory symlink still works, and closing that needs `openat2` with `RESOLVE_NO_SYMLINKS`, -which Ruby does not expose. Add a sticky exchange directory so the cell cannot unlink the app's files, -have the app pre-create both files, validate the descriptor with `fstat` for a regular file and an exact -size, and never list the directory. That is a checklist that must each be remembered forever, each item -failing silently. Descriptors have no such checklist, because there is no path. - -The checklist is not hypothetical. A volume-based service arrives at it item by item: a token validated -against `\A[a-f0-9]{32}\z`, a path built server-side from one component, -`O_RDONLY|O_NOFOLLOW|O_NONBLOCK`, an `fstat` on the descriptor for a regular file and an exact size under -a maximum, and then **the open descriptor rather than the path handed to the tool.** Which is to say it -arrives at descriptor passing anyway, having paid for the volume first. - -**Cross-request isolation is a smaller reason than it first appears.** Under a shared volume every worker -can read every request's bytes in the directory, including requests not currently running. Under -descriptors only in-flight requests have bytes inside the cell. That narrower window is the real gain — -and it is a window, not a wall. See "Worker isolation". - -**Nothing is left at rest.** A volume is persistent and outlives requests, so it holds user content -somewhere neither side owns, and a crashed worker leaves it there. The socket directory holds a socket and -no data. - -**What descriptors do not buy, so nobody oversells them.** The bytes are copied to the cell's tmpfs when -an operation asks for a path, so the cell has full read and write on its own copy. The output is a file -the app created and the cell writes arbitrary bytes into it either way, so a compromised cell can return a -malicious image and the app will publish it. Descriptors protect *naming*. They do not protect or validate -content. - -**The cost, accepted knowingly.** A bind mount crosses Docker Desktop's VM boundary and a descriptor does -not, so a volume-based design would let macOS developers run the real container with the real hardening. -Descriptors cost us that, which is why development runs a cell uncontainerized on every platform. Removing -a whole class of naming bug is worth more than development parity, because that class fails silently and -each mitigation for it has to be remembered forever. - -**Where a volume wins, and it is not a corner case.** An input larger than the cell's tmpfs cannot be -posted in at all. Video shows it plainly: a 3GB blob will not spool onto a 512MB tmpfs at any body limit, -and `ffprobe` reads only a container header, so store-and-forward costs roughly 300× I/O amplification on -its commonest call. Note carefully what this is *not* an argument against — descriptor passing has no -such problem, because the descriptor already refers to a file on the application's own filesystem. The amplification comes from the **copy** an input performs when asked for -its path, not from the transport. So the fix is per-operation and stays inside this design: an operation -that can consume a descriptor directly never asks, and never pays. - -## What the overhead measures at - -Read this section for its negative results rather than its numbers. The numbers came off a laptop from a -prototype; the things that were ruled out are properties of the design. For the reuse trade specifically, -[ADR 0001](../adr/0001-reuse-workers-across-requests.md) supersedes this with measurements taken in the -deployed artifact. - -Three arms, each producing a 256×256 PNG with `resize_to_fill`: in process, a native cell over a Unix -socket, and a containerized cell with the hardening flags on. The cell arms ran 2.3×, 1.9×, and 1.7× -slower as the source grew from a 12KB JPEG to a 328KB one. - -**The overhead is roughly fixed at 25 to 35 ms**, which is why the multiplier falls as the work grows. -Expect the ratio to look worst on the smallest thumbnails and to stop mattering on anything expensive. - -Four candidates were tested and eliminated, recorded so nobody re-runs them: - -| Candidate | Verdict | -| --- | --- | -| The container | Within noise of the native cell on two of three sources, 12ms on the largest. | -| The `fork` syscall | 2.8 ms of a 20–30 ms cost. | -| `RLIMIT_AS`, which the worker sets and the in-process arm does not | No effect. 8 GB, 2 GB, and unset are indistinguishable. | -| libvips thread pool startup per worker | Not it. The first-versus-second-call gap is *largest* at concurrency 1. | -| The libvips operation cache, as a confound favouring the in-process arm | Not it. `Vips.cache_set_max 0` moved that column by less than the error bars. | - -**The overhead is copy-on-write, measured rather than inferred.** The identical pipeline runs about 14 ms -slower in a freshly forked worker than in a long-lived process. Run it *twice* in the same child and the -second call matches a warm process, so it is one cost per process, not per call. Minor fault counts say -what it is: **7,920 faults on a child's first call against 1,564 warm**, about 25 MB of pages copied at -roughly 1.7 µs each. - -**So the cheapest optimisation is the counter-intuitive one: keep the supervisor small, and treat -`before_fork` as having a per-request price.** The instinct is to preload generously so workers start -fast. It is backwards here — every megabyte the supervisor holds is partly copied by every worker for the -rest of the deployment. `before_fork` must still `require`, because of the fork hazard in item 1 above, so -the resolution is to require only what that cell's own operations need. That is a third argument for one -cell per toolchain, and a measured reason for `hotcell-server` not to depend on `activesupport`. - -Do not spend effort making the container cheaper. There is nothing there to win. - -**Synthetic pre-warming is not a free alternative to worker reuse.** Tested, because it would have been -the ideal answer: running a synthetic 64×64 pipeline in the worker before the real transform moved the -real transform's faults from 7,885 only to 6,184, against a warm floor of about 3,300 — roughly a quarter -of the excess. Most of the cost is proportional to the real image's own work, not to shared code paths a -synthetic image would touch. - -Budget the fixed cost against inline processing. A variant generated during an upload request, as Active -Storage's `process: :immediately` does, adds this overhead to that request. Whether that is acceptable is -a product decision, and it is the main reason to measure it early on real hardware. diff --git a/docs/IMAGEMAGICK.md b/docs/IMAGEMAGICK.md deleted file mode 100644 index a4cff62..0000000 --- a/docs/IMAGEMAGICK.md +++ /dev/null @@ -1,248 +0,0 @@ -# ImageMagick's resource limits in a cell - -ImageMagick bounds itself through the `MAGICK_*_LIMIT` environment variables and a `policy.xml`. The -scaffold's `Dockerfile` installs no ImageMagick and sets none of them; an image that installs ImageMagick -must set them. Part one describes the variables and how they interact with a cell. Part two is how to -set them. - -[docs/DEPLOYMENT.md](DEPLOYMENT.md) covers the container flags and cell settings that part two starts -from: `memory`, `concurrency`, `file_size`, and "Where scratch lives". - -- [Part one: what the variables do](#part-one-what-the-variables-do) - * [The variables](#the-variables) - * [How they interact in a cell](#how-they-interact-in-a-cell) -- [Part two: how to set them](#part-two-how-to-set-them) - * [The formulas](#the-formulas) - * [A worked example](#a-worked-example) - * [Checking a built image](#checking-a-built-image) - * [When to recompute](#when-to-recompute) - -## Part one: what the variables do - -### The variables - -ImageMagick decodes every frame into a *pixel cache*, an uncompressed `width × height × bytes per pixel` -array, and the limits decide where each cache lives. ImageMagick's -[resources page](https://imagemagick.org/script/resources.php) is the reference; this table is the summary. - -| Variable | Unit | Bounds | -| --- | --- | --- | -| `MAGICK_MEMORY_LIMIT` | bytes | Caches on the heap, all frames together. | -| `MAGICK_MAP_LIMIT` | bytes | Caches that missed the heap and are memory-mapped files. | -| `MAGICK_DISK_LIMIT` | bytes | Cache files on disk, mapped or not, all frames together. Past it the operation fails with `cache resources exhausted`. | -| `MAGICK_AREA_LIMIT` | pixels | The largest single frame allowed on the heap. Over it, the frame goes to a file even when the heap would hold it. It never refuses a frame. | -| `MAGICK_THREAD_LIMIT` | threads | The OpenMP team ImageMagick asks for. | -| `MAGICK_TMPDIR` | path | Where the cache files go. Read before `TMPDIR`. | - -Bytes per pixel come from the build: 8 on ImageMagick 6 Q16 without HDRI, which is Ubuntu's -`imagemagick-6.q16` (four 16-bit channels), 10 with an index or black channel; an HDRI build stores -floats and doubles both. A 4096 × 4096 image is 128MiB before anything is done to it, a layered PSD holds one cache -per layer at once, and a transform holds source and result at once. - -The three byte limits are a chain: heap, then mapped file, then plain file, and a cache takes one tier -whole. Both file tiers count against `disk`. So a single frame is readable only if it fits `memory` or -`disk` on its own. A multi-frame file needs every frame placed in turn, each in the first tier with room -left, so its frames can total at most `memory` plus `disk`, and less when they pack badly: a frame that -misses the heap's remainder must fit the disk's remainder on its own. - -Each limit has three sources. The build's default (most of the host's RAM, unbounded disk) is replaced -by the environment, upward or downward; `policy.xml` is a ceiling on both, which the environment can -lower and never raise. They are read once: when the `magick` process starts, or at ImageMagick's first -use inside a process that loaded the library. Limits are per process. `identify -list resource` prints -the result. - -### How they interact in a cell - -**The environment reaches both ImageMagicks.** ImageMagick runs in a cell two ways. In-process, when -libvips delegates PSD, BMP and ICO to `magickload`; the library reads the worker's environment. As a -child, when mini_magick spawns `identify` or `magick` for `analyzers.image.magick` and -`transformers.image.magick`. mini_magick runs with `restricted_env`, which passes the child only `HOME`, -`PATH`, `LANG` and `MiniMagick.cli_env`, so `MagickOperation` rebuilds `cli_env` for every request from -the worker's `MAGICK_*_LIMIT`, `TMPDIR` and `MAGICK_TMPDIR`. - -**`MAGICK_TMPDIR` is the gem's.** `Operation#initialize` maps it from the request's `TMPDIR`, so in-process -cache files land in the request's directory and are removed with it. Do not set it in the image, and do -not set a `temporary-path` policy, which overrides it. - -**Every limit divides by `concurrency`.** The limits are per process, and the cell runs `concurrency` -processes against one `/tmp` and one cgroup. - -**`file_size` cannot do the disk limit's job.** `file_size` is `RLIMIT_FSIZE`, a limit on each file. A -layered PSD writes one cache file per layer, and twenty 40MiB caches total 800MiB with no file near a -48MiB `file_size`. Only `MAGICK_DISK_LIMIT` bounds the sum. Where the two meet, the smaller fires -first, as an `fsize` kill or as `cache resources exhausted`; both are permanent verdicts on the file. - -**A disk limit larger than the scratch is not a limit.** The scratch fills first. A full scratch fails -every request on the cell that needs scratch, and a write that fails inside libvips is a `Vips::Error`, -which the gem declares `unreadable`: permanent, and recorded against a customer's file. On 2026-09-01 one -PSD under a `10GiB` disk limit filled a 4G scratch on six hosts and convicted 3,393 blobs. Under a -disk limit sized to a worker's share, the same file is `unreadable` on its own first request, in -milliseconds, with the scratch empty afterwards. - -**For `magickload`, the cache files are the spill.** libvips writes a decode above `VIPS_DISC_THRESHOLD` -to `TMPDIR`; when the decode is ImageMagick's, its cache files stand in for that temporary rather than -adding to it. So the disk limit and the transformer's `file_size` describe the same bytes. - -**A heap cache is charged to `RLIMIT_DATA` and to the cgroup.** It is private anonymous memory, so the -operation's `memory` counts it and so does the container's. `MAGICK_MEMORY_LIMIT` plus the worker's own -footprint must sit under a worker's share of the cgroup and under the `memory` of every operation that -can reach ImageMagick. Above either, the process dies (`killed`, or libgomp's `exit(1)` on a thread it -cannot create) instead of refusing the frame, and a death is transient: the request is retried against -a limit that fails it again. - -**`MAGICK_AREA_LIMIT` adds nothing.** It moves the heap-or-file line in pixels; `memory` already draws it -in bytes. Set below `memory`, it sends frames to scratch the heap could have held. - -**`MAGICK_THREAD_LIMIT` raises the OpenMP count itself**, and `OMP_THREAD_LIMIT` caps it. See -[Bound the OpenMP thread pools](DEPLOYMENT.md#bound-the-openmp-thread-pools). - -## Part two: how to set them - -### The formulas - -Every "MiB" is `1024²` bytes, as `config.rb` writes it. Work each operation that can reach ImageMagick -separately and set the image to the smallest result. - -**`MAGICK_DISK_LIMIT`** - - scratch ÷ concurrency − what else the operation writes on scratch - -Scratch is the tmpfs `size=` or the host filesystem's usable size as `df` reports it, which on an ext4 -made with default options is 5% under the nominal size; a named volume has no size to divide. The -subtraction is the operation's own files: its output, and its input if it stages one. The shipped image -operations read their input through the descriptor and stage nothing; a transformer writes its output on -scratch before copying it out. The result must be zero or more. Below zero, enlarge the scratch or lower -`file_size` or `concurrency`; ImageMagick reads a negative value as a huge one. - -**`MAGICK_MAP_LIMIT`** = `MAGICK_DISK_LIMIT`. The mapped caches are the same files on the same scratch. - -**`MAGICK_MEMORY_LIMIT`** - - (memory − tmpfs) ÷ concurrency − a worker's own footprint, rounded down - -On a disk-backed scratch there is no tmpfs term. The footprint is the worker's resident size with the -gems loaded plus the child it may spawn; measure it as `VmRSS` in `/proc//status` after a request. -Round down to leave the worker something for what the limit does not count. Then check the result plus -the footprint against each reaching operation's `memory`. - -**`MAGICK_AREA_LIMIT`**: unset. **`MAGICK_THREAD_LIMIT`**: `OMP_NUM_THREADS`, or unset. -**`MAGICK_TMPDIR`**: unset. - -**`policy.xml`**: the same `disk` value. - -### A worked example - -A production cell serving image transforms and analysis for a Rails application. - -**The volume mount.** `/tmp` is a bind mount of a loopback ext4 image on the host, sized so that `df` -reports 4096MiB available when the mount is empty, mounted -`nosuid,nodev,noexec` and chowned to the cell's uid by the host's configuration management: - -```yaml -accessories: - active_storage: - volumes: - - hotcell-sockets:/run/hotcell/cell - - /var/lib/hotcell-scratch:/tmp - options: - cpus: 2 - memory: 2g - memory-swap: 2g -``` - -It is disk rather than the scaffold's tmpfs because a tmpfs is RAM charged to the cgroup: every byte of -scratch is a byte the workers cannot have. It is a dedicated filesystem rather than a named volume or a -plain directory because the filesystem's size is the cap and its mount flags reach the container; Docker -can set neither on a bind mount. [docs/DEPLOYMENT.md](DEPLOYMENT.md#a-host-mounted-filesystem) has the recipe. - -**The constraints.** - -| Input | Value | From | -| --- | --- | --- | -| scratch | 4096MiB | `df --output=avail` on the empty loopback filesystem | -| container `memory` | 2048MiB | `memory: 2g`, no tmpfs term | -| `concurrency` | 4 | `config.rb`, twice `cpus` | -| cell ceiling | `memory: 1536MB`, `file_size: 768MB` | `config.rb`; an operation's own limits are clamped to these | -| `transformers.image.vips` | `memory: 1280MB`, `file_size: 768MB` | `file_size` raised in the application's operations file | -| `analyzers.image.vips` | `memory: 1024MB`, `file_size: 48MB` | gem defaults | -| `analyzers.image.magick` | `memory: 1024MB`, `file_size: 48MB` | gem defaults | -| `OMP_NUM_THREADS` | 2 | the image, matching `cpus` | -| ImageMagick | 6 Q16 | 8 bytes per pixel | - -ImageMagick is reached three ways: the vips transformer and analyzer through `magickload`, and the -magick analyzer through an `identify` child. All three read their input through the descriptor. The -transformer writes its output on scratch; the analyzers write nothing. - -**Disk.** Nothing is staged, so the only thing to hold back is the transformer's output, measured at a -peak of 256MiB per worker: - - MAGICK_DISK_LIMIT = 4096 ÷ 4 − 256 = 768MiB - MAGICK_MAP_LIMIT = 768MiB - -It equals the transformer's `file_size` because that was sized by the same arithmetic, the -application's test holds it, and the cell ceiling allows it. Four workers spilling 768MiB and writing -256MiB beside it fill the 4096MiB exactly. That is why the scratch input is what `df` reports available -and not the image's size: ext4's metadata and its default 5% root reserve take part of the image, so a -4G image reports under 3891MiB, for which the same arithmetic gives 716MiB. Size the image over what the -arithmetic needs, and take the number from `df` on a cell host. On the analyzer, whose `file_size` is 48MB, a cache file over that takes the `fsize` kill -first. - -**Memory.** - - per worker = 2048 ÷ 4 = 512MiB - footprint = 70MiB resident + 12MiB for an `identify` child = 82MiB - ceiling = 512 − 82 = 430MiB - MAGICK_MEMORY_LIMIT = 384MiB, the round number under it - -Four workers at the limit are `4 × (384 + 82) = 1864MiB` under the 2048MiB cgroup. `384 + 82 = 466MiB` -sits under the analyzers' 1024MB and the transformer's 1280MB. - -**Together.** - - MAGICK_MEMORY_LIMIT = 384MiB - MAGICK_MAP_LIMIT = 768MiB - MAGICK_DISK_LIMIT = 768MiB - MAGICK_THREAD_LIMIT = 2 - MAGICK_AREA_LIMIT = unset - -and `disk` at 768MiB in `policy.xml`. - -**What that buys**, at 8 bytes a pixel and summed over every cache open at once: - -| Limit | Pixels | For instance | -| --- | --- | --- | -| 384MiB heap | 50.3 million | 8192 × 6144 | -| 768MiB disk | 100.7 million | 16384 × 6144 | -| both, multi-frame | up to 151 million | a 19-layer PSD at 2740 × 2740: 6 layers on the heap, 13 on disk | - -A same-size transform holds source and result, so it decodes half a tier. A larger file is `unreadable` -on its first request, and the scratch is empty afterwards. The limits the image had inherited from the -application were `1GiB`, `5GiB` and `10GiB`: memory twice a worker's share of the cgroup, disk two and a -half times the scratch. - -### Checking a built image - -Inside the image, as the cell's user, `identify -list resource` prints the effective limits: `Disk` -should be the number computed, `Area` unbounded. `identify -list policy` shows what `policy.xml` set, -and that `temporary-path` is absent. - -To watch the tiers: - -``` -MAGICK_MEMORY_LIMIT=64MiB magick -size 4000x4000 xc:red -debug cache info: -``` - -On ImageMagick 6 the command is `convert`. - -The cache opens as a file under `MAGICK_TMPDIR`. Raise the size until it passes the disk limit too and -the command fails with `cache resources exhausted`. - -End to end, send a layered file larger than the disk limit through the cell: the answer is -`unreadable`, and the scratch root is empty afterwards. A cache file left behind means the spill went -to `/tmp` rather than the request's directory. - -### When to recompute - -When any input moves: the scratch's size, the container's `memory`, `concurrency`, an operation's -`file_size` or `memory`, the worker footprint, which operations reach ImageMagick, or the ImageMagick -build's bytes per pixel. Keep the formulas next to the values in the `Dockerfile`. diff --git a/docs/LOGS.md b/docs/LOGS.md deleted file mode 100644 index fe1f2bd..0000000 --- a/docs/LOGS.md +++ /dev/null @@ -1,171 +0,0 @@ -# Log lines - -The cell writes one JSON object per line to stdout. Field names follow -[ECS](https://www.elastic.co/guide/en/ecs/current/index.html), the schema the rest of the fleet's -structured logs already speak. Anything ECS has no name for lives under the `hotcell` namespace, so -no future ECS field can collide with ours. - -## The envelope - -Every line carries these five fields: - -| Field | Type | Meaning | -| --- | --- | --- | -| `@timestamp` | string | When it happened. UTC, ISO 8601, millisecond precision. | -| `service.name` | string | Always `"hotcell"`. This is the routing key: the log collector selects our lines by it. | -| `event.action` | string | Which event this is. The full list is below. | -| `log.level` | string | `"INFO"`, `"WARN"` or `"ERROR"`. The emitter decides severity, not the collector, so adding an event never requires a collector change. | -| `process.pid` | integer | The process the event is about: the worker's pid for `worker.*` events, the supervisor's own for `cell.*`. Absent where no process is the subject. | - -## Shared fields - -Where ECS has a name, we use it: - -| Field | Type | Used by | -| --- | --- | --- | -| `error.type` | string | Exception class name, wherever an exception is reported. | -| `error.message` | string | The sanitized exception message beside `error.type`. | -| `message` | string | Human-readable detail on events whose meaning needs prose (`cell.ptrace_scope_unknown`, `slot.uncleaned`, `worker.unreadable_report`). | -| `event.outcome` | string | `"success"` or `"failure"`, on `request` only. | -| `event.duration.ms` | number | Wall time of the thing that ended. This is the fleet's dialect (rails logs use `event.duration.ms`), not stock ECS (`event.duration` in nanoseconds). | -| `process.exit_code` | integer | The worker's exit status, on `worker.reaped`. | - -Everything else is ours and sits under `hotcell.*`: - -| Field | Type | Meaning | -| --- | --- | --- | -| `hotcell.slot` | integer | The slot number. On nearly every event. | -| `hotcell.op` | string | The operation the line is about, on `request`, `request.abandoned`, `worker.crashed`, `worker.killed` and `worker.undispatchable`. `null` where the name was not known: a request that never parsed, a crash between requests, a worker that died before it reported. Never the name of an earlier request. | -| `hotcell.code` | string | The response code of a `request` (`"ok"`, `"failed"`, `"killed"`, ...). | -| `hotcell.permanent` | boolean | Whether a `request` failure is permanent. | -| `hotcell.cause` | string | Why a worker was killed (`"deadline"`, `"memory"`, `"fsize"`, ...). | -| `hotcell.signal` | string | Signal name (`"SIGKILL"`, `"SIGSEGV"`, ...). ECS has no field for signals. | -| `hotcell.served` | integer | Requests a worker served before it was reaped. | -| `hotcell.swept` | integer | Discarded trees a sweeper found cleared, on `scratch.swept`: unlinked by it, or already gone when it reached them. | -| `hotcell.home` | string | The scratch directory a cleanup could not clear: a request's `$HOME` from a worker, the slot directory from the supervisor. | -| `hotcell.directory` | string | The cell's working directory, on `cell.boot`. | -| `hotcell.operations` | array | Registered operation names, on `cell.boot`. | -| `hotcell.configuration` | object | The full configuration inventory (same shape as `hotcell.describe`), on `cell.boot`. | -| `hotcell.running` / `hotcell.queued` | integer | In-flight and queued requests, on `cell.stopping`. | -| `hotcell.timing` | object | A request's phase timings: `queued_ms`, `perform_ms`, and any measured phases. | -| `hotcell.deadline_s` / `hotcell.grace_s` / `hotcell.waited_s` | number | The limit that was hit, on the event that reports hitting it. | -| `hotcell.path` | string | The file the cell could not verify, on `cell.ptrace_scope_unknown`; the scratch entry, or the scratch itself, a boot could not remove, on `scratch.unswept`. | -| `hotcell.stderr` | string | The tail of what a dying worker wrote to file descriptor 2, at most 512 bytes. On `worker.killed` only, and absent when it wrote nothing. See below. | - -## Events - -| `event.action` | `log.level` | Fields beyond the envelope | -| --- | --- | --- | -| `cell.boot` | INFO | `hotcell.directory`, `hotcell.operations`, `hotcell.configuration` | -| `cell.stopping` | INFO | `hotcell.running`, `hotcell.queued` | -| `cell.stopped` | INFO | — | -| `cell.ptrace_scope_unknown` | ERROR | `hotcell.path`, `message` | -| `request` | INFO | `hotcell.slot`, `hotcell.op`, `hotcell.code`, `hotcell.permanent`, `event.outcome`, `event.duration.ms`, `hotcell.timing` | -| `request.abandoned` | WARN | `hotcell.slot`, `hotcell.op` | -| `worker.forked` | INFO | `hotcell.slot` | -| `worker.reaped` | INFO | `hotcell.slot`, `hotcell.served`, `hotcell.signal`, `process.exit_code` | -| `worker.crashed` | ERROR | `hotcell.slot`, `hotcell.op`, `error.type`, `error.message` | -| `worker.killed` | WARN | `hotcell.slot`, `hotcell.op`, `hotcell.cause`, `hotcell.signal`, `event.duration.ms`, `hotcell.stderr` | -| `worker.deadline` | WARN | `hotcell.slot`, `hotcell.deadline_s` | -| `worker.lingered` | WARN | `hotcell.slot`, `hotcell.grace_s` | -| `worker.unforkable` | ERROR | `hotcell.slot`, `error.type`, `error.message` | -| `worker.undispatchable` | ERROR | `hotcell.slot`, `hotcell.op`, `error.type` | -| `worker.unreadable_report` | ERROR | `message` | -| `sweeper.forked` | INFO | — | -| `sweeper.deadline` | WARN | `hotcell.deadline_s` | -| `sweeper.died` | WARN | `hotcell.signal`, `process.exit_code`; a sweeper that ended abnormally by anything but the deadline kill | -| `sweeper.unforkable` | ERROR | `error.type`, `error.message` | -| `sweeper.crashed` | ERROR | `error.type`, `error.message` | -| `scratch.swept` | INFO | `hotcell.swept`, `event.duration.ms` | -| `control.abandoned` | WARN | `hotcell.waited_s` | -| `control.unanswerable` | WARN | `error.type`, `error.message` | -| `slot.uncleaned` | WARN | `hotcell.slot`, `hotcell.home`, `message` (boot sweep only) | -| `slot.undiscarded` | WARN | `hotcell.slot`, `hotcell.home` | -| `slot.unswept` | WARN | `hotcell.slot`, `hotcell.home`; from the worker that answered on the slot or from the sweeper | -| `scratch.unswept` | WARN | `hotcell.path`; `error.type` and `error.message` when the scratch itself could not be listed | - -## What a worker wrote to fd 2 - -A worker's fd 2 is a pipe to the supervisor, which drains it as the worker runs and attaches the tail to -the `worker.killed` reporting its death. The same text rides the failure the caller receives, so an -application logs `killed: crashed (libgomp: ...)` rather than a bare `crashed`. - -It exists for the death nothing else in a cell can describe. `HotCell::Worker#run` rescues `Exception`, so -a worker that died with no `worker.crashed` line probably died without Ruby raising at all: a C library -called `exit()`, and said why on fd 2. - -``` -libgomp: Thread creation failed: Resource temporarily unavailable -``` - -**The text is not evidence — it establishes neither who wrote it nor which request it belongs to**, the -caveat `hotcell.signal` and `hotcell.cause` already carry. It comes from the one process in a cell that -runs untrusted code, over an unauthenticated channel: everything a worker spawned inherits fd 2, so a tool -can write long after the request it belongs to finished, and a sibling worker can open `/proc//fd/2` -and write whatever it likes, since workers share a uid and `kernel.yama.ptrace_scope` protects memory -rather than descriptors. The supervisor clears the buffer at each dispatch, which keeps an old warning off -an unrelated death in the ordinary case. It is not a boundary. - -**The capture is best effort, because fd 2 is non-blocking.** A C library writing to a full pipe gets `EAGAIN` -and loses the line, and a fatal handler cannot retry — it writes once and calls `exit()`. That costs nothing -in the case this exists for, where libgomp's one short line meets an empty pipe. What it loses is the fatal -from a decoder that had already filled the pipe with warnings, where the field reports the tail of those -warnings instead. The alternative was refused: a blocking fd 2 would make a warning written from inside -libvips wait on the supervisor's scheduling, in a C call Ruby cannot interrupt, and that wait is nearest -exactly when the host is under pressure. - -**Only a death is reported.** A worker that warns and then answers normally leaves no field behind, on any -event. So a tool that dies while its worker survives is not described here, and a cell's stderr no longer -reaches the container's log driver at all — which loses nothing, because the fleet's OTel collector drops -complete non-JSON lines at ingest. - -## Examples - -A request: - -```json -{"@timestamp":"2026-08-14T21:05:28.252Z","service":{"name":"hotcell"},"event":{"action":"request","outcome":"success","duration":{"ms":9.6}},"log":{"level":"INFO"},"process":{"pid":83},"hotcell":{"slot":0,"op":"active_storage.transform_image","code":"ok","permanent":null,"timing":{"queued_ms":0.4,"perform_ms":0.52}}} -``` - -A crash: - -```json -{"@timestamp":"2026-08-14T21:05:29.107Z","service":{"name":"hotcell"},"event":{"action":"worker.crashed"},"log":{"level":"ERROR"},"process":{"pid":83},"error":{"type":"NoMethodError","message":"undefined method 'blur' for nil"},"hotcell":{"slot":0,"op":"active_storage.transform_image"}} -``` - -## Which operation a line is about - -`hotcell.op` is what makes a cell's own logs answer "which operation did this?". A cell runs several -operations at once and they do not share limits, so an unattributed `worker.killed` cannot be acted on, and -nothing else supplies the name: the response carries no operation, and `hotcell_killed` is tagged `cell` -and `cause` only. - -The two sides learn it differently, which is why it can be absent. - -A **worker** parses it out of the request it is serving, so `request`, `request.abandoned` and -`worker.crashed` carry it from the moment the request parses until the worker goes back to waiting. A -request that never parsed has no name, and neither does a crash between requests. - -The **supervisor** never reads a request a worker is going to serve, since staying out of it is what lets -it dispatch a connection whose descriptors are still queued on it. It learns the name from the worker's -report, sent once the request has parsed and before the worker touches an untrusted byte. That is what lets -`worker.killed` name an operation the dead worker cannot report, and a worker that died first leaves the -field null rather than borrowing the last request's name. - -That report comes from the one process here that runs untrusted code, so the name is bounded to an -operation this cell registered. The bound is on the report and nowhere else: a `request` line names -whatever the caller asked for, including a name no operation answers to, which is what `unsupported` is -about and worth seeing. A name whose report would not fit one control line is not sent at all, so -`worker.killed` goes unattributed rather than losing the narrowed deadline that shares the line. - -`worker.undispatchable` is the exception, and the one line where the supervisor reads a request: the worker -died between the fork and the dispatch write, so nothing else has read it. It peeks rather than reads, so -neither the bytes nor the caller's descriptors leave the connection, and it never waits, so a request that -has not arrived leaves the field null. - -## The contract with the collector - -The fleet's log collector routes on `service.name == "hotcell"`, sets the record timestamp from -`@timestamp`, and takes severity from `log.level`. Those three fields are therefore load-bearing: -renaming any of them silently drops or mislabels every cell log line in production. The rest of the -schema can evolve freely. diff --git a/docs/TUNING.md b/docs/TUNING.md deleted file mode 100644 index c02154a..0000000 --- a/docs/TUNING.md +++ /dev/null @@ -1,189 +0,0 @@ -# Tuning a cell - -[docs/DEPLOYMENT.md](DEPLOYMENT.md) lists every setting and what it does. This is how to arrive at the -numbers for your own workload. - -The short version: ship the defaults, instrument first, and tighten in one direction only. - -## Instrument before you tune - -You cannot tune what you cannot see, and two signals are independent of each other. Get both in before you -change any number. - -**The `perform.hot_cell` notification.** It fires on every call, success or failure, and carries the -operation, the cell, the failure's `code`, `cause`, `signal` and `permanent`, `bytes_in`, `bytes_out`, -`perform_ms` and the full timing. It is the only signal that survives a dead cell — an unreachable socket -arrives here as `unavailable`, so the primary alarm belongs on this and not on the cell's own metrics. - -Classify by `permanent`, not by `code`. A `killed` is permanent for `fsize` and `memory` and transient -for `deadline` and `crashed`, so the code alone cannot say which side of the split a kill is on; -`permanent` is the cell's own answer, and `cause` is why. - -**The cell's `metrics`, on its control socket.** Poll it on a schedule. It answers while the work socket is -saturated, and it is host-local, so the poller has to be a process on the cell's own host. It reports -`running`, `queued`, `queue_high_water`, `cancelled`, the request counts by code, and `killed_by` broken -down by cause. - -`killed_by` is what workers reported, not what the supervisor observed. A worker decides its own `memory` -and `fsize` — the supervisor cannot tell either from a wait status without believing a signal a sibling -could have sent — and reports the cause when it reports itself idle. So the count arrives just after the -caller has its answer rather than before, it is lost if the worker dies in the window between the two, and -a compromised worker can report a cause its request never had. Size limits by it; do not read it as -evidence about any particular document. - -## Which number comes from which measurement - -| Setting | Where the number comes from | -| --- | --- | -| `memory` | one worker's peak on your largest real inputs | -| `file_size` | the largest thing one operation writes, plus any input it stages | -| `deadline` | the tail of `perform_ms` on real traffic | -| `concurrency` | a load test, against `cpus` | -| `queue_size`, `queue_wait` | a load test, against what the caller can wait for | -| container `memory`, `cpus`, tmpfs size | `concurrency` times the above | - -Only the scheduling numbers need contention to measure. The rest come from single requests, and a load -test tells you nothing useful about them. - -## Start generous on two of them - -`memory` and `file_size` are the two where a wrong value is expensive to undo. - -**A limit that is too high costs headroom.** The container's own memory limit still bounds the cell. - -**A limit that is too low kills a legitimate file.** The cell answers `killed: memory` or `killed: fsize`. -Both verdicts are permanent, which means the caller is entitled to write them down — and one shipped path -does. See below. - -The two mistakes are not equal, so use this order: - -1. Set `memory` and `file_size` to the defaults. -2. Run real traffic for at least a week. -3. Read the peak your operations reach. -4. Lower each limit to a value above that peak. - -Do not use the opposite order. Do not set a low limit and raise it when files fail. Every failure in that -period is a durable record against a customer's file. - -One gap in step 3: the cell's metrics report `killed_by`, which counts failures. A count of zero tells you -the limit is high enough. It does not tell you how much room is left. To find the room, measure the -operation outside the cell — run it on your largest inputs and read its peak. - -`deadline` needs none of this care. It is transient, so a value that is too low costs a retry. - -### Why those two are different - -A permanent verdict is only irreversible if the application writes it down. In the shipped Active Storage -integration, analysis does and nothing else does. - -Rails persists a blob's analysis like this: - -```ruby -# ActiveStorage::Blob::Analyzable -def analyze - update! metadata: metadata.merge(extract_metadata_via_analyzer) -end - -def extract_metadata_via_analyzer - analyzer.metadata.merge(analyzed: true) -end -``` - -`analyzed: true` is merged whatever the analyzer returned, including an empty hash. Rails never asks -whether the analysis worked. So the chain is: - -1. The blob is attached. Rails enqueues `ActiveStorage::AnalyzeJob`, once. -2. The analyzer calls the cell and gets `killed: memory`, which the client raises as the application's - permanent class. -3. `Analyzers::Analyzing#metadata` rescues that class, logs it, and returns `{}`. -4. Rails merges `analyzed: true` and writes the row. - -The blob's `metadata` is now `{"identified"=>true, "analyzed"=>true}` — analyzed, with no dimensions. -Nothing re-enqueues the job, because `analyze_later` runs once at first attachment. - -A transient failure is not rescued at step 3. It escapes into the job, which retries it, and `analyzed` -stays false. - -Undoing a permanent one is a backfill: - -```ruby -blob.update!(metadata: blob.metadata.except("analyzed")) -blob.analyze_later -``` - -**Previews and variants write no durable failure record.** `Preview#processed?` is `image.attached?`, and a -variant is recorded by its `active_storage_variant_records` row. A failure attaches nothing and creates -nothing, so the job simply retries. Only analysis needs the generous-first treatment. - -## Make the timeouts agree - -The defaults do not agree with each other, and this is the first thing you will hit. - -A cell reports `answer_within`, which is `queue_wait + deadline + 1`. On the defaults that is **71 -seconds**. The client's default `timeout` is **30 seconds**. - -Under that, a saturated cell reaches the caller as a transport failure instead of as `capacity` or -`killed` — losing exactly the signal you need to size anything. The client warns at boot when its timeout -does not clear the cell's number. - -Which way to fix it depends on the caller: - -- **A background job** wants a loose timeout, above `answer_within`, so it receives the cell's verdict and - can act on it. Active Storage's analysis, preview and variant work is all jobs. -- **A synchronous request** wants the opposite: a short `deadline` on the cell and a tight timeout on the - client, because a thread held for a minute is a thread not serving traffic. - -Both outcomes are transient, so neither choice misclassifies anything. That is the only reason this is -safe to decide per caller. - -## What to watch - -| Signal | What it means | -| --- | --- | -| `killed_by` by cause | the only legitimate reason to tighten a limit | -| `queued_ms` p95 rising, `perform_ms` p95 flat | the cell needs more workers, not faster ones | -| `perform_ms` p95 rising | the work got more expensive; check for a library upgrade | -| `queued` near `queue_size`, or `queue_high_water` rising toward it | no headroom left; `queue_high_water` resets only at boot | -| `capacity` above zero in steady state | under-provisioned | -| `unavailable` | the cell is down, restarting, or unreachable | -| `unreadable` rate | worth watching after a toolchain upgrade | -| `worker.crashed` in the log | should be zero; anything else is a bug worth reporting | - -## Where `bin/load` fits - -`bin/load` answers one question: does the queue behave the way you configured it? It shows that -`queue_size` and `queue_wait` produce `capacity` where you expect, and it finds the knee for a known -service time. - -``` -bin/load IMAGE [SCENARIO] [SECONDS] [THREADS] -``` - -It runs a cell in a container and drives it from a second one. It reports throughput, the verdict -breakdown, and latency split into time queued against time performing. That split is what separates -saturation from slowness: queued time that grows while perform time stays flat means the cell needs more -workers. - -It cannot give you your own numbers. It drives the example operations rather than yours, and it fixes -`cpus`, `memory`, the tmpfs size and `pids-limit` — only the cell settings vary, through -`EXAMPLE_CONCURRENCY`, `EXAMPLE_QUEUE_SIZE`, `EXAMPLE_QUEUE_WAIT`, `EXAMPLE_DEADLINE`, `EXAMPLE_MEMORY_MB` -and `EXAMPLE_FILE_SIZE_MB`. Treat it as a check on the scheduling, not as a capacity plan. - -## A starting point - -Deploy the documented defaults unchanged, with `max_requests_per_worker: 1`. Raise the client's timeout -above `answer_within` for any job path. Then let it run for a week and read the numbers above. - -`max_requests_per_worker: 1` is the right default to start from because it is the isolating one. Raise it -only with a measurement that says the copy-on-write cost matters for your workload, and read -[ADR 0001](../adr/0001-reuse-workers-across-requests.md) first — it costs an isolation property, not just -memory. - -## A caution about staging - -A cell's behaviour is a function of the files it is given. A staging or beta destination with synthetic -uploads will not predict production, and the measurements that matter here were all taken on the deployed -image for that reason — see [ADR 0001](../adr/0001-reuse-workers-across-requests.md). - -Tune against production traffic with generous limits. That direction fails slow rather than failing -closed. diff --git a/docs/active-storage.md b/docs/active-storage.md new file mode 100644 index 0000000..0934313 --- /dev/null +++ b/docs/active-storage.md @@ -0,0 +1,113 @@ +--- +type: Reference +title: "Active Storage operations" +order: 12 +description: "The Active Storage client classes, the cell operations that serve them, how their failures are retried, and the limits each declares." +sources: + - activestorage-hotcell-client/lib + - activestorage-hotcell-server/lib +--- + +# Active Storage operations + +This page describes the `activestorage-hotcell-client` and `activestorage-hotcell-server` gems, which run +Active Storage's variants, analysis, and previews in a cell. For a walkthrough, see +[Using the Active Storage operations](../README.md#using-the-active-storage-operations) in the README. + +The Active Storage gems need Rails 8.2, which is unreleased. Rails can swap out variant processing only +from [rails/rails#58384](https://github.com/rails/rails/pull/58384) +([`5ea765e5`](https://github.com/rails/rails/commit/5ea765e5b00085a22f5cbe863c0d2ac765428242)) onward. The +`hotcell-*` gems themselves don't require Rails. + +## Classes + +For every class that you name in `config.active_storage`, the cell must load the matching operation, and +the cell's image must install the underlying library or tool. + +| Application class, in `ActiveStorage::HotCell::Client` | Cell operation file, in `active_storage/hot_cell/server` | Routing name | Needs | +| --- | --- | --- | --- | +| `Transformers::Image::Vips` | `transformers/image/vips` | `active_storage.transformers.image.vips` | libvips | +| `Transformers::Image::Magick` | `transformers/image/magick` | `active_storage.transformers.image.magick` | ImageMagick | +| `Analyzers::Image::Vips` | `analyzers/image/vips` | `active_storage.analyzers.image.vips` | libvips | +| `Analyzers::Image::Magick` | `analyzers/image/magick` | `active_storage.analyzers.image.magick` | ImageMagick | +| `Analyzers::Video::FFprobe`, `Analyzers::Audio::FFprobe` | `analyzers/media/ffprobe` | `active_storage.analyzers.media.ffprobe` | `ffprobe` | +| `Previewers::Pdf::Mutool` | `previewers/pdf/mutool` | `active_storage.previewers.pdf.mutool` | `mutool` | +| `Previewers::Pdf::Poppler` | `previewers/pdf/poppler` | `active_storage.previewers.pdf.poppler` | `pdftoppm` | +| `Previewers::Video::FFmpeg` | `previewers/video/ffmpeg` | `active_storage.previewers.video.ffmpeg` | `ffmpeg` | + +The `Magick` classes replace Rails' `variant_processor = :magick`. If your application uses ImageMagick +today, use `Magick` wherever the README uses `Vips`, and `magick` wherever it uses `vips`. + +If an image that installs ImageMagick serves any of these operations, set ImageMagick's own limits. libvips +also delegates PSD, BMP, and ICO to ImageMagick. See [ImageMagick](imagemagick.md). + +### Mix with Rails' own classes + +Rails' own classes mix freely with these in the `analyzers` and `previewers` arrays, so you can move only +some work into a cell: + +```ruby +# PDF previews handled by HotCell, video previews still in the application +config.active_storage.previewers = [ ActiveStorage::HotCell::Client::Previewers::Pdf::Mutool, + ActiveStorage::Previewer::VideoPreviewer ] +``` + +### Route an operation to another cell + +Every client class talks to the cell registered as `active_storage` by default. A cell carries one +toolchain by design, so to send one operation to a different cell, name that cell on the operation's +client class, for example in an initializer: + +```ruby +ActiveStorage::HotCell::Client::Operations::Previewers::Video::Ffmpeg.hotcell "video" +``` + +Register that cell too. See [Client API](client-api.md#register-a-cell). + +## Remove packages from the application image + +After a file type's processing moves into the cell, remove its packages, such as libvips, from the +application image. That removal is the security improvement. + +**Caution:** Remove a package only after the cell handles that file type. Rails' own previewers and +analyzers look for their tool in `accept?`: `MuPDFPreviewer.accept?` calls `mutool_exists?`, and +`VideoPreviewer.accept?` calls `ffmpeg_exists?`. A package removed too early turns that processing off, +with no error and nothing in a log. The Hot Cell previewers' `accept?` doesn't look for a binary. + +## Failures + +The client raises the cell's `permanent:` or `transient:` class. See [Response codes](codes.md). + +- **Analyzers** rescue the permanent class and return no metadata, so Rails marks the blob analyzed. They + don't rescue the transient class. See [What Active Storage records](codes.md#what-active-storage-records). +- **Jobs** retry the transient class. At boot and on every code reload, the gem adds + `retry_on , wait: :polynomially_longer, attempts: 10` to `ActiveStorage::AnalyzeJob`, + `ActiveStorage::CreateVariantsJob`, `ActiveStorage::PreviewImageJob`, and `ActiveStorage::TransformJob`, + for each one that this Rails version has. Each registered cell that a shipped client names contributes + its transient class. A cell that isn't registered yet contributes nothing. + +## Limits + +The shipped operations declare these limits. A cell clamps each one to its own limits. To change one, see +[Change a shipped operation's limits](operation-api.md#change-a-shipped-operations-limits). + +| Routing name | `deadline` | `memory` | `file_size` | `open_files` | +| --- | --- | --- | --- | --- | +| `active_storage.transformers.image.vips` | 30 | 1280MB | 48MB | 256 | +| `active_storage.transformers.image.magick` | 30 | 1280MB | 48MB | 256 | +| `active_storage.analyzers.image.vips` | 10 | 1024MB | 48MB | 64 | +| `active_storage.analyzers.image.magick` | 10 | 1024MB | 48MB | 64 | +| `active_storage.analyzers.media.ffprobe` | 30 | 1024MB | 48MB | 128 | +| `active_storage.previewers.pdf.mutool` | 30 | 1024MB | 48MB | 128 | +| `active_storage.previewers.pdf.poppler` | 30 | 1024MB | 48MB | 128 | +| `active_storage.previewers.video.ffmpeg` | 120 | 1536MB | 128MB | 128 | + +`MB` means 1024² bytes. + +The 48MB is arithmetic from the README's accessory (four workers, two staged files each, on a 512MB +tmpfs), not a property of any operation. An operation that reads its input through the descriptor stages +only its output, so it needs half of what that arithmetic assumed. The shipped operations all read their +input through the descriptor. + +A cell must allow the highest of these limits for every operation that it carries. See +[Tuning](tuning.md#size-against-the-container). diff --git a/docs/cell-settings.md b/docs/cell-settings.md new file mode 100644 index 0000000..2e03634 --- /dev/null +++ b/docs/cell-settings.md @@ -0,0 +1,147 @@ +--- +type: Reference +title: "Cell settings" +order: 8 +description: "HotCell.limits: the scheduling settings and request limits with their defaults, the environment variables, the load order, and development mode." +sources: + - hotcell-server/lib/hot_cell/configuration.rb + - hotcell-server/lib/hot_cell/limits.rb + - hotcell-server/lib/hot_cell/server.rb + - hotcell-server/exe/hotcell + - hotcell-client/lib/hot_cell/install/Dockerfile.tt +--- + +# Cell settings + +This page lists the settings that a cell reads at boot: its scheduling settings, its request limits, the +environment variables, and the order in which it loads files. For the container flags, see +[Container](container.md). For how to choose the numbers, see [Tuning](tuning.md). + +The code lives in +[`hotcell-server/lib/hot_cell/configuration.rb`](../hotcell-server/lib/hot_cell/configuration.rb) and +[`hotcell-server/lib/hot_cell/limits.rb`](../hotcell-server/lib/hot_cell/limits.rb). + +## Configure a cell + +Call `HotCell.limits` once, in `hotcell/config.rb`. Every setting has a default, so set only what you +change. For example: + +```ruby +# hotcell/config.rb +HotCell.limits concurrency: 4, queue_size: 8, queue_wait: 10, deadline: 30, + max_requests_per_worker: 1, control_deadline: 5, + memory: 1536 * 1024**2, file_size: 48 * 1024**2, open_files: 256 +``` + +An unknown setting raises `HotCell::ConfigurationError`, and so does an explicit `nil` for a request +limit. + +## Scheduling settings + +| Setting | Default | Description | +| --- | --- | --- | +| `concurrency` | `4` | Workers that run at once, and the number of slots. Start at twice `cpus`. See [Settings that trade isolation](#settings-that-trade-isolation). | +| `queue_size` | `8` | Connections that can wait for a worker. When `running + queued` reaches `concurrency + queue_size`, the cell answers `capacity`. To refuse instead of queueing, use `0`. | +| `queue_wait` | `10` | Seconds that a queued connection can wait before the cell answers `capacity`. A saturated cell then answers with a verdict instead of holding the caller until the caller's own timeout. | +| `max_requests_per_worker` | `1` | Requests that one worker serves before the cell discards it. `1` forks for each request. `:unlimited` keeps a worker for the life of the cell. A reused worker skips the copy-on-write cost of a new fork, which is a performance setting with an isolation cost. See [Settings that trade isolation](#settings-that-trade-isolation). | +| `control_deadline` | `5` | Seconds that a control connection can take to send its request. | +| `sweep_interval` | `10` | How often, in seconds, the supervisor looks for the directories that killed requests left behind. When it finds one, it forks a sweeper process to delete them. The sweeper runs under `deadline` like a worker, so the supervisor and the requests in flight never wait on the deletion. | + +## Request limits + +These four settings are sized like performance settings, and they exist as limits on a hostile input. + +| Setting | Default | Description | +| --- | --- | --- | +| `deadline` | `60` | Maximum wall-clock seconds for one request. The supervisor kills the worker's process group and answers `killed` with cause `deadline`. This is the only bound on a stuck or deliberately slow input. | +| `memory` | `1536MB` | `RLIMIT_DATA` for each worker, and the bound on a decompression bomb. A breach answers `killed` with cause `memory`. The floor is `1024MB`: a cell or an operation below it raises `HotCell::ConfigurationError` and doesn't boot. | +| `file_size` | `64MB` | `RLIMIT_FSIZE` for each worker. A breach raises `SIGXFSZ` and answers `killed` with cause `fsize`. | +| `open_files` | `256` | `RLIMIT_NOFILE` for each worker. | + +`MB` here means 1024² bytes. Every limit must be positive. + +An operation can declare its own `limits`. Those values narrow the limit for that operation's requests, +and never widen it, because the cell's values are the ceiling. See +[Operation API](operation-api.md#limitsvalues). This is [invariant 6](design/invariants.md). + +`memory` doesn't multiply by `concurrency`. It's an address-space charge on one worker, and about 450MB of +it is Ruby's own reservation. See [Tuning](tuning.md#sizing-guidelines). + +On macOS, `RLIMIT_DATA` can't be set, so a cell there runs with `memory` unenforced and warns once. Every +other limit is enforced on both platforms. See [experiment 17](contributing/experiments.md). + +### Values that you can't configure + +- The one-second grace between the signal to a worker and the kill of its process group. +- The absence of an `RLIMIT_CPU`. The deadline covers CPU use, and it also catches a worker that is blocked + on a stuck subprocess, which uses no CPU. `HotCell::Limits` explains the rest. +- The sockets' file mode, `0660`. See [Client API](client-api.md#the-shared-group). + +## Settings that trade isolation + +**`max_requests_per_worker` above `1`** lets one request reach another. A worker holds each of its +requests in the same address space, so an input that runs code can read and change every later request +that the worker serves. [ADR 0001](../adr/0001-reuse-workers-across-requests.md) records the measurements +and the trade. + +At `:unlimited`, it also gives up the deadline. A worker reports itself idle when it finishes, and the +supervisor can't tell a true report from a false one, so a worker that lies stops being timed. At every +finite setting, the supervisor still retires the worker and kills it after a grace period. At `:unlimited`, +nothing retires it, and the cell can't end it. Prefer a finite number, however large. + +**`concurrency`** sets how many requests hold bytes in the cell at once. Files aren't isolated between +concurrent workers, so this value is also the width of that exposure. See +[Worker isolation](design/worker-isolation.md). + +## Environment variables + +The installed `Dockerfile` sets all of these unless the table says otherwise, so set one only to override +it. + +| Variable | Default | Description | +| --- | --- | --- | +| `HOTCELL_DIR` | `/run/hotcell/cell` | Where the cell creates `work.sock` and `control.sock`. The application must reach the same directory. | +| `HOTCELL_OPERATIONS` | `/hotcell/operations` | The directory of operation files that the cell loads at boot. See [Load order](#load-order). | +| `HOTCELL_CONFIG` | `/hotcell/config.rb` | The configuration file that the cell loads before the operations, if the file exists. | +| `HOTCELL_WORKSPACE` | a directory under `TMPDIR` | Where the cell makes and removes each request's directory. On the default accessory, this is the tmpfs. Must be absolute. Emptied at boot. See [Scratch](scratch.md#boot-sweep). | +| `TMPDIR` | unset, so `/tmp` | The scratch. Emptied at boot of every entry that the cell's uid owns. On the accessory, `/tmp` is the cell's own mount. Under `hotcell --development`, it's only the scratch's parent and is never swept. See [Scratch](scratch.md#boot-sweep). | +| `HOTCELL_HEALTH_TIMEOUT` | `5` | Seconds that `hotcell-health` waits for an answer before it reports unhealthy. | +| `HOME` | `/tmp` | Bundler needs a home directory, and the cell's user has none. A worker replaces `HOME` with a directory made for the request and removed with it. | +| `OMP_NUM_THREADS` | `2` | The OpenMP pool size that libvips and ImageMagick use. Match it to `cpus`. See [Bound the OpenMP thread pools](container.md#bound-the-openmp-thread-pools). | +| `OMP_THREAD_LIMIT` | `8` | The ceiling on that pool, including for a library that raises the count itself. | +| `MAGICK_MEMORY_LIMIT`, `MAGICK_MAP_LIMIT`, `MAGICK_DISK_LIMIT` | unset | ImageMagick's pixel cache limits. The installed `Dockerfile` installs no ImageMagick. An image that installs it sets these from the container's `memory`, the scratch, and `concurrency`. See [ImageMagick](imagemagick.md). | + +## Load order + +At boot, the cell does the following, in order: + +1. Loads `HOTCELL_CONFIG`, if the file exists. +2. Requires every `.rb` file under `HOTCELL_OPERATIONS`, including subdirectories, in sorted path order. + Requiring an operation's file is what makes the cell serve it. +3. Runs every operation's `before_fork` blocks. + +For an operations file that redeclares a shipped operation's limits, see +[Change a shipped operation's limits](operation-api.md#change-a-shipped-operations-limits). + +A cell for custom operations names `hotcell-server` in its `Gemfile` directly, plus the gems that its +operations use. It needs the Active Storage server gem only for the shipped operations: + +```ruby +# hotcell/Gemfile -- the cell +gem "hotcell-server" +gem "my_image_processor" +``` + +## Development mode + +`hotcell --development` boots a cell as a plain process, without a container, for example beside the Rails +server in `Procfile.dev`. The resource limits and the deadline apply as they do in a container. + +At boot, a cell empties its `TMPDIR` of every entry that its uid owns. When `TMPDIR` is unset, it empties +the system temporary directory. On a developer's machine, that's `/tmp`, or on macOS the per-user +`TMPDIR` that every shell sets, and both are shared with everything else that the developer runs. + +With `--development`, the cell never sweeps the directory that it's given. Its scratch is +`hotcell-` beneath that directory. `HOTCELL_WORKSPACE` defaults to a +directory under the scratch. If you point it elsewhere, the cell sweeps its parent too, with or without +`--development`. See [Scratch](scratch.md#boot-sweep). diff --git a/docs/client-api.md b/docs/client-api.md new file mode 100644 index 0000000..29a6fbe --- /dev/null +++ b/docs/client-api.md @@ -0,0 +1,240 @@ +--- +type: Reference +title: "Client API" +order: 3 +description: "HotCell.register and its options, HotCell::Client, the errors a bad call raises, the boot checks, diagnosis, and the group the application shares with a cell." +sources: + - hotcell-client/lib/hot_cell/cells.rb + - hotcell-client/lib/hot_cell/cell.rb + - hotcell-client/lib/hot_cell/client.rb + - hotcell-client/lib/hot_cell/diagnosis.rb + - hotcell-client/lib/hot_cell/failures.rb + - hotcell-core/lib/hot_cell/errors.rb + - hotcell-core/lib/hot_cell/payload.rb +--- + +# Client API + +This page describes the application side of HotCell: registering cells with `HotCell.register`, writing +`HotCell::Client` subclasses, the boot checks, and the group that the application shares with each cell. +For the cell side of the call, see [Operation API](operation-api.md). + +The code lives in [`hotcell-client/lib/hot_cell/`](../hotcell-client/lib/hot_cell/), mainly `cells.rb`, +`cell.rb`, and `client.rb`. + +## Configure Hot Cell + +Configure Hot Cell in an initializer. Pass exception classes that fit your domain; your own base classes +can be useful when you wrap an existing library. + +```ruby +HotCell.root = ENV["HOTCELL_ROOT"] # unset turns every cell off +HotCell.group = ENV["HOTCELL_GROUP"] # the cell's gid; unset where both sides are one user + +HotCell.register "images", + timeout: 30, + permanent: ActiveStorage::PreviewError, + transient: MyApp::ConversionTemporarilyUnavailable, + on_contract_skew: ->(error, cell) { Sentry.capture_exception(error) } +``` + +| Setting | Default | Description | +| --- | --- | --- | +| `HotCell.root` | none | The parent directory that cell names resolve under. A cell's sockets are in `/`. | +| `HotCell.group` | none | The gid that both sides hold, so that a cell can open a caller's file by name. Set it to the cell's gid. A value that isn't a number raises `HotCell::ConfigurationError`. Leave it unset only where both sides already run as one user, which is how development runs. See [The shared group](#the-shared-group). | +| `HotCell.diagnostics_controller_parent` | `"ActionController::Base"` | The name of the superclass of `HotCell::DiagnosticsController`. Set it in an initializer, because the controller reads it when it loads. See [Observability](observability.md#rails-healthcheck). | +| `HotCell.logger` | a `Logger` on standard error | Where the boot checks write their warnings. | + +### Turn cells off + +When `HotCell.root` is unset and a cell has no `dir:`, the cell is off: `enabled?` returns `false`, and +`perform_in_hotcell` raises `HotCell::CellNotConfigured`. Hot Cell has no automatic in-process fallback. +A caller that wants one checks `enabled?` and takes its old path. This is how an application rolls out a +cell as a configuration change rather than a release. + +The shipped Active Storage classes don't check `enabled?`. With no root, every variant, analysis, and +preview that they handle raises `HotCell::CellNotConfigured`. + +## Register a cell + +`HotCell.register(name, **options)` registers one cell. Call it once for each cell, at boot. + +| Option | Default | Description | +| --- | --- | --- | +| `dir:` | `/` | An explicit socket directory for this cell. To make a change of path a configuration change instead of a deploy, pass a lambda: the client resolves the directory on every call. | +| `timeout:` | `30` | Seconds that this caller waits for an answer to a work request. It must be more than the cell's `answer_within`. See [Tuning](tuning.md#make-the-timeouts-agree). | +| `control_timeout:` | `5` | Seconds that this caller waits for `describe` or `metrics`. The supervisor answers both inline, with no fork and no queue, so keep this short and don't raise it toward `timeout`. It bounds app boot when a cell accepts connections and never answers, and it lets a health check report a stuck cell instead of waiting out a work timeout. | +| `permanent:` | `HotCell::PermanentFailure` | The exception class that the client raises for a permanent failure. See [Response codes](codes.md#exception-classes). | +| `transient:` | `HotCell::TransientFailure` | The exception class that the client raises for a transient failure. It must not descend from `permanent:`. | +| `on_contract_skew:` | none | A callable, `->(error, cell)`, that the client calls when a cell answers `protocol`, before it raises. Use it to make a protocol version mismatch visible to an application that rescues broadly. | + +`timeout:` and `control_timeout:` must be positive, finite numbers. `register` raises +`HotCell::ConfigurationError` for anything else, including `nil`, so a bad value stops the boot rather +than leaving the wait for a cell to decide. `register` also raises `HotCell::ConfigurationError` when +`transient:` descends from `permanent:`. + +Both timeouts bound the answer, not the connection. See [Boot checks](#boot-checks). + +### Look up a cell + +| Method | Description | +| --- | --- | +| `HotCell.cell(name)` | Returns the registered cell. Raises `HotCell::UnregisteredCell` if no cell has that name. | +| `HotCell.cell?(name)` | Returns whether a cell with that name is registered. | +| `HotCell.cells` | Returns a `Hash` of every registered cell, by name. | + +## Write a client + +A client is a subclass of `HotCell::Client`: + +```ruby +class TransformImage < HotCell::Client + hotcell "images" # the cell that serves this call, by registered name + operation "images.transform" # the routing name an operation must answer to +end + +result = TransformImage.perform_in_hotcell source, destination, format: "png" +``` + +| Class method | Description | +| --- | --- | +| `hotcell(name)` | Names the registered cell that serves this client. A subclass inherits it. | +| `operation(name)` | Sets the routing name. A subclass doesn't inherit it. By default, the name derives from the class path the same way an operation's does. See [Operation API](operation-api.md#operationname--nil). | +| `perform_in_hotcell(inputs, outputs, payload = {})` | Sends the request and blocks until the cell answers or `timeout` passes. Returns the operation's result. | +| `enabled?` | Returns whether the client's cell has a socket directory. | +| `registered?` | Returns whether the client's cell is registered. Use it in a boot hook that must not raise in an application that hasn't written its initializer yet. | + +`perform_in_hotcell` accepts the following: + +- `inputs` and `outputs`: one IO, or an `Array` of IOs. Each input must be open read-only and each output + write-only, and each must be a regular file not opened with `O_APPEND`. A descriptor that fails these + checks raises `HotCell::AccessModeError` before anything is sent. +- `payload`: a `Hash` that JSON can carry faithfully. A value that JSON would change, such as a `Symbol` + or a `Time`, raises `HotCell::SerializationError` before anything is sent. + +On failure, `perform_in_hotcell` raises the cell's `permanent:` or `transient:` class. Either way, it +publishes a `perform.hot_cell` notification. See +[Observability](observability.md#per-call-notification). + +### Errors that the client raises for a bad call + +These classes descend from `HotCell::Error`. The client raises them for a mistake in the call itself, +outside its transport rescue, so an application whose transient class descends from `IOError` can't +retry them forever. + +| Class | Raised when | +| --- | --- | +| `HotCell::ConfigurationError` | Either side is configured wrongly. Raised at boot where possible. | +| `HotCell::UnregisteredCell` | A client names a cell that isn't registered. | +| `HotCell::CellNotConfigured` | A client calls a cell that has no socket directory. | +| `HotCell::AccessModeError` | A descriptor has the wrong access mode, isn't a regular file, or is open with `O_APPEND`. | +| `HotCell::SerializationError` | A payload or result value can't travel as JSON faithfully. | +| `HotCell::MessageError` | A message is unparseable, too long, or missing a field. | +| `HotCell::ReadTimeout` | A peer stopped partway through a message. | + +## Boot checks + +`HotCell.describe_cells` asks each registered cell for `describe` over its control socket. Call it once at +boot, after registering: + +```ruby +Rails.application.config.after_initialize { HotCell.describe_cells } +``` + +It returns each cell's description, by name, and warns through `HotCell.logger` about the following: + +- This process doesn't hold `HotCell.group`. This check is local and needs no cell, so it reports a + missing `group-add` even when every cell is down. +- The cell is unreachable. +- The client's `timeout` doesn't clear the cell's `answer_within`. +- The cell doesn't run in `HotCell.group`. `describe` reports the groups that a cell holds. The number in + your deploy file and the gid built into the cell's image come from different places, so nothing else + catches a cell image whose gid moved. A cell too old to report its groups gets no warning. +- The cell's `hotcell-server` version differs from this `hotcell-client` version. + +`describe_cells` warns and never raises. A cell that's down at boot is a degraded deployment rather than a +broken one. + +The description comes from the process that runs untrusted content, so the client reads it inside a +rescue. The client logs and ignores an answer that it can't use, and treats the cell as one that answered +nothing. This catches a misdeployment, not a compromised cell: a compromised cell can send a well-formed +description that is false, and the client can't tell. + +The rescue has the following limits: + +- It covers what a cell says, not a cell that never answers at boot. `control_timeout` bounds the answer + and not the connection, so a listener that stops accepting until its backlog fills can hold the + connect. See [#20](https://github.com/basecamp/hotcell/issues/20). +- It covers what raises at boot, not what a value does later. `describe_cells` returns the description + whole, so it can hold a string that isn't valid UTF-8, or a number that parses to infinity. Validate the + returned description for whatever you do with it: serializing, matching, storing, and rendering can all + raise on these values. + +### Diagnose cells + +`HotCell.diagnose(work: false)` runs checks against every registered cell and returns a +`HotCell::Diagnosis`. Call `as_json` for the result: the time, the host, `healthy`, and each cell's checks. + +| Check | Socket | Description | +| --- | --- | --- | +| `describe` | control | The cell answers `describe`. | +| `metrics` | control | The cell answers `metrics`. | +| `echo` | work | A `health.echo` round trip. Only with `work: true`. | +| `reopen` | work | A `health.reopen` round trip, which proves that the cell can open an input by name. Only with `work: true`. | + +`healthy` is true when at least one cell is registered and every check passes. The work checks need +`require "hot_cell/health_operations"` in one of the cell's operation files. See +[Observability](observability.md#rails-healthcheck) for the controllers that serve these checks. + +## The shared group + +Set `HotCell.group` to the cell's gid, and put the application in that group: + +```yaml +# config/deploy.yml — the app +servers: + web: + options: + group-add: 10001 +``` + +The group is required for the following reasons: + +- **Connecting.** The cell creates both sockets with mode `0660`, owned by its own user and group. A UNIX + socket needs write permission to connect, so without the group, the application can't reach the cell. +- **Opening files by name.** An operation that hands a tool a filename doesn't copy the input. It opens + the descriptor again as `/dev/fd/N`. That's a new open, and the kernel checks it against the cell's + user, not the caller's. The two sides don't share a uid, so a mode `0600` file that the application owns + gives `EACCES`. An Active Storage tempfile is such a file, and every shipped Active Storage operation + hands a tool a filename. + +Without the group, every call fails immediately with `EACCES` from `connect`. There's no partly working +state to mistake for healthy, and `HotCell.describe_cells` reports it at boot, before any traffic depends +on it. + +### What the client does with the group + +The client puts each descriptor in the group and sets its mode before sending: `0640` for an input and +`0620` for an output. It does this through the open file, with `fchown` and `fchmod`, so it names no path. + +**Caution:** The client doesn't restore the group or the mode. The file that you pass keeps the cell's +group and the new mode after the call returns, so a `0600` file of your own comes back readable by that +group. Active Storage passes tempfiles and unlinks them, so this doesn't matter there. If you write your +own client, pass files that you're willing to share with the cell's group, and don't pass a file whose +permissions something else depends on. + +These modes enforce [invariant 4](design/invariants.md): a cell can read an input and write an output, +and can't write an input or read an output. The cell can't widen either mode, because it doesn't own the +files, and `cap-drop ALL` leaves it no capability that overrides a mode. The application must own the +files and be in the group to set the group at all. + +### Don't share a uid instead + +Sharing a uid between the application and the cell is the obvious shortcut, and it costs two things: + +- The cell would own the files, so it could set any mode, and inputs and outputs would stop being + one-way. +- A cell that escaped its container would run as the application's own user on the host, where it can + read the environment of the application's processes. + +Keep the uids separate and share only the group. diff --git a/docs/codes.md b/docs/codes.md new file mode 100644 index 0000000..3e94557 --- /dev/null +++ b/docs/codes.md @@ -0,0 +1,175 @@ +--- +type: Reference +title: "Response codes" +order: 5 +description: "Every failure code and kill cause, whether each is permanent or transient, the exception classes the client raises, and what Active Storage records." +sources: + - hotcell-core/lib/hot_cell/codes.rb + - hotcell-core/lib/hot_cell/failure.rb + - hotcell-client/lib/hot_cell/failures.rb + - hotcell-server/lib/hot_cell/worker.rb + - activestorage-hotcell-client/lib/active_storage/hot_cell/client/analyzers/analyzing.rb +--- + +# Response codes + +Every failed call carries a code, and every code is permanent or transient. This page lists the codes, the +causes of `killed`, and how the Active Storage integration records a permanent failure. + +The permanent or transient split is the only distinction that changes what a caller must do: + +- **Permanent:** the same request fails the same way until the input, the code, or the limits change. A + change in load doesn't fix it. A caller can record a permanent failure against the input, for example + against an Active Storage blob, and serve it from a cache. +- **Transient:** the request might succeed on a retry. A caller must retry a transient failure and must + never record it. + +The definitions live in [`hotcell-core/lib/hot_cell/codes.rb`](../hotcell-core/lib/hot_cell/codes.rb). + +## Codes + +| Code | Split | Meaning | +| --- | --- | --- | +| `unreadable` | Permanent | The operation couldn't decode the input. The operation said so explicitly: it raised one of the classes that it declared with `unreadable`. | +| `invalid` | Permanent | The request is malformed, or a descriptor failed its access-mode check. | +| `failed` | Transient | The operation raised an exception that nothing classified. | +| `unsupported` | Transient | The cell doesn't carry the requested operation. | +| `protocol` | Transient | The two sides speak different protocol versions. It heals when the accessory reboots on an image with the application's protocol version. | +| `capacity` | Transient | The cell's queue is full, or a queued request waited longer than `queue_wait`. | +| `unavailable` | Transient | The client couldn't connect, or the connection closed with no response. The client also reports `unavailable` when a cell reports success and writes no bytes to a non-empty set of outputs. | +| `timeout` | Transient | The client's own `timeout` passed before the cell answered. | +| `killed` | Depends on the cause | The supervisor killed the worker, or the worker died. See [Causes of `killed`](#causes-of-killed). | + +### Why `failed` is transient + +`failed` is what an unclassified exception becomes, so it can't be permanent. A worker rescues +`StandardError` around the whole request and reports `failed`. `Errno::ENOSPC`, `EMFILE`, `EIO`, `ENOENT`, +and `ENOMEM` are all `StandardError`s. Each of the following raises inside staging or writeback and arrives +as `failed`: + +- A shared tmpfs that concurrent requests filled. +- A full disk under the caller's own output. +- A descriptor table that load exhausted. +- A fork that can't get memory under host pressure. +- A tool that's missing during a broken deploy. + +`ENOMEM` looks like the input's fault and isn't. When an input drives the worker past its own memory +limit, Ruby raises `NoMemoryError`, which the worker reports as `killed` with cause `memory`. + +A permanent `failed` would record each of these against a customer's file forever, for a condition that +would succeed on a retry. Permanence must be claimed, never inferred from not knowing: an operation reports +`unreadable` for an input that it couldn't decode, and the protocol reports `invalid` for a caller that +broke its own contract. + +The cost of this choice is that a broken operation is retried. The job's attempts bound that cost, the +`failed` rate shows it, and it's recoverable. + +### Why `unsupported` is transient + +A Kamal deploy doesn't update an accessory. An application that ships a client for a new operation before +anyone reboots the cell gets `unsupported` on every request until the reboot. Recording that window as +permanent condemns every blob uploaded during it, and only a hand-written backfill undoes it. Retrying a +caller's typo costs some work, and the refusal names the operation in the worker's message and in the +`unsupported` rate. + +## Causes of `killed` + +A `killed` failure carries a `cause`. The cause decides the split, because a caller can't otherwise tell a +decompression bomb from a slow afternoon. + +| Cause | Split | Meaning | +| --- | --- | --- | +| `fsize` | Permanent | A write by the worker returned `EFBIG`: the request passed its `file_size` limit. | +| `memory` | Permanent | The worker caught `NoMemoryError`: the request passed its `memory` limit. | +| `deadline` | Transient | The request passed its `deadline`, and the supervisor killed the worker's process group. | +| `crashed` | Transient | The worker died without answering, for any other reason. | + +Size and memory are properties of the input, so the same bytes fail again on an idle cell. A deadline is +as much a property of the load: a permanent `deadline` would condemn whatever was uploaded during a busy +hour. + +`crashed` is the cell's own fault rather than the input's. A misconfigured cell crashes on every request, +so a permanent `crashed` would condemn everything uploaded during a broken deploy. A `crashed` failure +carries `signal` when a signal ended the worker, and `stderr` when the worker wrote to file descriptor 2. +See [What a worker wrote to fd 2](observability.md#what-a-worker-wrote-to-fd-2). + +The supervisor never infers `memory` or `fsize` from a signal. A signal says how a process died, never +why. Apart from the `SIGKILL` that the supervisor sends for a deadline, every signal comes from somewhere +that the supervisor can't see: a cgroup out-of-memory kill chosen on aggregate pressure, or one worker +signalling another, which nothing prevents because workers share a uid. The worker that holds the request +decides `memory` and `fsize` itself. + +## Unknown codes and causes + +A code or cause that a client doesn't recognize isn't permanent: + +- The `permanent` flag travels on the wire, set by the side that knows. A client that's older than a code + still disposes of that code correctly. +- When the wire carries no boolean `permanent`, the client derives it. An unknown code is transient. +- An unknown `killed` cause is transient. Adding a kill reason to the supervisor without a row in + `Codes::PERMANENT_BY_CAUSE` can't make it permanent. + +Retrying something permanent costs some work. Recording something transient is irreversible. + +**Caution:** A compromised cell can forge the split. The client believes a boolean `permanent` from the +wire, and a worker that stole `work.sock` can write any answer. See +[Worker isolation](design/worker-isolation.md). + +## Exception classes + +The client raises one exception class for permanent failures and another for transient ones. Set them +with the `permanent:` and `transient:` options of `HotCell.register`. See +[Client API](client-api.md#register-a-cell). + +By default, the client raises `HotCell::PermanentFailure` and `HotCell::TransientFailure`. No application +rescues these classes already, so an unclassified failure surfaces as an error rather than as a silent +permanent mark. `transient:` must not descend from `permanent:`, because the inheritance graph is the +classification. + +The exception's message is the failure's `code`, `cause`, error class, and message, joined with `: `, then +the `stderr` tail in parentheses when it exists. For example, `killed: crashed (libgomp: ...)`. + +## What Active Storage records + +A permanent verdict is irreversible only if the application records it. In the shipped Active Storage +integration, analysis records it and nothing else does. + +Rails persists a blob's analysis like this: + +```ruby +# ActiveStorage::Blob::Analyzable +def analyze + update! metadata: metadata.merge(extract_metadata_via_analyzer) +end + +def extract_metadata_via_analyzer + analyzer.metadata.merge(analyzed: true) +end +``` + +Rails merges `analyzed: true` whatever the analyzer returned, including an empty hash, and never checks +whether the analysis worked. A permanent failure takes this path: + +1. The blob is attached. Rails enqueues `ActiveStorage::AnalyzeJob` once. +2. The analyzer calls the cell and gets `killed` with cause `memory`. The client raises the application's + permanent class. +3. `Analyzers::Analyzing#metadata` rescues that class, logs it, and returns `{}`. +4. Rails merges `analyzed: true` and writes the row. + +The blob's `metadata` is now `{"identified"=>true, "analyzed"=>true}`: analyzed, with no dimensions. +Nothing enqueues the job again, because `analyze_later` runs once, at first attachment. + +`Analyzers::Analyzing#metadata` doesn't rescue a transient failure. The failure escapes into the job, the +job retries, and `analyzed` stays false. + +To undo a permanent failure, backfill the blob: + +```ruby +blob.update!(metadata: blob.metadata.except("analyzed")) +blob.analyze_later +``` + +Previews and variants record no durable failure. `Preview#processed?` is `image.attached?`, and a variant +is recorded by its `active_storage_variant_records` row. A failure attaches nothing and creates nothing, so +the job retries. For this reason, only analysis needs the generous-first order in +[Tuning](tuning.md#start-generous-on-memory-and-file_size). diff --git a/docs/concepts.md b/docs/concepts.md new file mode 100644 index 0000000..9120a93 --- /dev/null +++ b/docs/concepts.md @@ -0,0 +1,123 @@ +--- +type: Glossary +title: "Concepts" +order: 1 +description: "The terms the Hot Cell documentation uses: cell, supervisor, worker, slot, operation, client, tool, payload, inputs and outputs, and codes." +sources: + - hotcell-server/lib/hot_cell/supervisor.rb + - hotcell-server/lib/hot_cell/worker.rb + - hotcell-server/lib/hot_cell/slot.rb + - hotcell-server/lib/hot_cell/operation.rb + - hotcell-client/lib/hot_cell/client.rb + - hotcell-core/lib/hot_cell/descriptors.rb + - hotcell-core/lib/hot_cell/naming.rb +--- + +# Concepts + +This page defines the terms that the rest of the Hot Cell documentation uses. The +[design overview](design/threat-model.md#what-this-is) explains the "hot" and "cold" vocabulary. + +## Cell + +A cell is one deployment of the hot side: a supervisor, the workers that it forks, and the operations +that they serve. An application reaches a cell through two UNIX sockets in one directory, `work.sock` and +`control.sock`. + +A cell is the unit of isolation and the unit of capacity. To use more than one cell, call +`HotCell.register` once for each cell. See [Client API](client-api.md#register-a-cell). + +## Supervisor + +The supervisor is process 1 in the cell. It does the following: + +- Accepts connections and queues them. +- Dispatches each connection to a worker. +- Enforces the wall-clock deadline from outside the worker. +- Kills and reaps workers. + +The supervisor never evaluates image data, and it doesn't read a request that it dispatches. It passes the +accepted connection itself to a worker over `SCM_RIGHTS` and never calls `recvmsg`, so the caller's +descriptors are still queued on the connection when the worker reads them. When a dispatch fails, the +supervisor peeks at the request to name its operation in the `worker.undispatchable` event. + +## Worker + +A worker is a child process that the supervisor forks before a request needs it. The supervisor forks one +worker for each slot at boot, and forks a replacement as soon as it reaps a worker that served a request. + +A worker is the only process that touches untrusted bytes. It applies the cell's resource limits before it +touches the socket, serves `max_requests_per_worker` requests, and then exits without running finalizers. + +## Slot + +A slot is the numbered workspace that a worker borrows. A slot holds one directory for each request. That +directory is the request's `$HOME`, and the request's staged files are in it. + +The worker creates the directory when the request starts and removes it before the caller receives the +answer. Nothing that a tool writes reaches the next request on that slot. + +## Operation + +An operation is the unit of work that a cell offers. It's a subclass of `HotCell::Operation` with the +following: + +- A routing name. +- Its own `limits`. +- A `perform(inputs, outputs, **payload)` method that declares the payload keys that it accepts as keyword + arguments. + +By default, both sides derive the same routing name from the class path, and the cell side strips the +`Operation` suffix. For example, `ExtractTextOperation` in the cell and `ExtractText` in the application +both answer to `extract_text`. See [Operation API](operation-api.md). + +## Inventory + +A cell's inventory is the set of operations that it carries. The cell logs its inventory at boot, in +`cell.boot`, and reports it on the control socket, in `describe`. + +## Client + +A client is the application-side counterpart of an operation. It's a subclass of `HotCell::Client` that +names the cell with `hotcell` and the operation with `operation`, and provides `perform_in_hotcell`. + +`perform_in_hotcell` is a blocking call. It sends the request and waits for the cell's answer, up to the +`timeout` that the cell was registered with. See [Client API](client-api.md). + +## Tool + +A tool is a program that an operation runs in a subprocess with `run_tool`, for example `mutool` or +`ffmpeg`. The tool gets a fully written environment, and `run_tool` bounds how much of its output is +captured. The worker waits for the tool, and the tool sees only the environment that its operation wrote +for it. + +Not every operation uses a tool. For example, the Vips operations call libvips directly from the Ruby +worker process. + +## Payload and result + +The payload is a `Hash` of options. It travels in the request as one JSON object and arrives in `perform` +as keyword arguments. + +The result is the `Hash` that an operation returns. It travels in the response the same way. + +Neither the payload nor the result carries file contents. + +## Inputs and outputs + +Inputs and outputs are the open file descriptors that a caller passes. Inputs are read-only and outputs +are write-only. They are the only way that file contents enter or leave a cell. + +An operation can read and write the descriptors directly, which is the efficient path. To give a tool a +file on the worker's scratch, ask the descriptor for its `path`: + +- An input copies its bytes onto scratch the first time that the operation asks. +- An output's file is sent back through the descriptor when `perform` returns. + +See [Inputs and outputs](operation-api.md#inputs-and-outputs). + +## Code + +Every failure carries a code, and each code is permanent or transient. A permanent failure means that the +input fails this way every time, so the caller can record that verdict against the input. Every uncertain +failure is transient, which means that it might succeed on a retry. See [Response codes](codes.md). diff --git a/docs/conformance.md b/docs/conformance.md new file mode 100644 index 0000000..0f82c1a --- /dev/null +++ b/docs/conformance.md @@ -0,0 +1,86 @@ +--- +type: Reference +title: "Conformance" +order: 11 +description: "What bin/conformance checks about a cell image, and what it cannot check." +sources: + - bin/conformance + - bin/example-image + - examples/conformance + - examples/lib +--- + +# Conformance + +This page describes `bin/conformance`, which checks whether a container image can run a cell, and what it +can't check. Use it when you build your own image instead of the one that `hotcell:install` writes. + +## Run the conformance checks + +``` +docker build -t my-cell:test hotcell/ +bin/conformance my-cell:test +``` + +`bin/conformance IMAGE` does the following: + +1. Boots the image with the flags in [Container flags](container.md#container-flags). +2. Mounts the example operations over `/hotcell/operations`. +3. Runs a battery of checks from a second container over a shared volume: descriptor round trips, each + kill verdict, refusal at capacity, and the isolation flags. +4. Runs the same battery again against the image booted without one security flag at a time, and requires + each run to fail at that flag's own check. + +It exits non-zero on the first failed check. + +The negative runs make the isolation results a test of the flags rather than a restatement of them: a +check that can't see a flag would pass a cell running without it. A runtime that force-mounts `noexec` +onto every tmpfs, as Docker Desktop does, can't host the `noexec` negative, and that one run reports +`SKIP` with the reason. + +The mount hides the operations that your image carries. The checks cover the image's runtime (its Ruby, +its gems, and its socket behavior), not the work that your operations do. + +`bin/example-image` builds an image to try it against. CI runs both on every push. + +## What conformance can't check + +### `kernel.yama.ptrace_scope` + +The host must set `kernel.yama.ptrace_scope` to 1 or higher. No container flag can set it, and it's what +stops one worker from reading another request's memory through `/proc//mem`. A cell refuses to boot +below it: + +``` +kernel.yama.ptrace_scope is 0 on this host, so one worker can read another request's memory +through /proc//mem. No container flag can set it. Set it to 1 or higher and boot again. +``` + +A host that doesn't expose the file at all logs `cell.ptrace_scope_unknown` and boots anyway. See +[Worker isolation](design/worker-isolation.md). + +### Some security flags + +Conformance reads these flags from inside the cell: + +- `network: none`: the only interface is `lo`. +- `read-only`: the root filesystem is mounted `ro`. +- The tmpfs `noexec` flag. +- `cap-drop`: the bounding capability set is empty. +- `no-new-privileges`. +- The uid. + +A flag that the cell can't read fails the check rather than passing it. + +Conformance doesn't observe the tmpfs `nosuid` and `nodev` flags, `pids-limit`, or the resource limits. +You must get those right yourself. To check them, see [Verify an accessory](container.md#verify-an-accessory). + +### The application's group membership + +Conformance checks the shared group from the cell's side, because it runs the cell as `10001` against +files that a different user owns: `health.reopen` proves that a cell can open an input by name, and +`example.tamper` proves that it can do nothing else with either file. + +It can't check the application's side. Its driver owns the files as root, which needs no group membership +to set a group. A missing membership fails as `EPERM` in the application rather than in the cell. See +[The shared group](client-api.md#the-shared-group). diff --git a/docs/container.md b/docs/container.md new file mode 100644 index 0000000..66167ed --- /dev/null +++ b/docs/container.md @@ -0,0 +1,193 @@ +--- +type: Reference +title: "Container" +order: 6 +description: "Building the cell's image, every container flag and what it does, bounding OpenMP, and checking an accessory before and after a deploy." +sources: + - hotcell-client/lib/hot_cell/install + - hotcell-client/test/install_test.rb + - bin/conformance +--- + +# Container + +This page describes the cell's container: how to build its image, what each container flag does, how to +bound OpenMP, and how to check an accessory before and after you deploy it. + +A cell is a second container beside the application, on the same host, that shares one volume holding its +UNIX sockets. The examples use Kamal. Any orchestrator that can set the same `docker run` flags works. The +complete Kamal configuration for both containers is in the README, under +[Build and deploy the cell](../README.md#5-build-and-deploy-the-cell). + +For the settings inside the cell, see [Cell settings](cell-settings.md). For how the numbers constrain +each other, see [Tuning](tuning.md). For where scratch lives, see [Scratch](scratch.md). To test an image +that you built yourself, see [Conformance](conformance.md). + +## Build an image + +Hot Cell publishes no base image. `bin/rails hotcell:install` writes a `hotcell/` directory into the +application that holds a complete `Dockerfile`, the cell's `Gemfile`, its `config.rb`, and an +`operations/` directory. That `Dockerfile` is the whole recipe, and you can customize it. Build it from +its own directory: + +``` +docker build -t your-image:latest hotcell/ +``` + +Keep the following in mind: + +- **Every gem is inside the blast radius.** Keep the cell's `Gemfile` short. A larger supervisor heap also + costs every request. See [Tuning](tuning.md#sizing-guidelines). +- **Load only the operations that your image has tools for.** A cell reports what it loaded in `describe`, + and a client checks that inventory at boot to catch a cell that doesn't carry an operation that it + wants. An operation whose tool is missing passes that check and fails at the first request instead. So + requiring an operation without installing its tool is worse than not requiring it. +- **The cell's image must create its own socket mount point**, owned by the cell's user: + `/run/hotcell/cell`, not only `/run/hotcell`. Docker creates a missing last level as root, and the cell + then can't create a socket in it. The installed `Dockerfile` does this. The application's image needs + nothing at its own mount point. +- **Keep the application's and the cell's lockfiles in step.** They resolve Hot Cell separately. + `HotCell.describe_cells` warns at boot when the cell's `hotcell-server` version differs from the + application's `hotcell-client` version. + +## Container flags + +These are `docker run` flags. Under Kamal, they go in an accessory's `options:`, and Kamal supplies no +value of its own for any of them. + +`network` is the exception, and it's the flag that the design depends on most. It's an accessory key of +its own, a sibling of `image:` and `roles:`. Kamal always emits its own `--network`, `kamal` by default. +An entry under `options:` adds a second `--network` rather than replacing the first, and Docker refuses +the container: + +``` +docker: conflicting options: cannot attach both user-defined and non-user-defined network-modes +``` + +Give each cell its own socket volume. Two accessories that share one would write `work.sock` over each +other. + +Without Kamal, the cell container needs `--volume hotcell-sockets:/run/hotcell/cell` and the security flags +below. The application container needs `--volume hotcell-sockets:/run/hotcell/active_storage`, +`--group-add 10001`, and `HOTCELL_ROOT=/run/hotcell`. + +### Performance flags + +These flags have no defaults: Docker applies no limit for a flag that you omit, so a cell without them can +take the whole host. Tune each one to your workload and your hardware. The examples are from the README's +accessory, and they aren't recommendations. + +| Flag | Example | Description | +| --- | --- | --- | +| `cpus` | `2` | The share of the host that this cell can use. Start the cell's `concurrency` at twice this number, and match the image's `OMP_NUM_THREADS` to it. See [Bound the OpenMP thread pools](#bound-the-openmp-thread-pools). | +| `memory` | `2g` | The cgroup limit, which counts every worker and the tmpfs. Size it from `concurrency × peak RSS` plus the tmpfs. Keep it above the cell's `memory`. On a disk-backed scratch, there's no tmpfs term. See [Scratch](scratch.md). | +| `memory-swap` | `2g` | Set it equal to `memory`. If you omit it, Docker allows twice `memory` in swap, and the memory limit no longer holds. | +| `tmpfs` size | `size=512m` | Scratch for all concurrent workers together. Size it from `concurrency` times the most scratch that one request holds. See [Constraints between the numbers](tuning.md#constraints-between-the-numbers). Moving scratch onto disk separates it from `memory`. See [Scratch](scratch.md). | +| `ulimit: stack` | leave it unset | See [Don't lower the stack limit](#dont-lower-the-stack-limit). | + +### Security flags + +Use these values. They're what a cell is for. If you omit one, the protection is gone, and the cell still +serves requests exactly as before. + +| Flag | Recommended | Description | +| --- | --- | --- | +| `network` | `none` | Removes every network interface. A tool that's persuaded to fetch a URL can't reach anything. Set it as an accessory key, not under `options:`. | +| `read-only` | `true` | Makes the root filesystem read-only. A cell writes only to `/tmp` and to the socket volume. | +| `tmpfs` flags | `nosuid,nodev,noexec` | `noexec` stops a dropped binary from running from `/tmp`, where a request's files live. It doesn't cover the socket volume, and it doesn't stop `ruby payload.rb`. On a disk-backed scratch, Docker sets none of these flags. See [Scratch](scratch.md). | +| `cap-drop` | `ALL` | Removes every Linux capability. | +| `security-opt` | `no-new-privileges:true` | Prevents a setuid binary from regaining what `cap-drop` removed. | +| `user` | `10001:10001` | Runs the cell with no home directory and no shell. Without user namespace remapping, this is a host uid in the ordinary range, so pick one that your hosts don't give to a person. | +| `pids-limit` | `512` | Bounds the number of processes in the cell. It must clear `concurrency` plus the threads and subprocesses that one toolchain starts, so check it when you raise `concurrency`. | + +### Don't lower the stack limit + +Leave `ulimit: stack` unset. A container inherits the Docker daemon's value, normally 8MB. + +Lowering it to 2MB buys a worker about 24MB more room, and nothing for an operation that shells out. It +costs far more than that: a thread that overflows the smaller stack dies on `SIGSEGV`, and the cell +reports that as `killed` with cause `crashed`, which is transient. The caller's job retries the request +against a limit that fails it again, for as long as the job keeps trying, and nothing in the verdict +points at the setting. Raise the cell's `memory` instead. + +The stack limit is a container flag rather than a `config.rb` setting because glibc reads it at exec, +before `Process.setrlimit` could run. `bin/conformance` and `bin/load` set no stack limit, and +`examples/gate` checks that they don't. + +## Bound the OpenMP thread pools + +Set `OMP_NUM_THREADS` and `OMP_THREAD_LIMIT` in the image. The installed `Dockerfile` sets both. On a large +host, neither is optional: without them, a cell dies. `hotcell:install` doesn't change an existing +`Dockerfile`, so for a cell installed before these variables existed, add both by hand and rebuild the +image. + +OpenMP sizes its thread pool from the host's core count. A `cpus:` limit is a CFS quota, not an affinity +mask, so it doesn't lower that count: on a 98-core host, libvips and ImageMagick ask for 98 threads. Each +thread stack is 8MB of private anonymous memory, which the cell's `memory` limit charges as +`RLIMIT_DATA`. Under a 1280MB limit, 96 of those stacks fit and 104 don't. Past that line, +`pthread_create` returns `EAGAIN`, which libgomp treats as unrecoverable. It writes the following line to +standard error and calls `exit(1)`: + +``` +libgomp: Thread creation failed: Resource temporarily unavailable +``` + +The worker dies before it answers, so the caller gets a transient failure, and the job retries against a +host that fails the same way. This killed 285 Basecamp workers in production on 2026-08-31 and forced a +rollback. + +That line now reaches a log. The supervisor captures what a worker writes to fd 2 and attaches its tail to +the `worker.killed` event, as `hotcell.stderr`, and to the failure that the caller receives. See +[What a worker wrote to fd 2](observability.md#what-a-worker-wrote-to-fd-2). + +Size `OMP_NUM_THREADS` from the container's `cpus`. The number follows the allocation, not the example. It's +a thread count, so round a fractional quota down, and round a value below 1 up to 1. `OMP_THREAD_LIMIT` is +the backstop against a library that raises the count itself by calling `omp_set_num_threads`, which is +what ImageMagick does for `MAGICK_THREAD_LIMIT`. + +The cell forwards both variables to the tools that it runs. A tool sees the environment that its +operation wrote for it, not the worker's own, which is [invariant 9](design/invariants.md). So the image's +variables alone would bound in-process libvips and nothing else. `Operation#run_tool` and mini_magick both +pass the pair from the cell's environment. + +A deploy to staging or beta can't catch a regression here, because the failure exists only at +production's core count. That's how it reached production. So the guard is a test: +`hotcell-client/test/install_test.rb` checks the installed `Dockerfile`'s variables, and an image that you +customize needs its own test. + +To verify a built image, write a GIF inside it and count `/proc/self/task` during the write. GIF is the +cheapest probe: it's the only common output format that quantizes, and quantization is where the threads +appear. Unbounded, the count tracks the visible cores. Bounded, it stays at the limit. + +## Verify an accessory + +`bin/conformance` supplies its own Docker flags, so it proves that the image works when the flags are +right. It doesn't read your Kamal configuration, and it passes against an image that you're about to +deploy with `cap-drop` missing. + +### Before you deploy + +Print the command that Kamal runs. `kamal config` doesn't show it: it prints the merged configuration, not +the command, so a `--network` emitted twice doesn't appear there at all. + +``` +bundle exec ruby -rkamal -e ' + config = Kamal::Configuration.create_from( + config_file: Pathname.new("config/deploy.yml"), destination: "production", version: "check") + puts Kamal::Commands::Accessory.new(config, name: :images).run.flatten.join(" ") +' +``` + +`--network` must appear once on that line, as `--network none`. Two of them is the `options:` mistake +described under [Container flags](#container-flags), and Docker rejects the container at boot with exit +status 125. + +### After you deploy + +Read the flags on the running container: + +``` +docker inspect --format '{{json .HostConfig}}' | jq '{ + NetworkMode, ReadonlyRootfs, CapDrop, SecurityOpt, PidsLimit, Memory, MemorySwap, Tmpfs, Binds +}' +``` diff --git a/docs/contributing/decisions.md b/docs/contributing/decisions.md new file mode 100644 index 0000000..40df57a --- /dev/null +++ b/docs/contributing/decisions.md @@ -0,0 +1,23 @@ +--- +type: Contributing +title: "Architecture decision records" +order: 3 +description: "The decisions that were argued rather than obvious, each with what it was decided against and what it costs." +sources: + - adr +--- + +# Architecture decision records + +Decisions that were argued rather than obvious, kept so the argument does not have to be rerun. Each record +states what was decided, what it was decided against, and what it costs. + +One file per decision, numbered in the order they were accepted, named `NNNN-what-was-decided.md`. A record +is written when the decision is made and is not edited afterwards. A later decision that changes an earlier +one gets its own record and marks the earlier one superseded. + +| # | Decision | Status | +| --- | --- | --- | +| [0001](../../adr/0001-reuse-workers-across-requests.md) | A worker may serve more than one request, and serves one by default | Accepted | +| [0002](../../adr/0002-keep-the-warm-slot-home.md) | Keep the warm slot home, as an accepted trade-off | Superseded by [0003](../../adr/0003-remove-the-persistent-slot-home.md) | +| [0003](../../adr/0003-remove-the-persistent-slot-home.md) | Remove the persistent slot home; a request's `$HOME` is made and removed with it | Accepted | diff --git a/docs/contributing/docs.md b/docs/contributing/docs.md new file mode 100644 index 0000000..de75f56 --- /dev/null +++ b/docs/contributing/docs.md @@ -0,0 +1,49 @@ +--- +type: Contributing +title: "Keep the docs current" +order: 2 +description: "The rake docs tasks, the order to run them in before a pull request, and the tests that hold tables to the code." +sources: + - rakelib/docs.rake + - hotcell-core/test/docs_test.rb + - hotcell-server/test/docs_test.rb + - activestorage-hotcell-server/test/docs_test.rb +--- + +# Keep the docs current + +The reference pages describe behavior, so a change to the code can make a page wrong. Treat the docs as +part of the change: + +1. Before you open a pull request, run `rake docs:stale`. It lists each page whose `sources` you changed + while the page didn't change. Read each page it names, and update what your change made wrong. A page + that's still right needs no edit. CI prints the same list as warnings on the pull request. +2. When a page starts or stops describing a file, add it to or remove it from the page's `sources`. A + source can be a file or a directory. +3. When you add a page or change a page's `title` or `description`, run `rake docs:index` to regenerate + the page lists in every `index.md`. +4. Run `rake docs:check`. It fails on a page without `type`, `title`, or `description`, on a source that + doesn't exist, and on an index that's out of date. CI runs it on every push, and so does `rake`. + +## Frontmatter + +Each page starts with +[Open Knowledge Format](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md) +(OKF) frontmatter: + +- `type`: `Reference` for a page that describes behavior, `Glossary` for the terms, `Design` for the + rationale, and `Contributing` for the pages about working on Hot Cell itself. +- `title` and `description`: the page's name and one-line summary, which the section's index lists. +- `order`: optional. An index lists the pages that have an `order` first, in that order, and the rest by + file name. +- `sources`: the code that the page describes. When that code changes, the page might need to change too. + +## Tables held to the code + +Some tables are held to the code by tests, which fail when the two disagree: + +| Table | Test | +| --- | --- | +| Codes and kill causes in `docs/codes.md` | `hotcell-core/test/docs_test.rb` | +| Events in `docs/observability.md`, and defaults in `docs/cell-settings.md` | `hotcell-server/test/docs_test.rb` | +| Shipped operation limits in `docs/active-storage.md` | `activestorage-hotcell-server/test/docs_test.rb` | diff --git a/docs/contributing/experiments.md b/docs/contributing/experiments.md new file mode 100644 index 0000000..ec459ae --- /dev/null +++ b/docs/contributing/experiments.md @@ -0,0 +1,86 @@ +--- +type: Contributing +title: "Established by experiment" +order: 4 +description: "The numbered facts that were measured rather than reasoned about. Code and docs cite them by number." +--- + +# Established by experiment + +Each of these was measured, not reasoned about. A specification cannot derive them, getting them wrong +produces failures that are hard to diagnose, and several constrain the architecture rather than the +implementation. + +1. **libvips cannot survive `fork` once it has evaluated an image.** After `require` and + `Vips.concurrency_set` the process has three threads and forks children that work; the first image + evaluation takes it to five, and from then on **every** forked child deadlocks in `futex_do_wait`, + permanently. Reproducible. This is the whole reason for the `before_fork` and `before_worker_boot` + split, and the reason `before_fork` may require and configure but must never evaluate. +2. **Two descriptors over `SCM_RIGHTS` work end to end**, with `Vips::Source.new_from_descriptor` and + `Vips::Target.new_to_descriptor`, and the kernel enforces the access modes: writing an input or reading + an output raises `Errno::EBADF`. +3. **Reopening `/proc/self/fd/N` defeats a read-only descriptor**, because it is a fresh `open` rechecked + against the inode and does not inherit the original flags. Never use it to turn a descriptor into a + filename. Copy instead. +4. **An empty Docker named volume takes its ownership from the image of whichever container mounts it**, + even if an earlier container already mounted it, provided it is still empty. So accessory and app boot + order does not matter. A bind mount instead takes the host directory's ownership, which is why local + development needs the directory created first. +5. **Kamal 2.11 hard-codes `--network kamal` for app roles.** Only accessories accept `network`, and only + as an accessory key. Under `options:` it is additive rather than overriding — Kamal emits its own + `--network` first — and Docker then refuses the container. Accessories can target `roles: [web, jobs]`, + and are not updated by a deploy. +6. **A worker killed by a resource limit produces a bare end of stream**, which is why the supervisor must + hold the connection and report `killed`. A `memory` breach does this too, roughly a third of the time: + libvips 8.18 dereferences null on its own out-of-memory path and takes `SIGSEGV` at + `vips_image_decode` *after* printing the correct diagnostic, and GLib's non-nullable `g_malloc` aborts. + So the reap-and-report path carries `memory` as well as `fsize`. +7. **A sibling process's `/proc//mem` is `EACCES` at `ptrace_scope = 1`, but its + `/proc//environ` is readable.** Verified with two same-UID siblings forked from one parent: the + environ read returned the victim's canary. Yama restricts `PTRACE_MODE_ATTACH`, which `mem` needs, and + does not restrict `PTRACE_MODE_READ`, which `environ` needs. +8. **A forked process cannot change what its own `/proc/self/environ` shows.** That view is the exec-time + environment, so `ENV.delete` in a worker is invisible to a reader. Only an `exec`ed child gets a fresh + one, which is why `unsetenv_others` on the tool spawn is the control. +9. **Docker cannot mount `/proc` with `hidepid`.** `--security-opt proc-opts=hidepid=2` is a Podman + feature; Docker rejects it outright, and remounting inside the container needs `CAP_SYS_ADMIN`. +10. **LibreOffice corrupts itself when two instances share a `$HOME` profile.** That is the origin of + slots. A hardened conversion measures at roughly 613ms, which sizes a soffice cell's deadline + concretely. +11. **`RLIMIT_AS` is unusable and `RLIMIT_DATA` is expensive.** For a real variant whose peak RSS is 45MB, + `RLIMIT_AS` must be at least 1536MB to succeed reliably and fails nondeterministically for a 400MB + band below that; `RLIMIT_DATA` works at 704MB. `RLIMIT_DATA` charges private writable anonymous + mappings and ignores `PROT_NONE` reservations, read-only private file mappings, and `MAP_SHARED` + entirely. It does charge thread stacks, so shrinking `RLIMIT_STACK` at container entry is worth real + headroom — and `RLIMIT_STACK` cannot be changed after `exec`, because glibc snapshots it at init. +12. **Ruby reserves about 450MB of `RLIMIT_DATA` at boot and never touches it.** A single ~404MB writable + anonymous region, introduced in 3.3 and unchanged in 3.4 and 4.0, unaffected by the GC and malloc + environment knobs. It is why the `memory` floor is what it is, and why `memory` cannot be read as how + much a bomb may consume. +13. **A cgroup memory kill is a prompt, silent `SIGKILL` with no diagnostic**, and the kernel chose the + allocating worker rather than the supervisor in every trial, because badness is RSS-proportional. The + argument for a per-worker limit is the diagnostic, not the choice of victim. +14. **Plain `require "vips"` leaves the libheif plugins un-`dlopen`ed.** They then load lazily inside the + worker, after its limits are on, where `dlopen` fails with only a warning and the process continues + with HEIC and AVIF missing — turning a limit breach into `unreadable` for a whole format family. + `require "image_processing/vips"` or `Vips.block_untrusted true` maps them in the supervisor instead. +15. **`fork` costs about 2.8ms**, measured as `fork` + `exit!` + `wait` from a 58MB three-thread parent + with libvips required and never evaluated. It is a small part of the cell's fixed overhead, not most + of it. +16. **The cell's fixed overhead is copy-on-write settling, and it is proportional to the supervisor's + resident heap.** A worker's first pipeline run takes about 7,900 minor faults against 1,564 for a warm + process. Forking the same work from a 265MB parent instead of a 58MB one took faults to about 25,900 + and added roughly 52ms per request. So preloading generously in the supervisor makes every request + slower for the life of the deployment. `RLIMIT_AS` and libvips thread-pool size were both tested and + neither moves it. +17. **macOS has no finite `RLIMIT_DATA`.** `Process.setrlimit` rejects one with `EINVAL`, and the + inherited hard limit is already infinity, so it is not a privilege problem. A cell there runs with its + memory clamp unenforced and warns once. Every other limit is strict on both platforms. +18. **Re-opening `/dev/fd/N` is checked against the opening process's credentials and the file's mode.** + Not against the caller's. So a cell handed a descriptor for a mode `0600` file the application owns + cannot open it by name at all, and every operation that gives a tool a filename dies as `EACCES`. + Operations that read the descriptor directly are unaffected, which is why the failure looks selective. + The mode is also what enforces invariant 4 once a tool holds a filename: at `0400` even the owner is + refused `O_RDWR`, and at `0200` even the owner is refused a read. That holds only while the cell does + not own the file. Changing a mode needs ownership, and `cap-drop ALL` leaves no capability that + overrides it — so a shared group enforces the invariant and a shared uid cannot. diff --git a/docs/contributing/guide.md b/docs/contributing/guide.md new file mode 100644 index 0000000..ae3eee4 --- /dev/null +++ b/docs/contributing/guide.md @@ -0,0 +1,305 @@ +--- +type: Contributing +title: "Working on the gems" +order: 1 +description: "Setting up, running the tests, the repository layout, how a cell gets exercised, style, the changelog, CI, and making a release." +sources: + - Rakefile + - bin + - .github/workflows +--- + +# Working on the gems + +This is a guide to working on the gems themselves. If you are building *on* Hot Cell — writing operations for +your own application — start with [README.md](../../README.md) instead. + +## Setting up + +``` +bundle install +``` + +That is enough for the hotcell gems and for the example cell. It is deliberately enough: **the +hotcell gems need no container and no tool installed**, and `rake test:hotcell` is what keeps that honest — +CI runs it on a machine with nothing on it. Fixture operations stand in for the work, so the protocol, the +fork, the descriptor passing, the limits and the reap are all exercised in milliseconds. + +Two things need more, and only when you touch them: + +| To run | You need | +| --- | --- | +| `rake test:activestorage` | libvips, mutool, ffmpeg, ffprobe, ImageMagick, pdftoppm | +| `bin/conformance`, `bin/load` | Docker, and Linux | + +## Running the tests + +``` +rake # everything, then rubocop +rake test:hotcell # everything that needs no tool installed +rake test:activestorage # everything that needs the converters installed +rake test:devcell # boot the real development cell and run the example battery +rake test:gem:hotcell-core # one gem's suite +rake rubocop # style, one configuration for every gem +``` + +One file, or one test, from inside a gem directory: + +``` +cd hotcell-server +rake test TEST=test/slot_test.rb +rake test TEST=test/slot_test.rb TESTOPTS="-n/reuse/" +``` + +The container checks are scripts rather than rake tasks, because they need Docker and a built image: + +``` +bin/example-image # install the cell scaffold and build what it wrote +bin/conformance IMAGE # does this image support hotcell? +bin/load IMAGE [SCENARIO] [SECONDS] [THREADS] +``` + +## The layout + +``` +hotcell-core/ the wire protocol, descriptors, failures — both sides depend on it +hotcell-client/ the application side, and the installer that scaffolds a cell +hotcell-server/ the supervisor, workers, slots, limits, and exe/hotcell +activestorage-hotcell-server/ the media operations +activestorage-hotcell-client/ the Rails transformer, analyzers and previewers +yabeda-hotcell/ Yabeda metrics for the application side + +examples/ one cell's worth of sample operations and the battery that drives them +bin/ the container checks: example-image, conformance, load +docs/ the reference manual, one topic per page, and design/ for the rationale +rakelib/ rake docs:index, docs:check and docs:stale +adr/ decisions that were argued rather than obvious +``` + +## How a cell gets exercised + +There are two sets of operations in this repository, and reaching for the wrong one is the easiest mistake +to make here. + +**Fixture operations** — `HotCell::Fixtures` in `hotcell-server/lib/hot_cell/test_operations.rb`, named +`test.*`. These are for the minitest suites: forty-odd operations covering every edge the protocol has, from +`test.mojibake` to `test.early_idle`. They ship in the gem rather than sitting in its own test directory, +because `hotcell-client`'s suite boots a real cell and needs an inventory to point it at. If you are writing +a test for a gem, this is what you want. + +**Example operations** — `examples/operations`, named `example.*`. These are for exercising a *whole cell* +from the outside, where the suite does not own the process: a development cell booted from `exe/hotcell`, or +somebody's container image. They double as worked examples of how to write an operation, so they stay few, +readable and pure Ruby with no external toolchain — which is also what lets them run on macOS, where the +converters and Docker are absent. + +`echo` is also meant to be copied into a real cell and kept there. It is the only cheap way to prove the +work socket answers, because `describe` and `metrics` cross the control socket instead. See +[Rails healthcheck](../observability.md#rails-healthcheck). + +| op | behavior | what it proves | +| --- | --- | --- | +| `echo` | read the input fd, write it back to the output fd | descriptor passing round-trips; a throughput baseline | +| `sleep` | block for `seconds` | the deadline kills and answers; head-of-line behavior under load | +| `greedy` | allocate `megabytes` | the `memory` clamp → `killed: memory`, where enforceable | +| `overflow` | write `megabytes` to the output | the `file_size` clamp → `killed: fsize` | +| `crash` | raise, or die by signal | worker death → a clean verdict and continued service | +| `spawn` | start a grandchild that would outlive the worker | the group kill leaves nothing behind | +| `probe` | whether a pid is alive, asked from inside the cell | how `spawn` is watched, in any pid namespace | +| `isolation` | interfaces, root writability, scratch `noexec`, a tool's environment | the container-only checks, from the only place that can make them | + +`examples/lib` holds the thin `HotCell::Client` classes that call them and the **battery** — one list of +checks built from those clients, so every consumer asks a cell the same questions. Consumers drive through +the client classes rather than a raw `Connection`, because that is what an application does, and the wrapper +costs little enough that load numbers still measure the cell. + +Three consumers run that battery. + +### `rake test:devcell` — does the development configuration work? + +Boots the real `exe/hotcell` as a plain host process on the example operations, exactly the way you run it +yourself, runs the battery, then sends `SIGTERM` and asserts exit 0 with no socket left behind. It is part +of `rake test:hotcell`, so it runs with the rest of the suite, on Linux and macOS both. + +It is Docker-free on purpose. See "Cells run uncontainerized in development" below. + +### `bin/conformance IMAGE` — does this image support hotcell? + +The real use case is somebody building their own cell image and wanting to know whether it works. It boots +`IMAGE` with the accessory's real flags — `network: none`, `cap-drop ALL`, a read-only root, a `noexec` +tmpfs — with `HOTCELL_DIR` pointed somewhere non-default, and asserts that the server starts, that the socket +appears where `HOTCELL_DIR` said it would, that the battery passes, and that the isolation holds. It exits +non-zero on the first failure, so it gates. + +The driver runs in a **second container** over a shared volume, so a descriptor crossing the container +boundary is part of every check. That container is a stock Ruby image rather than the image under test, so a +minimal cell image never has to carry the client's dependencies. + +The example operations are mounted over `/hotcell/operations`, shadowing whatever the image baked in. That is +deliberate: what is under test is the image's runtime — its Ruby, its gems, its user, and the flags — rather +than the work it does. + +CI runs it against what `bin/example-image` builds. That script runs the installer and builds exactly what it +wrote, so the scaffold an application is actually given is the thing CI proves. Nothing derives from a +published base image, so the installed scaffold is the only kind of cell image there is. + +### `bin/load IMAGE` — how does it behave under pressure? + +The same operations at volume against a containerized cell. It reports throughput, latency split into +`queued_ms` against `perform_ms`, and the verdict breakdown, which is what tells saturation apart from +slowness. Scenarios: `echo` for a baseline, `sleep` for queueing, `greedy` and `overflow` for the resource +kills, `crash` for worker death, `spawn` for orphans, and `mix` for a weighted blend. + +The cell's knobs pass through the environment, so finding a knee needs no edit: + +``` +EXAMPLE_CONCURRENCY=4 EXAMPLE_QUEUE_SIZE=16 bin/load hotcell:example echo 30 16 +``` + +Heavy runs are manual. A short bounded run works as a CI gate if one is ever wanted. + +## Rules that are not obvious + +**Nothing loads libvips into a test process.** libvips creates its thread pool the first time it processes an +image, and that pool does not survive `fork`: a child forked afterwards waits forever for a worker thread +that does not exist. The Active Storage suites boot real cells by forking, so the operations load inside the +cell, the fixtures are generated by CLI tools, and `Cell.boot` refuses to fork a process that has libvips +loaded. `fork_safety_test.rb` holds both halves of that, because a test that only showed the good case would +not establish that the hazard is real. + +The same rule is what `before_fork` and `before_worker_boot` exist for. A hook that evaluates an image in the +supervisor is a silent hang on every request afterwards, not a crash — see the comments on +`HotCell::Operation.before_fork`. + +**Cells run uncontainerized in development**, on every platform, so there is one thing to document and one +thing to debug. The reason is macOS and it cannot be engineered around: Docker Desktop runs containers in a +Linux VM, a file descriptor is an index into one kernel's table, and `sendmsg` has nothing meaningful to hand +across two kernels. Native `AF_UNIX` and `SCM_RIGHTS` on macOS are fine; it is the host-to-VM boundary that +cannot carry a descriptor. That is why `rake test:devcell` exists and why it uses no Docker, and what it +leaves out — the isolation — is what `bin/conformance` covers on Linux. + +**macOS has no finite `RLIMIT_DATA`.** `Process.setrlimit` rejects one with `EINVAL`, so a cell there runs +with its memory clamp unenforced and says so once, on `$stderr`. Tests that assert the clamp skip themselves +where it cannot be set; a new one that asserts memory behaviour has to do the same. Every other limit is +strict on both platforms. + +**Testing a control.** Security controls here fail silently, so a test that would still pass with +the control removed is worse than no test: it reads as assurance. Every control is covered by a test +that observes the behavior the control produces, rather than by an assertion that the control is +written down — `unsetenv_others: true` is proved by setting a variable, running a tool, and finding +that the tool never saw it. Where a control has no reachable trigger and so cannot be tested, it +says so where it lives. + +**A verdict that cannot be taken back needs a reason.** `permanent` means an application may write a failure +down against a blob and serve it from a cache forever, so anything unclassified is transient by default. +`hotcell-core/lib/hot_cell/codes.rb` carries the argument for each code; a new code goes in that table, and a +new kill reason goes in `PERMANENT_BY_CAUSE`, or it silently becomes the wrong kind of permanent. + +## Style + +``` +rake rubocop +``` + +One configuration for every gem, because the style is one style. It runs in CI as its own job. + +## Writing it down + +[docs/design/](../design/index.md) holds the threat model and the numbered invariants, and +[Established by experiment](experiments.md) holds the facts that were measured rather than reasoned +about — the fork hazard, what `RLIMIT_DATA` charges, what `/proc` gives a sibling away. Read them before +changing anything a limit or an isolation claim rests on. Neither describes behavior. + +The rest of [docs/](../index.md) does describe behavior, and a change to the code can make it wrong. +[Keep the docs current](docs.md) says how: `rake docs:stale` names the pages +whose sources a branch changed, and `rake docs:check` holds the frontmatter and the indexes. + +The invariants are numbered, and code comments across three gems cite them by number. Do not renumber them. + +[The decision records](decisions.md) hold the decisions that were argued rather than obvious, so the argument does not have +to be rerun. Write one when a decision cost you a real debate; do not edit one afterwards — a later decision +that changes it gets its own record. + +Comments in this codebase carry the reasoning, not the mechanics. A comment that restates the line below it +is noise; a comment that says why the obvious version does not work is the whole value. Match what is +already there. + +### Accepted risk + +A comment opening with **`Accepted risk.`** marks a decision rather than a claim. Somebody weighed that +exposure with the risk in view and chose to carry it, and the comment states the premise the choice rests +on. Grep for the phrase to find them all. + +Two kinds of comment sit near risky code and they are read differently. A **safety claim** says how +something works — "`cap-drop ALL` leaves no capability that overrides a mode". It can be wrong, and a +review should try to falsify it. An **accepted risk** is a preference, and restating the exposure it names +is not a finding. + +**So an accepted risk may only be re-opened by attacking its premise, never by restating its risk.** If the +premise still holds, there is nothing to report, whatever the exposure looks like. If you can show the +premise is false, report the premise: that is what changed, and it is what makes the decision worth taking +again. [ADR 0003](../../adr/0003-remove-the-persistent-slot-home.md) is the shape of a good one — the warm slot +home was an accepted risk whose premise, "the exposure is a write rather than a read", turned out to be +false, because what a tool reads from `$HOME` is configuration and configuration is executable. + +This applies to security reviews in particular, including ones run by agents. Reporting a priced risk back +as a new finding costs a maintainer the same argument every time. + +### The changelog + +Every pull request that changes behavior should update `CHANGELOG.md` in the same pull request. One +reader is a busy developer who is scanning the changelog to determine if they care. Another reader +is an agent trying to determine if the application needs any changes to accommodate the new version. + +Add to the `next / unreleased` section, under the gem the change is in — or under `Tooling` for the +checks and scripts in `bin/` and `examples/`, which ship in no gem — and under `Breaking`, +`Security`, `Added`, `Fixed`, `Improved` or `Changed`. `Improved` is for a change that makes existing +behavior better rather than fixing a defect — a performance win, or a limit that no longer applies. +Keep it short, one sentence is preferred that explains what changed. Lead with why the change matters to +the reader: what is now faster, safer, simpler or possible. Describe the change as it ships. Leave out +known defects and planned follow-ups; each gets its own entry when it is fixed, so no entry goes stale or +depends on another. If it's a breaking change, it can run longer to explain how to tell if an +application will be impacted, and how to update the application in that case. + +Skip the entry only for a change no consumer can observe: a test, a refactor, a comment, a CI tweak. If you +are unsure, write one. + +This applies to agents. If you opened the pull request, the entry is part of it. + +When a change gives an application something to do — a field to log, a variable to set, a value to +recompute — add it to an `Upgrading` section at the top of the release. Each item is two sentences. The +first is the action, in the imperative. The second is the rationale, or a link to the doc that holds it. + +## CI + +| Job | Runs | +| --- | --- | +| `hotcell (ruby …)` | `rake test:hotcell` on 3.3, 3.4, 4.0 and head, on a machine with no tools | +| `hotcell (macos, ruby 3.4)` | the same, on macOS — advisory, because the runners are slow and bill about ten times the Linux rate | +| `activestorage (ruby …)` | `rake test:activestorage` with the converters installed | +| `container conformance` | `bin/example-image` then `bin/conformance` | +| `Style` | `rubocop` | +| `GitHub Actions audit` | actionlint and zizmor | + +Actions are pinned to SHAs and the workflow has no default permissions. `zizmor` will tell you if a change +breaks either. + +## Making a release + +The gems release together on one version, and `VERSION` at the repository root is what sets it. + +- Prechecks + - [ ] make sure CI is green + - [ ] `bundle exec rake` — the full suite and rubocop + - [ ] update `CHANGELOG.md`: retitle `next / unreleased` with the version and the date + - [ ] `bundle exec rake version:bump[1.2.3]` — writes `VERSION` and every gem's version constant + - [ ] commit, and tag as `v1.2.3` +- Release + - [ ] `bundle exec rake gems` — builds every gem into `pkg/`, which it empties first + - [ ] `git push && git push --tags` — **before** the gems: every gemspec's `changelog_uri` and + `source_code_uri` name the tag, and they 404 until it is on GitHub + - [ ] `for f in pkg/*.gem ; do gem push $f ; done` + - [ ] create a release at https://github.com/basecamp/hotcell/releases +- Post-release + - [ ] `bundle exec rake version:bump[1.3.0.dev]`, and open a new `next / unreleased` section in + `CHANGELOG.md`, so that `master` never reports a version that was released diff --git a/docs/contributing/index.md b/docs/contributing/index.md new file mode 100644 index 0000000..744862f --- /dev/null +++ b/docs/contributing/index.md @@ -0,0 +1,21 @@ +--- +title: "Contributing to Hot Cell" +description: "Working on the gems themselves: setup, tests, releases, keeping the docs current, the experiments and measurements the design rests on, and the decision records." +--- + +# Contributing to Hot Cell + +These pages are for people working on Hot Cell itself. If you're building an application on Hot Cell, start +with the [README](../../README.md) and the [reference manual](../index.md). + + + +| Page | Description | +| --- | --- | +| [Working on the gems](guide.md) | Setting up, running the tests, the repository layout, how a cell gets exercised, style, the changelog, CI, and making a release. | +| [Keep the docs current](docs.md) | The rake docs tasks, the order to run them in before a pull request, and the tests that hold tables to the code. | +| [Architecture decision records](decisions.md) | The decisions that were argued rather than obvious, each with what it was decided against and what it costs. | +| [Established by experiment](experiments.md) | The numbered facts that were measured rather than reasoned about. Code and docs cite them by number. | +| [Cell overhead](overhead.md) | The fixed time that a cell adds to each call, what causes it, and the causes that tests ruled out. | + + diff --git a/docs/contributing/overhead.md b/docs/contributing/overhead.md new file mode 100644 index 0000000..895885d --- /dev/null +++ b/docs/contributing/overhead.md @@ -0,0 +1,55 @@ +--- +type: Contributing +title: "Cell overhead" +order: 5 +description: "The fixed time that a cell adds to each call, what causes it, and the causes that tests ruled out." +--- + +# Cell overhead + +Read this page for its negative results rather than its numbers. The numbers came off a laptop from a +prototype; the things that were ruled out are properties of the design. For the reuse trade specifically, +[ADR 0001](../../adr/0001-reuse-workers-across-requests.md) supersedes this with measurements taken in the +deployed artifact. + +Three arms, each producing a 256×256 PNG with `resize_to_fill`: in process, a native cell over a Unix +socket, and a containerized cell with the hardening flags on. The cell arms ran 2.3×, 1.9×, and 1.7× +slower as the source grew from a 12KB JPEG to a 328KB one. + +**The overhead is roughly fixed at 25 to 35 ms**, which is why the multiplier falls as the work grows. +Expect the ratio to look worst on the smallest thumbnails and to stop mattering on anything expensive. + +Four candidates were tested and eliminated, recorded so nobody re-runs them: + +| Candidate | Verdict | +| --- | --- | +| The container | Within noise of the native cell on two of three sources, 12ms on the largest. | +| The `fork` syscall | 2.8 ms of a 20–30 ms cost. | +| `RLIMIT_AS`, which the worker sets and the in-process arm does not | No effect. 8 GB, 2 GB, and unset are indistinguishable. | +| libvips thread pool startup per worker | Not it. The first-versus-second-call gap is *largest* at concurrency 1. | +| The libvips operation cache, as a confound favouring the in-process arm | Not it. `Vips.cache_set_max 0` moved that column by less than the error bars. | + +**The overhead is copy-on-write, measured rather than inferred.** The identical pipeline runs about 14 ms +slower in a freshly forked worker than in a long-lived process. Run it *twice* in the same child and the +second call matches a warm process, so it is one cost per process, not per call. Minor fault counts say +what it is: **7,920 faults on a child's first call against 1,564 warm**, about 25 MB of pages copied at +roughly 1.7 µs each. + +**So the cheapest optimisation is the counter-intuitive one: keep the supervisor small, and treat +`before_fork` as having a per-request price.** The instinct is to preload generously so workers start +fast. It is backwards here — every megabyte the supervisor holds is partly copied by every worker for the +rest of the deployment. `before_fork` must still `require`, because of the fork hazard in [experiment 1](experiments.md), so +the resolution is to require only what that cell's own operations need. That is a third argument for one +cell per toolchain, and a measured reason for `hotcell-server` not to depend on `activesupport`. + +Do not spend effort making the container cheaper. There is nothing there to win. + +**Synthetic pre-warming is not a free alternative to worker reuse.** Tested, because it would have been +the ideal answer: running a synthetic 64×64 pipeline in the worker before the real transform moved the +real transform's faults from 7,885 only to 6,184, against a warm floor of about 3,300 — roughly a quarter +of the excess. Most of the cost is proportional to the real image's own work, not to shared code paths a +synthetic image would touch. + +Budget the fixed cost against inline processing. A variant generated during an upload request, as Active +Storage's `process: :immediately` does, adds this overhead to that request. Whether that is acceptable is +a product decision, and it is the main reason to measure it early on real hardware. diff --git a/docs/design/descriptors.md b/docs/design/descriptors.md new file mode 100644 index 0000000..f10fcc1 --- /dev/null +++ b/docs/design/descriptors.md @@ -0,0 +1,55 @@ +--- +type: Design +title: "Passing file descriptors" +order: 4 +description: "The argument for passing file descriptors instead of sharing a directory, and what it costs." +--- + +# Passing file descriptors + +The obvious alternative is a directory mounted into both containers: the app writes an input file and +names it, the cell writes an output file and names it. It is simpler, and it has one real advantage this +design gives up. + +**It removes a class of bug rather than a bug, and that is the deciding reason.** With a volume the app +must open a path the cell can write to. The cell creates that path as a symlink to +`/rails/config/master.key`, which resolves in the *app's* mount namespace; the app reads it and publishes +it as a public image variant. `O_NOFOLLOW` closes that, but only for the final component, so an +intermediate directory symlink still works, and closing that needs `openat2` with `RESOLVE_NO_SYMLINKS`, +which Ruby does not expose. Add a sticky exchange directory so the cell cannot unlink the app's files, +have the app pre-create both files, validate the descriptor with `fstat` for a regular file and an exact +size, and never list the directory. That is a checklist that must each be remembered forever, each item +failing silently. Descriptors have no such checklist, because there is no path. + +The checklist is not hypothetical. A volume-based service arrives at it item by item: a token validated +against `\A[a-f0-9]{32}\z`, a path built server-side from one component, +`O_RDONLY|O_NOFOLLOW|O_NONBLOCK`, an `fstat` on the descriptor for a regular file and an exact size under +a maximum, and then **the open descriptor rather than the path handed to the tool.** Which is to say it +arrives at descriptor passing anyway, having paid for the volume first. + +**Cross-request isolation is a smaller reason than it first appears.** Under a shared volume every worker +can read every request's bytes in the directory, including requests not currently running. Under +descriptors only in-flight requests have bytes inside the cell. That narrower window is the real gain — +and it is a window, not a wall. See [Worker isolation](worker-isolation.md). + +**Nothing is left at rest.** A volume is persistent and outlives requests, so it holds user content +somewhere neither side owns, and a crashed worker leaves it there. The socket directory holds a socket and +no data. + +**Descriptors protect naming, and only naming.** The bytes are copied to the cell's tmpfs when +an operation asks for a path, so the cell has full read and write on its own copy. The output is a file +the app created and the cell writes arbitrary bytes into it either way, so a compromised cell can return a +malicious image and the app will publish it. Validating content is the application's job. + +**The cost, accepted knowingly.** A bind mount crosses Docker Desktop's VM boundary and a descriptor does +not, so a volume-based design would let macOS developers run the real container with the real hardening. +Descriptors cost us that, which is why development runs a cell uncontainerized on every platform. Removing +a whole class of naming bug is worth more than development parity, because that class fails silently and +each mitigation for it has to be remembered forever. + +**Where a volume wins, and it is not a corner case.** An input larger than the cell's tmpfs cannot be +posted in at all. Video shows it plainly: a 3GB blob will not spool onto a 512MB tmpfs at any body limit, +and `ffprobe` reads only a container header, so store-and-forward costs roughly 300× I/O amplification on +its commonest call. The descriptor itself already refers to a file on the application's own filesystem; +the amplification comes from the **copy** an input performs when asked for its path. So the fix is per-operation and stays inside this design: an operation +that can consume a descriptor directly never asks, and never pays. diff --git a/docs/design/index.md b/docs/design/index.md new file mode 100644 index 0000000..cfb0785 --- /dev/null +++ b/docs/design/index.md @@ -0,0 +1,21 @@ +--- +title: "Design" +description: "The intent behind Hot Cell's design: the threat model, the invariants, worker isolation, and passing file descriptors." +--- + +# Design + +These pages describe the intent behind Hot Cell's design and implementation: why the work moves out of your +application's process, what a cell defends against, and the properties that the whole design exists to +hold. For how to use the gems, see the [reference manual](../index.md). + + + +| Page | Description | +| --- | --- | +| [Threat model](threat-model.md) | What Hot Cell is, the problem it solves, and what a cell is defended against. | +| [Invariants](invariants.md) | The numbered properties the design exists to hold. Code and tests cite them by number. | +| [Worker isolation](worker-isolation.md) | What one worker can and cannot reach of another, and the residuals that remain. | +| [Passing file descriptors](descriptors.md) | The argument for passing file descriptors instead of sharing a directory, and what it costs. | + + diff --git a/docs/design/invariants.md b/docs/design/invariants.md new file mode 100644 index 0000000..a2a70a3 --- /dev/null +++ b/docs/design/invariants.md @@ -0,0 +1,28 @@ +--- +type: Design +title: "Invariants" +order: 2 +description: "The numbered properties the design exists to hold. Code and tests cite them by number." +--- + +# Invariants + +These are the properties the design exists to hold. Code and tests across three gems cite them by number, +so the numbers never change. Number 1 is retired. + +2. A cell holds no application credentials, in its environment or on its filesystem. +3. The supervisor never evaluates image data, and a worker forked from it produces correct output. +4. Descriptor access modes are one-way: an input can't be written, and an output can't be read. +5. The cold side never opens a filesystem path that a hot side chose. +6. An operation can't exceed its cell's limits, whatever it declares. +7. A cell can't reach another cell's socket. +8. A worker can't read another request's memory, given `kernel.yama.ptrace_scope >= 1` on the host and + `max_requests_per_worker: 1`. [Worker isolation](worker-isolation.md) covers files and the environment. +9. A tool subprocess sees only the environment that its operation wrote for it. + +Tests cover the invariants where reading the code isn't enough: where the kernel enforces the property, +where a break would be silent, or where a plausible refactor would remove it. + +Invariant 2 is a property of a whole filesystem, so no single test asserts it. The tests check that the +built image contains no application source tree, and `bin/conformance` boots the hardened container and +checks its flags. diff --git a/docs/design/threat-model.md b/docs/design/threat-model.md new file mode 100644 index 0000000..92de6bc --- /dev/null +++ b/docs/design/threat-model.md @@ -0,0 +1,63 @@ +--- +type: Design +title: "Threat model" +order: 1 +description: "What Hot Cell is, the problem it solves, and what a cell is defended against." +--- + +# Threat model + +This page states what Hot Cell is, the problem it exists to solve, and what a cell is defended against. + +## What this is + +Hot Cell moves untrusted work out of a privileged application process into an unprivileged, +resource-capped, network-less sibling container. The application reaches in through a narrow interface. +The work never reaches out. + +A hot cell is a shielded chamber for handling highly radioactive material. The operator stays outside and +manipulates the contents remotely, and nothing leaves except by controlled transfer. The vocabulary +follows: **hot** is untrusted, **cold** is trusted, and material is **posted in** and **posted out**. +Everything else uses the ordinary word. + +## The problem + +Firejail and similar sandboxes do not work inside a container, because containers and sandboxing use the +same kernel features. An application deployed in a container therefore runs media libraries unconfined. A +vulnerability in libvips, `soffice`, `mutool`, `ffmpeg`, or ImageMagick, reached through an uploaded file, +executes with the application's secrets, its database credentials, its code, and its network. + +Two properties of `/proc` shape the response. Scrubbing a tool's environment does not help, because a +file-read primitive can retarget `/proc//environ` at a same-UID process and read the environment +that the scrubbed invocation was supposed to hide. And moving secrets out of the environment does not help +either, because the same primitive reads arbitrary files, including a credentials file such as +`config/master.key`. + +That leaves two remedies: a bubblewrap or nsjail wrapper inside the application image, or a separate +conversion service holding no secrets. Hot Cell is the second. The second also avoids an open question the +first carries, which is whether an in-image wrapper survives the container runtime it is deployed under. +That is the verification firejail does not pass. + +## Threat model + +Assume arbitrary code execution inside a cell, or merely an arbitrary file read, reached through a +malicious input file. The design must make that outcome cheap. + +In scope: reading application code or secrets, reaching the database, reaching the network, reading +another request's memory or environment, consuming the host's CPU or disk, escalating through a path the +application later opens, and reaching a different cell's toolchain. + +Out of scope: a kernel container escape, a malicious operation implementation, and denial of service +against a cell. + +Out of scope means the design does not promise to stop it, not that it is harmless. A malicious operation +— most plausibly a supply-chain compromise in a gem an operation depends on — is in fact contained by +everything here, because an operation runs in the same cell under the same limits as the library it +calls. It is excluded because it is not what this design is *for*, and because treating it as in scope +would drag operation review and dependency policy into a document about a transport. + +The minimum requirement is a PID and mount namespace exposing only the input and output, with no `/proc` +and no application filesystem. Hot Cell meets the namespace requirement by being a separate container, and +**exceeds the input/output part**: descriptors mean there is no path in the cell to expose or to traverse, +rather than a narrowed set of bind mounts. The `/proc` requirement is treated separately in +[Worker isolation](worker-isolation.md), because it is the one part a container boundary does not give us for free. diff --git a/docs/design/worker-isolation.md b/docs/design/worker-isolation.md new file mode 100644 index 0000000..23102ea --- /dev/null +++ b/docs/design/worker-isolation.md @@ -0,0 +1,105 @@ +--- +type: Design +title: "Worker isolation" +order: 3 +description: "What one worker can and cannot reach of another, and the residuals that remain." +--- + +# Worker isolation + +Workers in a cell are siblings under one UID in one PID namespace, so another worker's `/proc` entries are +same-UID reads. Measured, not assumed — items 7 and 8 of [Established by experiment](../contributing/experiments.md): + +| Target | At `ptrace_scope = 1` | Why | +| --- | --- | --- | +| `/proc//mem` | `EACCES` | Needs `PTRACE_MODE_ATTACH`, which Yama restricts to descendants. | +| `/proc//environ` | **Readable** | Needs only `PTRACE_MODE_READ`, which Yama does not restrict. | +| `/proc//fd/N` | **Readable** | Same `PTRACE_MODE_READ`. So are the scratch files themselves, by directory listing. | + +So invariant 8 has three parts and they have different answers. Only the first is about `ptrace_scope`. + +**Request memory** is protected by `kernel.yama.ptrace_scope >= 1`. That is a host sysctl a container +cannot set, so it is a deployment precondition. At `ptrace_scope = 0` the guarantee is gone, and the cell +therefore **refuses to boot** rather than logging a warning and serving anyway. A host sysctl is invisible +to the image, it silently voids the guarantee, and a warning in a log is how a dead control stays dead. + +**Above `max_requests_per_worker: 1` this invariant is about workers, not requests.** A worker serving +several requests in turn holds each of them in the same address space, so an input that achieves code +execution can read and tamper with every later request that worker handles — no race to win, and covering +requests that were never concurrent with it. That is a deliberate setting with a measured payoff; see +[ADR 0001](../../adr/0001-reuse-workers-across-requests.md). It is the only place in this design where the +isolation between two requests is a configuration value. + +**Nothing on disk carries from one request to the next.** A slot holds one directory per request, which is +that request's `$HOME` and also where its inputs and outputs are staged, and it is created when the request +starts and removed before the caller hears the answer. A home that outlived its request would carry +configuration to the next one, and for these toolchains configuration is executable: ImageMagick runs the +command lines in `delegates.xml` and applies the rights in `policy.xml`, both read from +`$HOME/.config/ImageMagick`. One input that achieved code execution could then reconfigure every later +request on that slot, which is the bound `max_requests_per_worker: 1` exists to hold. +[ADR 0003](../../adr/0003-remove-the-persistent-slot-home.md) records the decision. + +The directory carries a fresh unpredictable name for every request, and that is what makes the removal a +guarantee rather than an intention. A tool that reaches code execution runs as the user that owns the tree, +so it can `chmod 0500` its own configuration directory and the slot directory around it, and both the +worker's delete and the supervisor's rename then fail. A fresh name means the next request never receives +a tree that an earlier request prepared. A mode is also not a permission the process lost, so a cleanup +that fails on one is retried after putting the mode back, and what a cleanup that still fails costs is disk +rather than isolation. + +**That bounds what a finished request left behind, and not what a live process is doing.** Every worker runs +as the same uid, and `0700` is that uid's own mode, so a concurrent sibling can write into a home as soon as +it exists — and so can a `setsid` descendant of a request that has already answered, which process groups do +not contain. The slot directory itself is a name a worker can rename aside and replace, and a pathname +`chmod` follows what it finds. Those are the residuals below, and a fresh name does not close them: it +closes the offline route, where nothing of the attacker's is still running. + +**Files are not isolated between concurrent workers, and cannot be.** Every worker runs as the same uid in +one mount namespace, so a worker that reads another worker's scratch directory — by listing it, or through +`/proc//fd/N` — gets that request's input and output bytes. Unlinking the scratch file does not +close it, because the descriptor is still reachable through the sibling's `/proc`. The two fixes that +would work need `CAP_SETUID` for a per-worker uid or `CAP_SYS_ADMIN` for a per-request mount namespace, +and `cap-drop ALL` removes both. This is the same shape as the environment: a real residual, stated rather +than papered over. + +What bounds it is the size of the window and the value of the contents. Only requests actually in flight +have bytes inside a cell, a cell holds no credentials, and a cell carries one toolchain. So the exposure +is "the other conversions happening right now in this cell", which is why per-toolchain cells and a sober +`concurrency` are containment decisions and not just scheduling ones. + +**A compromised worker can steal the cell's sockets.** It unlinks `work.sock` and binds its own, and every +later request arrives at its listener with the caller's descriptors already attached. It may then read +those inputs, write those outputs, and answer `ok`. This works because the socket directory has to be +writable by the user the supervisor runs as, and workers run as that user. + +The worker that did it exits as designed and its listener does not, because a child it forked calls +`setsid` and leaves the process group the reap sweep kills. So the reach is every request the cell +serves from then on, rather than only the ones in flight: the bound above covers one worker reading +another's files, and it does not cover this. + +**Nothing stops this today.** Every prevention available needs something the deployment does not have. A +tighter directory mode is undone by the owner, and the sticky bit grants the owner what it withholds from +others. The immutable flag needs `CAP_LINUX_IMMUTABLE` and a uid per worker needs `CAP_SETUID`, and +`cap-drop ALL` removes both. An abstract-namespace socket has no name to unlink and cannot be reached +across `network: none`. + +[Landlock](https://github.com/basecamp/hotcell/issues/13) is the one prevention that fits an unprivileged +container: a worker gives up write access to the socket directory right after the fork, irreversibly and +with no capability. Until then this is a known gap, and the containment is the same as for a compromise +generally — a cell holds no credentials, carries one toolchain, and is replaced rather than repaired. + +Fork-per-request isolates each request's **memory**. The argument in +[Passing file descriptors](descriptors.md) is about the boundary between the application +and the cell, and it doesn't extend to workers inside one cell. + +**Environment** is not protected by `ptrace_scope` at all, and cannot be fixed inside the worker. A forked +process's `/proc/self/environ` is the exec-time environment of the process it was forked from, so a worker +calling `ENV.delete` changes nothing about what a sibling reads. Two controls replace it. Invariant 2 +keeps anything worth stealing out of a cell's environment in the first place. And invariant 9 requires +tools to be spawned with `unsetenv_others: true` and an explicitly written environment, because a tool is +`exec`ed and therefore does get a fresh `/proc//environ` — the one thing in this picture that is +actually under our control. + +`hidepid=2` on `/proc` would hide sibling processes entirely and remove the sysctl dependency. Docker +can't provide it: it rejects Podman's `--security-opt proc-opts=`, and remounting `/proc` inside the +container needs `CAP_SYS_ADMIN`, which `cap-drop ALL` removes. diff --git a/docs/imagemagick.md b/docs/imagemagick.md new file mode 100644 index 0000000..031a142 --- /dev/null +++ b/docs/imagemagick.md @@ -0,0 +1,269 @@ +--- +type: Reference +title: "ImageMagick" +order: 13 +description: "ImageMagick's MAGICK_* resource limits, how they interact with a cell, and the formulas to set them." +sources: + - activestorage-hotcell-server/lib/active_storage/hot_cell/server/magick_operation.rb + - activestorage-hotcell-server/lib/active_storage/hot_cell/server/operation.rb +--- + +# ImageMagick + +This page describes ImageMagick's resource limits and how to set them in a cell. ImageMagick bounds itself +through the `MAGICK_*_LIMIT` environment variables and a `policy.xml`. The installed `Dockerfile` installs +no ImageMagick and sets none of them. An image that installs ImageMagick must set them. + +The formulas start from the container flags and cell settings in [Container](container.md), +[Cell settings](cell-settings.md), and [Scratch](scratch.md): `memory`, `concurrency`, `file_size`, and +the scratch's size. + +## Variables + +ImageMagick decodes every frame into a *pixel cache*, an uncompressed `width × height × bytes per pixel` +array, and the limits decide where each cache lives. ImageMagick's +[resources page](https://imagemagick.org/script/resources.php) is the reference. This table summarizes it. + +| Variable | Unit | Bounds | +| --- | --- | --- | +| `MAGICK_MEMORY_LIMIT` | bytes | Caches on the heap, all frames together. | +| `MAGICK_MAP_LIMIT` | bytes | Caches that missed the heap and are memory-mapped files. | +| `MAGICK_DISK_LIMIT` | bytes | Cache files on disk, mapped or not, all frames together. Past it, the operation fails with `cache resources exhausted`. | +| `MAGICK_AREA_LIMIT` | pixels | The largest single frame allowed on the heap. A larger frame goes to a file even when the heap would hold it. It never refuses a frame. | +| `MAGICK_THREAD_LIMIT` | threads | The OpenMP team that ImageMagick asks for. | +| `MAGICK_TMPDIR` | path | Where the cache files go. Read before `TMPDIR`. | + +Bytes per pixel come from the build. ImageMagick 6 Q16 without HDRI, which is Ubuntu's +`imagemagick-6.q16` (four 16-bit channels), uses 8, or 10 with an index or black channel. An HDRI build +stores floats and doubles both. A 4096 × 4096 image is 128MiB before anything is done to it, a layered PSD +holds one cache for each layer at once, and a transform holds source and result at once. + +The three byte limits are a chain: heap, then mapped file, then plain file, and a cache takes one tier +whole. Both file tiers count against `disk`. So a single frame is readable only if it fits `memory` or +`disk` on its own. A multi-frame file needs every frame placed in turn, each in the first tier with room +left, so its frames can total at most `memory` plus `disk`, and less when they pack badly: a frame that +misses the heap's remainder must fit the disk's remainder on its own. + +Each limit has three sources: + +1. The build's default: most of the host's RAM, and unbounded disk. +2. The environment, which replaces the default, upward or downward. +3. `policy.xml`, a ceiling on both. The environment can lower it and can never raise it. + +ImageMagick reads the limits once: when the `magick` process starts, or at ImageMagick's first use inside a +process that loaded the library. Limits apply to each process. `identify -list resource` prints the +result. + +## How the limits interact in a cell + +**The environment reaches both ways that ImageMagick runs in a cell.** + +- In-process, when libvips delegates PSD, BMP, and ICO to `magickload`. The library reads the worker's + environment. +- As a child, when mini_magick spawns `identify` or `magick` for `analyzers.image.magick` and + `transformers.image.magick`. mini_magick runs with `restricted_env`, which passes the child only `HOME`, + `PATH`, `LANG`, and `MiniMagick.cli_env`. So `MagickOperation` rebuilds `cli_env` for every request from + the worker's `MAGICK_*_LIMIT`, `TMPDIR`, and `MAGICK_TMPDIR`. + +**The gem sets `MAGICK_TMPDIR`.** `Operation#initialize` sets it from the request's `TMPDIR`, so in-process +cache files land in the request's directory and are removed with it. Don't set it in the image, and don't +set a `temporary-path` policy, which overrides it. + +**Every limit divides by `concurrency`.** The limits apply to each process, and the cell runs +`concurrency` processes against one `/tmp` and one cgroup. + +**`file_size` can't do the disk limit's job.** `file_size` is `RLIMIT_FSIZE`, a limit on each file. A +layered PSD writes one cache file for each layer, and twenty 40MiB caches total 800MiB with no file near a +48MiB `file_size`. Only `MAGICK_DISK_LIMIT` bounds the sum. Where the two meet, the smaller fires first, as +an `fsize` kill or as `cache resources exhausted`. Both are permanent verdicts on the file. + +**A disk limit larger than the scratch isn't a limit.** The scratch fills first. A full scratch fails every +request on the cell that needs scratch, and a write that fails inside libvips is a `Vips::Error`, which the +gem declares `unreadable`: permanent, and recorded against a customer's file. On 2026-09-01, one PSD under +a `10GiB` disk limit filled a 4G scratch on six hosts and convicted 3,393 blobs. Under a disk limit sized +to a worker's share, the same file is `unreadable` on its own first request, in milliseconds, with the +scratch empty afterward. + +**For `magickload`, the cache files are the spill.** libvips writes a decode above `VIPS_DISC_THRESHOLD` to +`TMPDIR`. When the decode is ImageMagick's, its cache files take the place of that temporary file rather +than adding to it. So the disk limit and the transformer's `file_size` describe the same bytes. + +**A heap cache is charged to `RLIMIT_DATA` and to the cgroup.** It's private anonymous memory, so the +operation's `memory` counts it, and so does the container's. `MAGICK_MEMORY_LIMIT` plus the worker's own +footprint must be under a worker's share of the cgroup and under the `memory` of every operation that can +reach ImageMagick. Above either, the process dies (`killed`, or libgomp's `exit(1)` on a thread that it +can't create) instead of refusing the frame. A death is transient, so the request is retried against a +limit that fails it again. + +**`MAGICK_AREA_LIMIT` adds nothing.** It moves the heap-or-file line in pixels, and `memory` already draws +it in bytes. Set below `memory`, it sends frames to scratch that the heap could have held. + +**`MAGICK_THREAD_LIMIT` raises the OpenMP count itself**, and `OMP_THREAD_LIMIT` caps it. See +[Bound the OpenMP thread pools](container.md#bound-the-openmp-thread-pools). + +## Set the limits + +### Formulas + +Every "MiB" is `1024²` bytes, as `config.rb` writes it. Work out each operation that can reach ImageMagick +separately, and set the image to the smallest result. + +**`MAGICK_DISK_LIMIT`** + + scratch ÷ concurrency − what else the operation writes on scratch + +- Scratch is the tmpfs `size=`, or the host filesystem's usable size as `df` reports it, which on an ext4 + made with default options is 5% under the nominal size. A named volume has no size to divide. +- The subtraction is the operation's own files: its output, and its input if it stages one. The shipped + image operations read their input through the descriptor and stage nothing. A transformer writes its + output on scratch before copying it out. +- The result must be zero or more. Below zero, enlarge the scratch or lower `file_size` or `concurrency`. + ImageMagick reads a negative value as a huge one. + +**`MAGICK_MAP_LIMIT`** = `MAGICK_DISK_LIMIT`. The mapped caches are the same files on the same scratch. + +**`MAGICK_MEMORY_LIMIT`** + + (memory − tmpfs) ÷ concurrency − a worker's own footprint, rounded down + +- On a disk-backed scratch, there's no tmpfs term. +- The footprint is the worker's resident size with the gems loaded, plus the child that it can spawn. + Measure it as `VmRSS` in `/proc//status` after a request. +- Round down to leave the worker room for what the limit doesn't count. +- Check the result plus the footprint against the `memory` of each operation that reaches ImageMagick. + +**`MAGICK_AREA_LIMIT`**: unset. + +**`MAGICK_THREAD_LIMIT`**: `OMP_NUM_THREADS`, or unset. + +**`MAGICK_TMPDIR`**: unset. + +**`policy.xml`**: the same `disk` value as `MAGICK_DISK_LIMIT`. + +### Worked example + +This example is a production cell that serves image transforms and analysis for a Rails application. + +**The volume mount.** `/tmp` is a bind mount of a loopback ext4 image on the host. The image is sized so +that `df` reports 4096MiB available when the mount is empty. The host's configuration management mounts it +`nosuid,nodev,noexec` and chowns it to the cell's uid: + +```yaml +accessories: + active_storage: + volumes: + - hotcell-sockets:/run/hotcell/cell + - /var/lib/hotcell-scratch:/tmp + options: + cpus: 2 + memory: 2g + memory-swap: 2g +``` + +Scratch is on disk rather than on the installed tmpfs, because a tmpfs is RAM charged to the cgroup: every +byte of scratch is a byte that the workers can't have. It's a dedicated filesystem rather than a named +volume or a plain directory, because the filesystem's size is the cap and its mount flags reach the +container. Docker can set neither on a bind mount. [Host-mounted filesystem](scratch.md#host-mounted-filesystem) has the recipe. + +**The constraints.** + +| Input | Value | From | +| --- | --- | --- | +| scratch | 4096MiB | `df --output=avail` on the empty loopback filesystem | +| container `memory` | 2048MiB | `memory: 2g`, no tmpfs term | +| `concurrency` | 4 | `config.rb`, twice `cpus` | +| cell ceiling | `memory: 1536MB`, `file_size: 768MB` | `config.rb`. An operation's own limits are clamped to these. | +| `transformers.image.vips` | `memory: 1280MB`, `file_size: 768MB` | `file_size` raised in the application's operations file | +| `analyzers.image.vips` | `memory: 1024MB`, `file_size: 48MB` | gem defaults | +| `analyzers.image.magick` | `memory: 1024MB`, `file_size: 48MB` | gem defaults | +| `OMP_NUM_THREADS` | 2 | the image, matching `cpus` | +| ImageMagick | 6 Q16 | 8 bytes per pixel | + +Three operations reach ImageMagick: the vips transformer and analyzer through `magickload`, and the +magick analyzer through an `identify` child. All three read their input through the descriptor. The +transformer writes its output on scratch, and the analyzers write nothing. + +**Disk.** Nothing is staged, so the only thing to hold back is the transformer's output, measured at a +peak of 256MiB per worker: + + MAGICK_DISK_LIMIT = 4096 ÷ 4 − 256 = 768MiB + MAGICK_MAP_LIMIT = 768MiB + +This equals the transformer's `file_size`, because the same arithmetic sized it, the application's test +holds it, and the cell ceiling allows it. Four workers that spill 768MiB and write 256MiB beside it fill +the 4096MiB exactly. + +That's why the scratch input is what `df` reports available, not the image's size: ext4's metadata and +its default 5% root reserve take part of the image, so a 4G image reports under 3891MiB, for which the +same arithmetic gives 716MiB. Size the image above what the arithmetic needs, and take the number from +`df` on a cell host. + +On the analyzer, whose `file_size` is 48MB, a cache file over 48MB takes the `fsize` kill first. + +**Memory.** + + per worker = 2048 ÷ 4 = 512MiB + footprint = 70MiB resident + 12MiB for an `identify` child = 82MiB + ceiling = 512 − 82 = 430MiB + MAGICK_MEMORY_LIMIT = 384MiB, the round number under it + +Four workers at the limit use `4 × (384 + 82) = 1864MiB`, under the 2048MiB cgroup. `384 + 82 = 466MiB` +is under the analyzers' 1024MB and the transformer's 1280MB. + +**Together.** + + MAGICK_MEMORY_LIMIT = 384MiB + MAGICK_MAP_LIMIT = 768MiB + MAGICK_DISK_LIMIT = 768MiB + MAGICK_THREAD_LIMIT = 2 + MAGICK_AREA_LIMIT = unset + +and `disk` at 768MiB in `policy.xml`. + +**What that buys**, at 8 bytes a pixel and summed over every cache open at once: + +| Limit | Pixels | For instance | +| --- | --- | --- | +| 384MiB heap | 50.3 million | 8192 × 6144 | +| 768MiB disk | 100.7 million | 16384 × 6144 | +| both, multi-frame | up to 151 million | a 19-layer PSD at 2740 × 2740: 6 layers on the heap, 13 on disk | + +A same-size transform holds source and result, so it decodes half a tier. A larger file is `unreadable` +on its first request, and the scratch is empty afterward. The limits that the image had inherited from +the application were `1GiB`, `5GiB`, and `10GiB`: memory twice a worker's share of the cgroup, and disk two +and a half times the scratch. + +### Check a built image + +Inside the image, as the cell's user, run `identify -list resource` to print the effective limits. `Disk` +must be the number that you computed, and `Area` must be unbounded. Run `identify -list policy` to see what +`policy.xml` set, and check that `temporary-path` is absent. + +To watch the tiers, run the following: + +``` +MAGICK_MEMORY_LIMIT=64MiB magick -size 4000x4000 xc:red -debug cache info: +``` + +On ImageMagick 6, the command is `convert`. + +The cache opens as a file under `MAGICK_TMPDIR`. Raise the size until it passes the disk limit too, and +the command fails with `cache resources exhausted`. + +To check end to end, send a layered file larger than the disk limit through the cell. The answer must be +`unreadable`, and the scratch root must be empty afterward. A cache file left behind means that the spill +went to `/tmp` rather than to the request's directory. + +### When to recompute + +Recompute the limits when any input changes: + +- The scratch's size. +- The container's `memory`. +- `concurrency`. +- An operation's `file_size` or `memory`. +- The worker footprint. +- Which operations reach ImageMagick. +- The ImageMagick build's bytes per pixel. + +Keep the formulas next to the values in the `Dockerfile`. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..399c694 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,30 @@ +--- +okf_version: "0.2" +title: "Reference manual" +description: "One page for each part of Hot Cell, for agents and for readers who want the details." +--- + +# Reference manual + +This manual describes Hot Cell one topic per page. For an introduction and a quick start, see the +[README](../README.md). + + + +| Page | Description | +| --- | --- | +| [Concepts](concepts.md) | The terms the Hot Cell documentation uses: cell, supervisor, worker, slot, operation, client, tool, payload, inputs and outputs, and codes. | +| [Request lifecycle](request-lifecycle.md) | What happens between a call to perform_in_hotcell and its answer, step by step. | +| [Client API](client-api.md) | HotCell.register and its options, HotCell::Client, the errors a bad call raises, the boot checks, diagnosis, and the group the application shares with a cell. | +| [Operation API](operation-api.md) | HotCell::Operation's class and instance methods, run_tool, and the Input and Output descriptors that perform receives. | +| [Response codes](codes.md) | Every failure code and kill cause, whether each is permanent or transient, the exception classes the client raises, and what Active Storage records. | +| [Container](container.md) | Building the cell's image, every container flag and what it does, bounding OpenMP, and checking an accessory before and after a deploy. | +| [Scratch](scratch.md) | Where a cell stages files: the tmpfs, named volume, and host-mount layouts, and the boot sweep. | +| [Cell settings](cell-settings.md) | HotCell.limits: the scheduling settings and request limits with their defaults, the environment variables, the load order, and development mode. | +| [Tuning](tuning.md) | Which measurement sets each cell number, the constraints between the numbers, and how to use bin/load. | +| [Observability](observability.md) | Recommended alerts, the perform.hot_cell notification, the application log line, Yabeda metrics, cell metrics, the cell log schema, and the healthchecks. | +| [Conformance](conformance.md) | What bin/conformance checks about a cell image, and what it cannot check. | +| [Active Storage operations](active-storage.md) | The Active Storage client classes, the cell operations that serve them, how their failures are retried, and the limits each declares. | +| [ImageMagick](imagemagick.md) | ImageMagick's MAGICK_* resource limits, how they interact with a cell, and the formulas to set them. | + + diff --git a/docs/observability.md b/docs/observability.md new file mode 100644 index 0000000..f037c19 --- /dev/null +++ b/docs/observability.md @@ -0,0 +1,401 @@ +--- +type: Reference +title: "Observability" +order: 10 +description: "Recommended alerts, the perform.hot_cell notification, the application log line, Yabeda metrics, cell metrics, the cell log schema, and the healthchecks." +sources: + - hotcell-server/lib/hot_cell/log.rb + - hotcell-server/lib/hot_cell/counters.rb + - hotcell-server/lib/hot_cell/control.rb + - hotcell-server/lib/hot_cell/health_operations.rb + - hotcell-server/exe/hotcell-health + - hotcell-client/lib/hot_cell/client.rb + - hotcell-client/lib/hot_cell/log_subscriber.rb + - hotcell-client/lib/hot_cell/health_controller.rb + - hotcell-client/lib/hot_cell/diagnostics_controller.rb + - yabeda-hotcell/lib +--- + +# Observability + +This page describes the signals that Hot Cell produces and the alerts to set on them. The signals are as +follows: + +- The `perform.hot_cell` notification, which the application publishes for each call. +- The application log line that `HotCell::LogSubscriber` writes for each call. +- The metrics that `yabeda-hotcell` records. +- The counters that a cell reports on its control socket. +- The cell's own log. +- The container and Rails healthchecks. + +For which signal sets which limit, see [Tuning](tuning.md). + +## Recommended alerts + +- **Cell availability.** Alert when the `up` gauge is 0 or absent for any cell on any host. It reads 0 + first when a deploy missed a role, when the application lacks the cell's group, or when the supervisor is + dead. On a host without `HOTCELL_ROOT`, the gauge is absent. Calls on that host raise + `HotCell::CellNotConfigured`. +- **Failed calls.** Alert on the `requests` counter by `code`. The application records `unavailable` + when the cell is down, restarting, or unreachable. Any shift away from `ok` is an early warning. Put the + primary alarm on this signal rather than on the cell's own metrics, because the application records it + even when the cell is dead. +- **Queue headroom.** Alert when `queued` nears the cell's `queue_size`, when `queue_high_water` rises + toward it, or when `capacity` appears in steady state. Each means that the cell is under-provisioned. + `queue_size` is configuration, not a metric. `queue_high_water` resets only at boot, so alert on its + rise. A rising `cancelled` means that callers gave up waiting. +- **Scratch space.** Alert on free space on each host's scratch. For a disk-backed scratch, use + `node_filesystem_avail_bytes` from the node exporter. For a tmpfs, compare the container's memory usage + with the tmpfs `size=`. A full scratch fails every request that needs it. A write that fails inside + libvips gets `unreadable` from the cell, a permanent verdict against the file (see + [ImageMagick](imagemagick.md#how-the-limits-interact-in-a-cell)). [Scratch](scratch.md) covers the + layouts. +- **Cell errors.** Alert on any `ERROR` event in the cell log, such as `worker.crashed` or + `worker.unforkable`, which should never happen. Alert on a rise in the `killed` gauge by cause. A single + kill for `memory` or `fsize` is the cell rejecting a hostile file. + +## What to watch + +| Signal | What it means | +| --- | --- | +| `killed_by` by cause | The only legitimate reason to tighten a limit. | +| `queued_ms` p95 rising, `perform_ms` p95 flat | The cell needs more workers, not faster ones. | +| `perform_ms` p95 rising | The work got more expensive. Check for a library upgrade. | +| `queued` near `queue_size`, or `queue_high_water` rising toward it | No headroom is left. `queue_high_water` resets only at boot. | +| `capacity` above zero in steady state | The cell is under-provisioned. | +| `unavailable` | The cell is down, restarting, or unreachable. | +| `unreadable` rate | Worth watching after a toolchain upgrade. | +| `worker.crashed` in the log | Should be zero. Anything else is a bug worth reporting. | + +## Per-call notification + +The client publishes the `perform.hot_cell` Active Support notification for every call, whether it +succeeds or fails. It's the only signal that survives a dead cell: an unreachable socket arrives here as +`unavailable`. `HotCell::LogSubscriber` and `yabeda-hotcell` subscribe to it. To record anything else, +subscribe to it yourself. + +The payload has the following keys: + +| Key | Description | +| --- | --- | +| `operation` | The routing name. | +| `cell` | The registered cell name. | +| `code` | The failure's code. `nil` on success. | +| `cause` | The cause of a `killed` failure. | +| `signal` | The signal that ended the worker, if any. | +| `stderr` | The tail of what a dying worker wrote to file descriptor 2. Write it to a log field and nowhere else: a tool wrote it while it processed a hostile file. | +| `permanent` | Whether the failure is permanent. `nil` on success. | +| `bytes_in`, `bytes_out` | The total size of the inputs and of the outputs. `nil` when the client couldn't measure them, which isn't the same as zero. | +| `perform_ms` | Time that the cell spent in `perform`. | +| `timing` | Every timing that the cell reported, such as `queued_ms` and `perform_ms`. | + +Classify failures by `permanent`, not by `code`. A `killed` failure is permanent for `fsize` and `memory`, +and transient for `deadline` and `crashed`, so the code alone can't say which side of the split a kill is +on. `permanent` is the cell's own answer, and `cause` is the reason. See [Response codes](codes.md). + +A subscriber's own duration minus `perform_ms` is transport plus queueing. Track the two separately: a +rising `perform_ms` means that the work got more expensive, and a rising difference means that the cell is +saturated. + +## Application logs + +In a Rails application, `HotCell::LogSubscriber` writes one `info` line for each call to the Rails log: + +``` + HotCell (41.2ms) {"cell":"images","operation":"active_storage.transformers.image.vips","code":"ok","perform_ms":38,"duration_ms":41.2,"bytes_in":20480,"bytes_out":8192} +``` + +For a failed call, the line adds `cause` and `stderr` when they exist. For a call that an exception +interrupted, such as the application's own request timeout, the line has the exception's class in place +of the code. + +To turn the line off, call `HotCell::LogSubscriber.detach_from :hot_cell` in an initializer. + +To use the line without Rails, do the following: + +1. Require `hot_cell/log_subscriber`. +2. Call `HotCell::LogSubscriber.attach_to :hot_cell`. +3. Set `ActiveSupport::LogSubscriber.logger`. + +## Metrics + +The `yabeda-hotcell` gem records Hot Cell metrics in [Yabeda](https://github.com/yabeda-rb/yabeda). To +install it, add the gem to the application's `Gemfile` and call `Yabeda::HotCell.install!` once at boot: + +```ruby +# Gemfile +gem "yabeda-hotcell" + +# config/initializers/hotcell.rb +Yabeda::HotCell.install! +``` + +The metrics are in the `hotcell` group: + +| Metric | Type | Tags | Description | +| --- | --- | --- | --- | +| `requests` | counter | `cell`, `operation`, `code`, `cause` | Each call. `code` is `ok` on success, and `cause` is empty when there's none. | +| `perform` | histogram | `cell`, `operation` | Seconds that the cell spent in `perform`. | +| `up` | gauge | `cell` | 1 when the local cell answers its control socket, otherwise 0. | +| `running` | gauge | `cell` | Workers busy right now. | +| `queued` | gauge | `cell` | Connections waiting for a worker. | +| `queue_high_water` | gauge | `cell` | The deepest that the queue has been since boot. | +| `cancelled` | gauge | `cell` | Callers that gave up before the cell answered. This is a floor. | +| `killed` | gauge | `cell`, `cause` | Workers killed since boot, by cause. | +| `uptime_seconds` | gauge | `cell` | Seconds since the supervisor booted. | + +On each scrape, the gem reads `metrics` from each registered cell and sets the gauges from it. A scrape +never fails because a cell misbehaves: the gem reports the error to the Active Support error reporter. + +## Cell metrics + +A cell answers `metrics` on its control socket. It answers even when the work socket is saturated. The +control socket is local to its host, so the process that polls it must run on the cell's host. + +`metrics` reports `running`, `queued`, `queue_high_water`, `cancelled`, request counts by code, and +`killed_by`, broken down by cause. + +`killed_by` counts what workers reported, not what the supervisor observed. A worker decides its own +`memory` and `fsize` verdicts, because the supervisor can't tell either from a wait status without +believing a signal that a sibling could have sent. The worker reports the cause when it reports itself +idle. As a result: + +- The count arrives just after the caller has its answer, rather than before. +- The count is lost if the worker dies between the answer and the report. +- A compromised worker can report a cause that its request never had. + +Size limits from `killed_by`. Don't read it as evidence about any particular document. + +## Cell log + +The cell writes one JSON object for each event to standard output, so whatever ships your container logs +ships these too. + +Field names follow [ECS](https://www.elastic.co/guide/en/ecs/current/index.html), the schema that the +rest of the fleet's structured logs use. Every field that ECS has no name for is in the `hotcell` +namespace, so no future ECS field can collide with a Hot Cell field. + +### Envelope + +Every line carries these fields: + +| Field | Type | Description | +| --- | --- | --- | +| `@timestamp` | string | When the event happened. UTC, ISO 8601, millisecond precision. | +| `service.name` | string | Always `"hotcell"`. This is the routing key: the log collector selects Hot Cell lines by it. | +| `event.action` | string | Which event this is. See [Events](#events). | +| `log.level` | string | `"INFO"`, `"WARN"`, or `"ERROR"`. The cell decides severity, not the collector, so adding an event never requires a collector change. | +| `process.pid` | integer | The process that the event is about: the worker's pid for `worker.*` events, the supervisor's own for `cell.*` events. Absent where no process is the subject. | + +**Caution:** The fleet's log collector routes on `service.name == "hotcell"`, sets the record timestamp +from `@timestamp`, and takes severity from `log.level`. Renaming any of these three fields silently drops +or mislabels every cell log line in production. The rest of the schema can change. + +### Shared fields + +Where ECS has a name, the cell uses it: + +| Field | Type | Used by | +| --- | --- | --- | +| `error.type` | string | The exception class name, wherever an exception is reported. | +| `error.message` | string | The sanitized exception message, beside `error.type`. | +| `message` | string | Prose detail, on events whose meaning needs it (`cell.ptrace_scope_unknown`, `slot.uncleaned`, `worker.unreadable_report`). | +| `event.outcome` | string | `"success"` or `"failure"`, on `request` only. | +| `event.duration.ms` | number | Wall time of the thing that ended. This is the fleet's dialect (Rails logs use `event.duration.ms`), not stock ECS (`event.duration` in nanoseconds). | +| `process.exit_code` | integer | The worker's exit status, on `worker.reaped`. | + +### Hot Cell fields + +Every other field is in the `hotcell` namespace: + +| Field | Type | Description | +| --- | --- | --- | +| `hotcell.slot` | integer | The slot number. On nearly every event. | +| `hotcell.op` | string | The operation that the line is about, on `request`, `request.abandoned`, `worker.crashed`, `worker.killed`, and `worker.undispatchable`. `null` where the name wasn't known: a request that never parsed, a crash between requests, or a worker that died before it reported. Never the name of an earlier request. See [Which operation a line is about](#which-operation-a-line-is-about). | +| `hotcell.code` | string | The response code of a `request` (`"ok"`, `"failed"`, `"killed"`, and so on). | +| `hotcell.permanent` | boolean | Whether a `request` failure is permanent. | +| `hotcell.cause` | string | Why a worker was killed (`"deadline"`, `"memory"`, `"fsize"`, and so on). | +| `hotcell.signal` | string | The signal name (`"SIGKILL"`, `"SIGSEGV"`, and so on). ECS has no field for signals. | +| `hotcell.served` | integer | Requests that a worker served before it was reaped. | +| `hotcell.swept` | integer | Discarded trees that a sweeper found cleared, on `scratch.swept`: unlinked by it, or already gone when it reached them. | +| `hotcell.home` | string | The scratch directory that a cleanup couldn't clear: a request's `$HOME` from a worker, or the slot directory from the supervisor. | +| `hotcell.directory` | string | The cell's working directory, on `cell.boot`. | +| `hotcell.operations` | array | The registered operation names, on `cell.boot`. | +| `hotcell.configuration` | object | The full configuration inventory, in the same shape as `hotcell.describe`, on `cell.boot`. | +| `hotcell.running`, `hotcell.queued` | integer | In-flight and queued requests, on `cell.stopping`. | +| `hotcell.timing` | object | A request's phase timings: `queued_ms`, `perform_ms`, and any other measured phases. | +| `hotcell.deadline_s`, `hotcell.grace_s`, `hotcell.waited_s` | number | The limit that was hit, on the event that reports hitting it. | +| `hotcell.path` | string | On `cell.ptrace_scope_unknown`, the file that the cell couldn't verify. On `scratch.unswept`, the scratch entry, or the scratch itself, that a boot couldn't remove. | +| `hotcell.stderr` | string | The tail of what a dying worker wrote to file descriptor 2, at most 512 bytes. On `worker.killed` only, and absent when the worker wrote nothing. See [What a worker wrote to fd 2](#what-a-worker-wrote-to-fd-2). | + +### Events + +| `event.action` | `log.level` | Fields beyond the envelope | +| --- | --- | --- | +| `cell.boot` | INFO | `hotcell.directory`, `hotcell.operations`, `hotcell.configuration` | +| `cell.stopping` | INFO | `hotcell.running`, `hotcell.queued` | +| `cell.stopped` | INFO | — | +| `cell.ptrace_scope_unknown` | ERROR | `hotcell.path`, `message` | +| `request` | INFO | `hotcell.slot`, `hotcell.op`, `hotcell.code`, `hotcell.permanent`, `event.outcome`, `event.duration.ms`, `hotcell.timing` | +| `request.abandoned` | WARN | `hotcell.slot`, `hotcell.op` | +| `worker.forked` | INFO | `hotcell.slot` | +| `worker.reaped` | INFO | `hotcell.slot`, `hotcell.served`, `hotcell.signal`, `process.exit_code` | +| `worker.crashed` | ERROR | `hotcell.slot`, `hotcell.op`, `error.type`, `error.message` | +| `worker.killed` | WARN | `hotcell.slot`, `hotcell.op`, `hotcell.cause`, `hotcell.signal`, `event.duration.ms`, `hotcell.stderr` | +| `worker.deadline` | WARN | `hotcell.slot`, `hotcell.deadline_s` | +| `worker.lingered` | WARN | `hotcell.slot`, `hotcell.grace_s` | +| `worker.unforkable` | ERROR | `hotcell.slot`, `error.type`, `error.message` | +| `worker.undispatchable` | ERROR | `hotcell.slot`, `hotcell.op`, `error.type` | +| `worker.unreadable_report` | ERROR | `message` | +| `sweeper.forked` | INFO | — | +| `sweeper.deadline` | WARN | `hotcell.deadline_s` | +| `sweeper.died` | WARN | `hotcell.signal`, `process.exit_code`. A sweeper that ended abnormally by anything but the deadline kill. | +| `sweeper.unforkable` | ERROR | `error.type`, `error.message` | +| `sweeper.crashed` | ERROR | `error.type`, `error.message` | +| `scratch.swept` | INFO | `hotcell.swept`, `event.duration.ms` | +| `control.abandoned` | WARN | `hotcell.waited_s` | +| `control.unanswerable` | WARN | `error.type`, `error.message` | +| `slot.uncleaned` | WARN | `hotcell.slot`, `hotcell.home`, `message`. Boot sweep only. | +| `slot.undiscarded` | WARN | `hotcell.slot`, `hotcell.home` | +| `slot.unswept` | WARN | `hotcell.slot`, `hotcell.home`. From the worker that answered on the slot, or from the sweeper. | +| `scratch.unswept` | WARN | `hotcell.path`, and `error.type` and `error.message` when the scratch itself couldn't be listed. | + +### Examples + +A request: + +```json +{"@timestamp":"2026-08-14T21:05:28.252Z","service":{"name":"hotcell"},"event":{"action":"request","outcome":"success","duration":{"ms":9.6}},"log":{"level":"INFO"},"process":{"pid":83},"hotcell":{"slot":0,"op":"active_storage.transform_image","code":"ok","permanent":null,"timing":{"queued_ms":0.4,"perform_ms":0.52}}} +``` + +A crash: + +```json +{"@timestamp":"2026-08-14T21:05:29.107Z","service":{"name":"hotcell"},"event":{"action":"worker.crashed"},"log":{"level":"ERROR"},"process":{"pid":83},"error":{"type":"NoMethodError","message":"undefined method 'blur' for nil"},"hotcell":{"slot":0,"op":"active_storage.transform_image"}} +``` + +### What a worker wrote to fd 2 + +A worker's file descriptor 2 is a pipe to the supervisor. The supervisor drains it as the worker runs and +attaches the tail to the `worker.killed` event that reports the worker's death. The same text is in the +failure that the caller receives, so an application logs `killed: crashed (libgomp: ...)` rather than a +bare `crashed`. + +The field exists for the one death that nothing else in a cell can describe. `HotCell::Worker#run` +rescues `Exception`, so a worker that died with no `worker.crashed` line probably died without Ruby +raising at all: a C library called `exit()` and wrote why to fd 2. For example: + +``` +libgomp: Thread creation failed: Resource temporarily unavailable +``` + +**Caution:** The text isn't evidence. It doesn't establish who wrote it or which request it belongs to, +the same caveat that `hotcell.signal` and `hotcell.cause` carry. It comes from the one process in a cell +that runs untrusted code, over an unauthenticated channel: + +- Everything that a worker spawned inherits fd 2, so a tool can write long after its request finished. +- A sibling worker can open `/proc//fd/2` and write anything, because workers share a uid, and + `kernel.yama.ptrace_scope` protects memory, not descriptors. + +The supervisor clears the buffer at each dispatch, which keeps an old warning off an unrelated death in the +ordinary case. That isn't a boundary. + +The capture is best effort, because fd 2 is non-blocking. A C library that writes to a full pipe gets +`EAGAIN` and loses the line, and a fatal handler can't retry: it writes once and calls `exit()`. That costs +nothing in the case that the field exists for, where libgomp's one short line meets an empty pipe. It +loses the fatal message from a decoder that already filled the pipe with warnings: then the field reports +the tail of those warnings instead. A blocking fd 2 was rejected, because a warning written from inside +libvips would then wait on the supervisor's scheduling, in a C call that Ruby can't interrupt, and that +wait is longest exactly when the host is under pressure. + +Only a death is reported. A worker that warns and then answers normally leaves no field on any event. So a +tool that dies while its worker survives isn't described here. A cell's standard error doesn't reach the +container's log driver. + +### Which operation a line is about + +`hotcell.op` lets a cell's own logs answer "which operation did this?". A cell runs several operations at +once, and they don't share limits, so an unattributed `worker.killed` can't be acted on. Nothing else +supplies the name: the response carries no operation, and `hotcell_killed` is tagged with `cell` and +`cause` only. + +The two processes learn the name differently, which is why it can be absent: + +- A **worker** parses it from the request that it's serving. `request`, `request.abandoned`, and + `worker.crashed` carry it from the moment the request parses until the worker goes back to waiting. A + request that never parsed has no name, and neither does a crash between requests. +- The **supervisor** never reads a request that a worker will serve, because staying out of it is what + lets the supervisor dispatch a connection whose descriptors are still queued on it. It learns the name + from the worker's report, which the worker sends after the request parses and before it touches an + untrusted byte. That lets `worker.killed` name an operation that the dead worker can't report. If the + worker died first, the field is `null` rather than the last request's name. + +The report comes from the one process here that runs untrusted code, so the supervisor bounds the name to +an operation that this cell registered. The bound is on the report and nowhere else: a `request` line +names whatever the caller asked for, including a name that no operation answers to, which is what +`unsupported` is about and is worth seeing. A worker doesn't send a name whose report wouldn't fit one +control line, so `worker.killed` goes unattributed rather than losing the narrowed deadline that shares +the line. + +`worker.undispatchable` is the exception, and the one line where the supervisor reads a request: the +worker died between the fork and the dispatch write, so nothing else read the request. The supervisor +peeks rather than reads, so neither the bytes nor the caller's descriptors leave the connection. It never +waits, so a request that hasn't arrived leaves the field `null`. + +## Container healthcheck + +The installed `Dockerfile` sets `hotcell-health` as the Docker `HEALTHCHECK`. It probes the supervisor's +control socket from inside the container, where `network: none` doesn't apply. Healthy means that the +supervisor answers, not that a worker is free. If you use your own container, set this healthcheck too. + +`HOTCELL_HEALTH_TIMEOUT` sets how many seconds `hotcell-health` waits for an answer before it reports +unhealthy. See [Cell settings](cell-settings.md#environment-variables). + +## Rails healthcheck + +`hotcell-client` defines two controllers, `HotCell::HealthController` and `HotCell::DiagnosticsController`, +and no routes for them. Add a route for each to the application's `config/routes.rb`. + +- `HotCell::HealthController` asks each registered cell for `describe` and `metrics` over its control + socket. It returns `OK` with a 200 when at least one cell is registered and every cell answers. + Otherwise, it returns `FAIL` with a 503. These calls take no worker, so you can make the endpoint + public, like `/up`. +- `HotCell::DiagnosticsController` returns the result of every check as JSON, with a 503 if any check + fails. Along with `describe` and `metrics`, it sends `health.echo` and `health.reopen` over the work + socket. Each round trip takes a worker. Put this endpoint behind authentication. + +Of the cell's two sockets, only the work socket carries file descriptors. So `describe` and `metrics` +succeed even when the application can't use the work socket, and only the round trips test it. A cell +without the shared group passes `health.echo` and fails `health.reopen` with `EACCES`. + +To serve the round trips, add `require "hot_cell/health_operations"` to one of the cell's operation files. +Without it, the cell answers `unsupported` for both operations. + +To authenticate the diagnostics controller, set its superclass in an initializer, then add the routes: + +```ruby +# config/initializers/hotcell.rb +HotCell.diagnostics_controller_parent = "Admin::BaseController" + +# config/routes.rb +get "up/hotcell" => "hot_cell/health#show", as: :hotcell_health_check + +constraints subdomain: "admin" do + get "hotcell" => "hot_cell/diagnostics#show", as: :hotcell_diagnostics +end +``` + +If your authentication is a concern, subclass the controller instead: + +```ruby +# app/controllers/hotcell_diagnostics_controller.rb +class HotcellDiagnosticsController < HotCell::DiagnosticsController + include StaffOnly +end + +# config/routes.rb +get "up/hotcell/diagnostics" => "hotcell_diagnostics#show" +``` + +From a console, `HotCell.diagnose(work: true).as_json` returns the same checks. See +[Client API](client-api.md#diagnose-cells). diff --git a/docs/operation-api.md b/docs/operation-api.md new file mode 100644 index 0000000..c18e22f --- /dev/null +++ b/docs/operation-api.md @@ -0,0 +1,283 @@ +--- +type: Reference +title: "Operation API" +order: 4 +description: "HotCell::Operation's class and instance methods, run_tool, and the Input and Output descriptors that perform receives." +sources: + - hotcell-server/lib/hot_cell/operation.rb + - hotcell-server/lib/hot_cell/server/errors.rb + - hotcell-core/lib/hot_cell/descriptors.rb + - hotcell-core/lib/hot_cell/naming.rb + - hotcell-core/lib/hot_cell/declarations.rb +--- + +# Operation API + +This page describes `HotCell::Operation`, the class that you subclass to write work that runs in a cell, +and the `Input` and `Output` objects that `perform` receives. For the application side of the call, see +[Client API](client-api.md). + +The code lives in [`hotcell-server/lib/hot_cell/operation.rb`](../hotcell-server/lib/hot_cell/operation.rb) +and [`hotcell-core/lib/hot_cell/descriptors.rb`](../hotcell-core/lib/hot_cell/descriptors.rb). + +## Example + +The client class in the application and the operation class in the cell share a routing name. The +signature of `perform` (inputs, outputs, payload) is the contract for what crosses the socket. Descriptors +travel as they are, without copies, and the payload travels as one JSON object. + +In the application: + +```ruby +class TransformImage < HotCell::Client + hotcell "images" # the cell that serves this call, by registered name + operation "images.transform" # the routing name an operation must answer to +end + +# source and destination are Files the app already opened. This is a blocking call that waits for +# a response from the cell. +TransformImage.perform_in_hotcell source, destination, + format: "png", + operations: { resize_to_limit: [ 800, 600 ] } +``` + +In the cell: + +```ruby +require "active_support" +require "active_support/core_ext/numeric" # for 30.seconds and 1280.megabytes + +class TransformImageOperation < HotCell::Operation + operation "images.transform" # the same routing name + + # Ceilings for one request enforced by rlimits and the supervisor's clock, + # clamped to the cell's own limits. + limits deadline: 30.seconds, memory: 1280.megabytes, file_size: 48.megabytes + + before_fork { require "my_image_processor" } # once, in the supervisor + before_worker_boot { MyImageProcessor.concurrency_set 4 } # in forked worker, before it serves a request + + # The descriptors the caller passed, and the payload as keyword arguments. A missing or undeclared + # key raises, so the signature is the schema. Declare **payload instead to take the Hash whole. + def perform(inputs, outputs, format:, operations: {}) + source, = inputs + destination, = outputs + + # fd_path reads the caller's file in place, with no copy onto scratch, so an input of any size + # costs nothing against file_size. Reach for source.path only when a tool needs a distinct on-disk + # copy; that stages the bytes, and the kernel charges the write. + MyImageProcessor.source(source.fd_path) + .apply(format:, operations) + .write_to(destination.fd_path) + + # The result: one JSON object, which the caller receives as perform_in_hotcell's return value (in + # addition to the destination file descriptor) + { format: format, bytes: File.size(destination.fd_path) } + end +end +``` + +A cell serves an operation when one of its operation files requires the operation's file. See +[Cell settings](cell-settings.md#load-order). + +## Class methods + +### `operation(name = nil)` + +Sets the routing name, the name that the operation answers to on the wire. With no argument, returns the +routing name. + +If you don't call `operation`, the name derives from the class path: each namespace and the class name in +snake case, joined with dots, with an `Operation` suffix removed. For example, +`Thumbnails::ExtractTextOperation` answers to `thumbnails.extract_text`. An anonymous class must set its +name explicitly. + +A subclass doesn't inherit its parent's routing name. It derives its own from its class path unless it +declares one, so it never answers to the same name as its parent. + +### `limits(**values)` + +Declares this operation's ceilings for one request. The keys are the four request limits: `deadline` in +seconds, and `memory`, `file_size`, and `open_files`. See +[Cell settings](cell-settings.md#request-limits) for what each one enforces. Active Support helpers work: +`1280.megabytes` is an `Integer`, and `30.seconds` becomes a number on arrival. With no arguments, returns +the effective `HotCell::Limits`. + +The cell clamps every value to its own limit for the same key, so an operation can narrow a limit and +never widen it. The effective value is the smaller of the two. + +`limits` accumulates. Naming one limit changes that limit and keeps the rest of this class's declaration, +or of the nearest ancestor's declaration when this class has none yet. Naming a limit as `nil` withdraws +it, which gives that limit back to the cell's value. See +[Change a shipped operation's limits](#change-a-shipped-operations-limits). + +### `before_fork { ... }` + +Registers a block that runs once, in the supervisor, at boot. Use it to require and configure libraries. + +**Caution:** A `before_fork` block must never evaluate an image. libvips can't survive `fork` after it +evaluates an image: every worker forked after that deadlocks. See +[experiment 1](contributing/experiments.md). + +Every megabyte that the supervisor holds is partly copied by every worker, so require only what this +cell's own operations need. See [Cell overhead](contributing/overhead.md). + +### `before_worker_boot { ... }` + +Registers a block that runs in the worker after the fork and before the worker serves a request. Use it +to size a library, for example `Vips.concurrency_set`. Hot Cell doesn't size libraries for you: size them +against the cell's own `cpus` and `concurrency`. + +Configure libraries here rather than in `before_fork`, because library configuration is global. Two +operations that configured the same library in the supervisor would disagree, and the last one registered +would win. + +Above `max_requests_per_worker: 1`, one worker can serve operation A, then B, then A. The worker runs these +blocks again whenever the operation changes, so write them as re-entrant setters, not as one-time +initialization. + +Hooks from a superclass run before the hooks of its subclasses. This applies to `before_fork` and +`before_worker_boot`. + +### `unreadable(*classes)` + +Declares the library exception classes that mean "the input can't be decoded", rather than "the operation +broke". When `perform` raises one of them, the cell answers `unreadable`, which is +[permanent](codes.md). `HotCell::UnreadableInput` is always included. A subclass adds to the classes that +its ancestors declared. + +### `abstract_operation` + +Marks a class that exists to be inherited from, not to be dispatched to. + +Every subclass of `HotCell::Operation` registers itself. Without `abstract_operation`, an intermediate +class that holds shared setup would appear in `describe`, accept requests, and answer `failed` from a +`perform` that raises `NotImplementedError`. A subclass of an abstract operation is concrete unless it +also calls `abstract_operation`. + +## Instance methods + +### `perform(inputs, outputs, **payload)` + +Does the work. Hot Cell calls `perform` on a new instance for each request, so nothing in an instance +variable survives into the next request that a reused worker serves. + +- `inputs` is an `Array` of `Input` objects, and `outputs` is an `Array` of `Output` objects, in the + order that the caller passed them. +- The payload arrives as keyword arguments. Declare the keys that you accept, for example + `format:, operations: {}`, and Ruby validates them on arrival. To take the payload as a `Hash`, declare + `**payload`. If the operation takes no options, declare neither. +- A missing or undeclared key raises inside `perform`, and the cell answers `failed`. +- The return value is the result: a `Hash` that the caller receives as the return value of + `perform_in_hotcell`. It travels as JSON. + +### `run_tool(*command, env: {}, capture: 64 * 1024, pass: [])` + +Runs a tool in a subprocess and returns a `ToolResult` with the following members: + +- `status`: the `Process::Status`. +- `out`: the captured standard output. +- `err`: the captured standard error. +- `ok?`: whether `status` is a success. + +The arguments are as follows: + +| Argument | Description | +| --- | --- | +| `command` | The program and its arguments, for example `"mutool", "draw", ...`. | +| `env` | Environment variables to add or override. | +| `capture` | The maximum number of bytes kept from each of standard output and standard error. `run_tool` reads both streams to the end and discards everything past this limit as it arrives, so a noisy tool can't exhaust the worker's memory. | +| `pass` | Descriptors to hand to the tool at their own descriptor numbers. The tool reaches each one at its `fd_path`, and no byte is copied onto scratch. | + +The tool runs with `unsetenv_others: true` and a fully written environment, never a filtered copy of the +worker's environment. This is [invariant 9](design/invariants.md). The environment contains `HOME`, +`TMPDIR`, and `PATH` from the worker, `LANG` and `LC_ALL` set to `C.UTF-8`, and `OMP_NUM_THREADS` and +`OMP_THREAD_LIMIT` from the cell's environment, plus `env`. See +[Bound the OpenMP thread pools](container.md#bound-the-openmp-thread-pools). + +A tool's output is influenced by the attacker. Parsing it brings those bytes back into the worker. + +## Inputs and outputs + +`Input` and `Output` wrap the descriptors that the caller passed. Both sides verify each descriptor when +they wrap it: + +- An input must be read-only, and an output must be write-only. An access mode is fixed at open and can't + be narrowed afterward, so a cell can only decline a descriptor with the wrong mode. +- A descriptor must refer to a regular file. A pipe as an output can deadlock. +- A descriptor must not be open with `O_APPEND`, which would append the conversion to what the file + already holds. + +A failed check raises `HotCell::AccessModeError`. + +### `fd_path` + +Returns a path that reads or writes the descriptor in place, with no copy onto scratch: `/dev/fd/N` on +Linux. The worker can open it, and so can a tool that received the descriptor through `run_tool`'s +`pass:`. + +Prefer `fd_path`. Reading in place costs nothing against `file_size`, so an operation can read a +multi-gigabyte input under a small `file_size`. Each open of `/dev/fd/N` on Linux is a new file +description at offset zero. + +On macOS, `fd_path` returns the file's own name, because opening `/dev/fd/N` there shares the worker's +offset. macOS is a development platform only. + +### `Input#path` + +Copies the input onto the worker's scratch the first time that you call it, and returns the file's name. +Use it only when a tool needs a separate file on disk. + +The copy is a write, so `file_size` bounds it. An input larger than the operation's `file_size` fails here +as `killed` with cause `fsize`, which is permanent. The shipped Active Storage operations all read +`fd_path` for this reason. + +### `Output#path(extension: nil)` + +Names the file on scratch that the operation writes, the first time that you call it. When `perform` +returns, the worker copies that file back through the descriptor and flushes it before it reports success. + +With `extension:`, returns a sibling name with that suffix, for a tool that appends its own extension, +such as `pdftoppm`. Call `adopt(staged)` to rename the sibling onto `path` so that the worker ships it. + +An operation can instead write directly through the descriptor at `fd_path`. Then nothing is copied, and +the worker only flushes and measures the file. If the operation fails partway through a direct write, the +caller's file holds partial bytes, where a staged output leaves it empty. In either form, if the outputs +hold zero bytes in total after a successful `perform`, the client reports `unavailable`. + +### `staged?` + +Returns whether the descriptor has a file on the worker's scratch. + +## Change a shipped operation's limits + +To give a shipped operation a different budget without editing the gem, redeclare the one limit after the +operation loads, from an operations file: + +```ruby +# hotcell/operations/zz_limits.rb +require "active_storage/hot_cell/server/transformers/image/vips" + +ActiveStorage::HotCell::Server::Transformers::Image::Vips.limits file_size: 128 * 1024**2 +``` + +The transformer's `deadline`, `memory`, and `open_files` don't change. + +Keep the following in mind: + +- **Require the operation first.** The class must exist before you can redeclare its limits. `config.rb` + loads before any operations file, and operations files load in sorted order. Without the `require`, the + redeclaration must be in an operations file that sorts last, hence the `zz_` prefix. With the + `require`, the same lines work from any operations file or from `config.rb`. Prefer an operations file + anyway, so that every declaration about operations is in one place. +- **A subclass copies its parent's limits when it declares.** A subclass that declares + `limits deadline: 5` takes every other value from its parent, so a narrowing subclass writes only the + number that it narrows. It takes them once: `limits` resolves to the first ancestor that declared any, + and after a class stores its own, it stops looking up. Redeclaring the parent afterward doesn't reach a + child that already declared, so redeclare the child too. A subclass that never declares follows its + parent. +- **The cell's limits are still the ceiling.** The cell clamps a redeclaration the same way that it clamps + the original. To get 128MB, the cell's own `file_size` must be at least 128MB. A cell at 64MB gives that + redeclaration 64MB. + +The shipped operations' limits are in [Active Storage operations](active-storage.md#limits). diff --git a/docs/request-lifecycle.md b/docs/request-lifecycle.md new file mode 100644 index 0000000..d02d860 --- /dev/null +++ b/docs/request-lifecycle.md @@ -0,0 +1,60 @@ +--- +type: Reference +title: "Request lifecycle" +order: 2 +description: "What happens between a call to perform_in_hotcell and its answer, step by step." +sources: + - hotcell-client/lib/hot_cell/client.rb + - hotcell-client/lib/hot_cell/transport.rb + - hotcell-server/lib/hot_cell/supervisor.rb + - hotcell-server/lib/hot_cell/worker.rb + - hotcell-core/lib/hot_cell/descriptors.rb +--- + +# Request lifecycle + +This page describes what happens between a call to `perform_in_hotcell` and its answer. For the terms +used here, see [Concepts](concepts.md). + +```mermaid +sequenceDiagram + participant App as app process
(cold side, privileged) + participant Supervisor as supervisor, pid 1
(hot side, unprivileged) + participant Worker as worker
(forked ahead of dispatch) + + App->>Supervisor: one sendmsg -- JSON request + N descriptors + note over Supervisor: doesn't read the request
queues it, or answers capacity + Supervisor->>Worker: passes the connection itself over SCM_RIGHTS + note over Worker: applies the cell's limits before touching the socket
reads the request, narrows to the operation's limits + Worker->>Worker: perform(inputs, outputs, **payload)
an input copies to scratch when asked for a path + Worker->>App: posts the outputs, flushes, answers with one JSON line + note over Supervisor,Worker: a worker past its deadline is killed as a process group,
and the supervisor answers killed + Worker->>Worker: exits, or waits for the next dispatch +``` + +## Steps + +1. The application calls `perform_in_hotcell inputs, outputs, payload` on a client class. The client does + the following: + 1. Wraps each IO as an `Input` or an `Output`. + 2. Verifies each access mode. Inputs must be read-only and outputs must be write-only. + 3. Validates the payload. + 4. Connects to the cell's `work.sock`. +2. One `sendmsg` call carries one JSON line and every descriptor. +3. The supervisor accepts the connection and dispatches it to a worker. If the queue is full, the + supervisor answers `capacity` instead. +4. Before the worker reads any untrusted byte, it narrows its limits to the operation's limits, clamped to + the cell's. If the operation differs from the last one that the worker served, the worker runs + `before_worker_boot` again. +5. `perform` runs on a new operation instance. + - An `Input` copies itself onto the slot's scratch the first time that the operation asks for its + `path`. An operation that reads the descriptor directly doesn't pay for the copy. + - The worker posts outputs back through their descriptors and flushes them before it reports success. +6. The worker answers with one JSON line: `ok` with the result and the timing, or a failure with its code. +7. The supervisor enforces the deadline from outside the worker, because a thread inside a C extension + can't be interrupted from within. The supervisor kills a worker that is past its deadline as a process + group. The supervisor is then the only process that holds the connection, and it answers `killed` with + the cause. +8. The client raises the exception class that is registered for the failure's side of the permanent + split. On success and on failure, the client publishes a `perform.hot_cell` notification. See + [Response codes](codes.md) and [Observability](observability.md#per-call-notification). diff --git a/docs/scratch.md b/docs/scratch.md new file mode 100644 index 0000000..78d71be --- /dev/null +++ b/docs/scratch.md @@ -0,0 +1,183 @@ +--- +type: Reference +title: "Scratch" +order: 7 +description: "Where a cell stages files: the tmpfs, named volume, and host-mount layouts, and the boot sweep." +sources: + - hotcell-server/lib/hot_cell/supervisor.rb + - hotcell-server/lib/hot_cell/filesystem.rb + - hotcell-server/lib/hot_cell/slot.rb + - hotcell-server/lib/hot_cell/sweeper.rb +--- + +# Scratch + +This page describes the cell's scratch, the filesystem where requests stage their files, and the layouts +that you can choose for it: a tmpfs, a named volume, or a host-mounted filesystem. + +Scratch is `/tmp`. It holds staged inputs, staged outputs, and every intermediate file that a tool writes. +Each request's directory is under `HOTCELL_WORKSPACE`, which defaults to a directory under `TMPDIR`. See +[Cell settings](cell-settings.md#environment-variables). + +## Why the layout matters + +The README's accessory puts scratch on a RAM-backed tmpfs. That couples three settings: an operation's +`file_size`, the container's tmpfs `size=`, and the container's `memory`. Raising `file_size` requires +raising the other two, which can put memory pressure on the host. + +Pages on the tmpfs are charged to the container's cgroup. With `memory-swap` equal to `memory`, the kernel +can't reclaim or swap them, so the tmpfs `size=` and the container's `memory` grow together, byte for byte. +Every time that `file_size` goes up to admit a larger conversion, `memory` pays for it. + +Moving scratch onto disk breaks that coupling. Files written there pass through the page cache, and the +kernel writes those pages back and drops them under pressure. A large file on scratch can then no longer +run the cell out of memory, and `memory` sizes from `concurrency × peak RSS` alone, kept above the cell's +own `memory` limit. + +## Layouts + +| Layout | Backed by | Size cap | `nosuid,nodev,noexec` | Provisioning | +| --- | --- | --- | --- | --- | +| tmpfs | RAM, charged to the cgroup | the `size=` option | yes | none | +| named volume | disk, under `/var/lib/docker/volumes` | none | none | none | +| host mount | disk | the filesystem's size | from the host mount | each host | + +Both disk layouts give up speed, and speed is rarely the bottleneck: on NVMe, for spool-and-process +pipelines, the descriptor design already keeps the largest files off scratch entirely. Both disk layouts +also outlive the container, so cleanup becomes the cell's job rather than the mount's. See +[Boot sweep](#boot-sweep). + +### tmpfs + +Use a tmpfs when scratch is small and stays small. It's as fast as RAM, it disappears when the container +stops, and one flag carries the size cap and all three security flags. It's the only layout where Docker +sets those flags for you. The README's accessory uses it: + +```yaml +tmpfs: /tmp:rw,nosuid,nodev,noexec,size=512m +``` + +### Named volume + +A named volume moves scratch onto disk and needs nothing on the host. Replace the `tmpfs:` option with a +named volume on the same path: + +```yaml +# config/deploy.yml — the cell, changed from the README's accessory +accessories: + active_storage: + volumes: + - hotcell-sockets:/run/hotcell/cell # unchanged + - hotcell-scratch:/tmp # scratch, on disk + + options: + memory: 2g # no tmpfs term: concurrency × peak RSS + memory-swap: 2g # equal to memory + # tmpfs: gone. /tmp is the named volume above. +``` + +Docker creates the volume on first boot and initializes it from the image's own `/tmp`, both the content +and the permissions, which are `1777` on the Debian base. So the cell's uid can write to it with no host +work: no `mkdir`, no `chown`, and no fstab line. The socket volume depends on the same rule, and the +installed `Dockerfile` records it. See [experiment 4](contributing/experiments.md). + +You give up the size cap and the security flags, because Docker can set neither on a named volume. A +runaway write is then bounded only by the `file_size` limit on each file and by deadline × disk +throughput, and a fill lands on whatever filesystem holds `/var/lib/docker`, usually the root filesystem. +The `isolation` operation reports `scratch_noexec: false` against a cell configured this way, which is +true. + +### Host-mounted filesystem + +A host-mounted filesystem moves scratch onto disk and keeps the size cap and the security flags. Give the +accessory an absolute host path, which Kamal passes to `docker run -v` as a bind mount: + +```yaml +# config/deploy.yml — the cell, changed from the README's accessory +accessories: + active_storage: + volumes: + - hotcell-sockets:/run/hotcell/cell # unchanged + - /var/lib/hotcell-scratch:/tmp # scratch, from the host + + options: + memory: 2g # no tmpfs term: concurrency × peak RSS + memory-swap: 2g # equal to memory + # tmpfs: gone. /tmp is the bind mount above, and its nosuid,nodev,noexec + # must come from the host mount. +``` + +Give it a dedicated filesystem: a partition, a logical volume, or a loopback image file. Its size is then +the cap, so a fill stays inside scratch. A write that fails in staging or writeback raises +`Errno::ENOSPC`, which the cell classifies as `failed`: transient, retried, and never recorded against a +blob. A write that fails inside libvips gets `unreadable`, which is permanent. See +[Observability](observability.md#recommended-alerts). The security flags come +from the host mount options, and a bind mount carries its source's flags into the container, so +`scratch_noexec` still reports true. + +A plain host directory isn't worth it. It gives up the cap and the flags exactly as a named volume does, +and adds the `chown` work that a named volume avoids. + +Chown the source. A bind mount keeps the host directory's ownership rather than taking the image's, and +the cell runs as `10001:10001`. Docker creates a missing source owned by root, and the cell then fails +every request with `EACCES`. Create and chown the source before the first boot. For example, as a loopback +image, with no repartitioning, sized from `concurrency` times the most scratch that one request holds, plus headroom: + +``` +fallocate -l 8G /var/lib/hotcell-scratch.img +mkfs.ext4 /var/lib/hotcell-scratch.img +mkdir -p /var/lib/hotcell-scratch +echo '/var/lib/hotcell-scratch.img /var/lib/hotcell-scratch ext4 loop,nosuid,nodev,noexec 0 0' >> /etc/fstab +mount /var/lib/hotcell-scratch +chown 10001:10001 /var/lib/hotcell-scratch +``` + +**Caution:** Don't use the host's `/tmp` as the source. On most systemd distributions, it's itself a +tmpfs, which puts scratch back in RAM, still charged to the cell's cgroup because the writer pays. And +`systemd-tmpfiles` deletes old files out from under long requests. + +Check the source before you use it: + +``` +findmnt -T /var/lib/hotcell-scratch -o TARGET,SOURCE,FSTYPE,OPTIONS +``` + +`FSTYPE` must be a disk filesystem, and `OPTIONS` shows which of the three flags the cell gets. + +## Boot sweep + +At boot, the cell empties `TMPDIR` and the workspace's parent of every entry that its uid owns, except +`lost+found`. Rebooting the accessory therefore clears a scratch that a killed tool filled. A removal that +fails is logged as `scratch.unswept` and doesn't stop the boot. + +The cell refuses to boot if either directory: + +- Is missing. +- Is reached through a symlink that its uid owns. +- Holds `HOTCELL_DIR`. + +The rule is the same in every layout, so a cell's `TMPDIR` must be a directory that nothing else uses. In +the container, `/tmp` is the cell's own. On a developer's machine, it's everyone's, so boot an +uncontainerized cell with `hotcell --development`, which never sweeps the directory that it's given. Its +scratch is a directory of its own beneath that directory, named for `HOTCELL_DIR`. An explicit +`HOTCELL_WORKSPACE` has its parent swept wherever it points, with or without the flag. See +[Development mode](cell-settings.md#development-mode). + +## What changes in the numbers + +The scratch constraint in [Tuning](tuning.md#constraints-between-the-numbers) applies to every layout: + +- On a tmpfs, the number to fit is the tmpfs `size=`, and `memory` includes the tmpfs. +- On either disk layout, `memory` loses its tmpfs term. On a host mount, the number to fit is the host + filesystem's size. On a named volume, there's no cap to fit inside. + +`file_size` limits each file, not the total that a request writes, so a capped filesystem is the only +bound on what a runaway request writes across many files. Without a cap, the bound is deadline × disk +throughput. A capped filesystem is also what makes a generous `file_size` safe to run. + +Two things don't change on any layout: + +- `HOTCELL_WORKSPACE` keeps its default. It's under `Dir.tmpdir`, and the server treats scratch as a plain + directory without checking the filesystem type. +- A full scratch reaches the caller as a full tmpfs does: `failed` when staging or writeback fails, and + `unreadable` when a write inside libvips fails. diff --git a/docs/tuning.md b/docs/tuning.md new file mode 100644 index 0000000..14504b8 --- /dev/null +++ b/docs/tuning.md @@ -0,0 +1,190 @@ +--- +type: Reference +title: "Tuning" +order: 9 +description: "Which measurement sets each cell number, the constraints between the numbers, and how to use bin/load." +sources: + - hotcell-server/lib/hot_cell/configuration.rb + - hotcell-client/lib/hot_cell/cell.rb + - bin/load + - examples/load +--- + +# Tuning + +This page describes how to choose a cell's numbers for your workload: which measurement sets each one, +the constraints between them, and the order in which to change them. For what each setting does, see +[Cell settings](cell-settings.md) and [Container](container.md). + +In short: ship the defaults, instrument first, and tighten in one direction only. + +## Instrument before you tune + +Before you change any number, collect both of the following signals. They're independent of each other. + +- The `perform.hot_cell` notification, from the application. See + [Per-call notification](observability.md#per-call-notification). +- The cell's `metrics`, from its control socket. See [Cell metrics](observability.md#cell-metrics). + +For which signal means what, see [What to watch](observability.md#what-to-watch). + +## Which measurement sets each number + +| Setting | Where the number comes from | +| --- | --- | +| `memory` | One worker's peak on your largest real inputs. | +| `file_size` | The largest thing that one operation writes, plus any input that it stages. | +| `deadline` | The tail of `perform_ms` on real traffic. | +| `concurrency` | A load test, against `cpus`. | +| `queue_size`, `queue_wait` | A load test, against what the caller can wait for. | +| Container `memory`, `cpus`, tmpfs size | `concurrency` times the above. | + +Only the scheduling numbers need contention to measure. The rest come from single requests, and a load +test tells you nothing useful about them. + +## Start generous on `memory` and `file_size` + +`memory` and `file_size` are the limits where a wrong value is expensive to undo: + +- **A limit that's too high costs headroom.** The container's own memory limit still bounds the cell. +- **A limit that's too low kills a legitimate file.** The cell answers `killed` with cause `memory` or + `fsize`. Both verdicts are permanent, so the caller is entitled to record them, and the shipped Active + Storage analyzers do. See [What Active Storage records](codes.md#what-active-storage-records). + +The two mistakes aren't equal, so use this order: + +1. Set `memory` and `file_size` to the defaults. +2. Run real traffic for at least a week. +3. Read the peak that your operations reach. +4. Lower each limit to a value above that peak. + +Don't use the opposite order: don't set a low limit and raise it when files fail. Every failure in that +period is a durable record against a customer's file. + +Step 3 has a gap. The cell's metrics report `killed_by`, which counts failures. A count of zero tells you +that the limit is high enough, and doesn't tell you how much room is left. To find the room, measure the +operation outside the cell: run it on your largest inputs and read its peak. + +`deadline` needs none of this care. A `deadline` kill is transient, so a value that's too low costs a +retry. + +## Make the timeouts agree + +The defaults don't agree with each other, and this is the first thing that you'll hit. + +A cell reports `answer_within`, which is `queue_wait + deadline + 1`. On the defaults, that's 71 seconds. +The client's default `timeout` is 30 seconds. + +When the client's `timeout` is below `answer_within`, a saturated cell reaches the caller as a transport +failure instead of as `capacity` or `killed`. That loses exactly the signal that you need to size +anything. The client warns at boot when its timeout doesn't clear the cell's number, and `describe` +reports `answer_within`. + +Which way to fix it depends on the caller: + +- **A background job** wants a loose timeout, above `answer_within`, so that it receives the cell's + verdict and can act on it. Active Storage's analysis, preview, and variant work all runs in jobs. +- **A synchronous request** wants the opposite: a short `deadline` on the cell and a tight timeout on the + client, because a thread held for a minute is a thread that isn't serving traffic. + +Both outcomes are transient, so neither choice misclassifies anything. That's the only reason that this +is safe to decide for each caller. + +## Constraints between the numbers + +Nothing checks these constraints for you, except where noted. + +- The client's `timeout` must be more than the cell's `answer_within`. See + [Make the timeouts agree](#make-the-timeouts-agree). The client warns at boot. +- The cell's `memory` must be less than the container's `memory`. At equal values, the cgroup fires first, + and a cgroup kill is a `SIGKILL` with no diagnostic. +- `concurrency × the most scratch that one request holds` must be no more than the scratch. `file_size` + limits each file, not their sum, and one request can hold a staged input, an output, and a tool's + intermediate files at once. Above it, concurrent workers fill the scratch, and requests fail with + `ENOSPC` instead of with a limit verdict. On the default accessory, the + scratch is the tmpfs, and its `size=` is the number to fit. See [Scratch](scratch.md#what-changes-in-the-numbers). +- `concurrency × (MAGICK_DISK_LIMIT + everything else that one worker writes on scratch)` must be no more + than the scratch. Above it, concurrent ImageMagick processes fill the scratch before any of them refuses + a frame. `file_size` can't prevent that, because it bounds each cache file and not their sum. See + [ImageMagick](imagemagick.md). +- `pids-limit` must clear `concurrency` plus the threads and subprocesses that one toolchain starts. + +## Size against the container + +These numbers are arithmetic against the container flags. For the README's accessory (`cpus: 2`, +`memory: 2g`, `size=512m`): + +- `concurrency: 4`, because a request spends much of its life off the CPU, so twice `cpus` is where to + start. +- `file_size: 48MB`, because that bounds what one worker writes. +- `memory: 1536MB`, because that's the measured working value for `RLIMIT_DATA`. + +Size the cell to its most demanding operation. The cell clamps each operation's own `limits` to its own, +so the cell's numbers only ever take away. Read the `limits` that each operation you carry declares, and +set the cell above the highest of them. The shipped video previewer asks for `deadline: 120` and +`file_size: 128MB`. A cell configured with 30 seconds and 48MB kills each video preview that runs longer than +30 seconds or writes a file larger than 48MB, and nothing else, which is a hard failure to place. See [Active Storage operations](active-storage.md#limits). + +### Sizing guidelines + +- **`memory` doesn't multiply by `concurrency`.** It's an address-space charge on one worker. About 620MB + of it is reserved and never touched, and about 450MB of that is Ruby's own reservation, which + `RLIMIT_DATA` charges in full. Subtract 450MB before you read `memory` as the amount that an input can + consume. The cgroup limit is what bounds real memory across the cell. See + [experiments 11 and 12](contributing/experiments.md). +- **An input is charged only when an operation asks for its path.** A descriptor that an operation reads in + place costs no tmpfs and no `file_size`, so a multi-gigabyte upload can be analyzed under a small + `file_size`. An operation that needs a filename copies the input onto scratch first, and the kernel + charges that copy exactly as it charges an output. Size `file_size` from the largest thing that your + operations write, and count a staged input as one of them. +- **A large supervisor makes every request slower.** A worker's fixed cost is copy-on-write settling, and + it's proportional to the supervisor's resident heap. A `before_fork` that requires more than the cell's + own operations need is paid on every request for the life of the deployment. See + [Cell overhead](contributing/overhead.md). + +## Check the queue with `bin/load` + +`bin/load` answers one question: does the queue behave the way that you configured it? It shows that +`queue_size` and `queue_wait` produce `capacity` where you expect, and it finds the knee for a known +service time. + +``` +bin/load IMAGE [SCENARIO] [SECONDS] [THREADS] +``` + +It runs a cell in a container and drives it from a second container. It reports throughput, the verdict +breakdown, and latency split into time queued and time performing. That split separates saturation from +slowness: queued time that grows while perform time stays flat means that the cell needs more workers. + +It can't give you your own numbers. It drives the example operations rather than yours, and it fixes +`cpus`, `memory`, the tmpfs size, and `pids-limit`. Only the cell settings vary, through the following +environment variables: + +- `EXAMPLE_CONCURRENCY` +- `EXAMPLE_QUEUE_SIZE` +- `EXAMPLE_QUEUE_WAIT` +- `EXAMPLE_DEADLINE` +- `EXAMPLE_MEMORY_MB` +- `EXAMPLE_FILE_SIZE_MB` + +Treat it as a check on the scheduling, not as a capacity plan. + +## A starting point + +Deploy the documented defaults unchanged, with `max_requests_per_worker: 1`. Raise the client's timeout +above `answer_within` for any job path. Then let it run for a week and read the signals in +[What to watch](observability.md#what-to-watch). + +`max_requests_per_worker: 1` is the right default to start from, because it's the isolating one. Raise it +only with a measurement that says the copy-on-write cost matters for your workload, and read +[ADR 0001](../adr/0001-reuse-workers-across-requests.md) first: it costs an isolation property, not just +memory. + +## Don't tune against staging + +A cell's behavior depends on the files that it's given. A staging or beta destination with synthetic +uploads won't predict production. The measurements that matter here were all taken on the deployed image +for that reason. See [ADR 0001](../adr/0001-reuse-workers-across-requests.md). + +Tune against production traffic with generous limits. That direction fails slow rather than failing +closed. diff --git a/examples/gate b/examples/gate index 556239e..0f18691 100755 --- a/examples/gate +++ b/examples/gate @@ -12,7 +12,7 @@ # rejects a non-empty capability set, a cleared no-new-privileges bit, a root uid, and — crucially — # a nil for any of them, the shape a cell that cannot report a flag would return. # -# 2. bin/conformance and bin/load must supply no `--ulimit stack`. docs/DEPLOYMENT.md forbids it: a +# 2. bin/conformance and bin/load must supply no `--ulimit stack`. docs/container.md forbids it: a # lowered stack turns an overflow into a SIGSEGV the cell reports as killed/crashed, a transient # verdict the caller retries against a limit that will fail it again, so a helper that set it would # measure a shape operators are told not to deploy. diff --git a/hotcell-client/lib/hot_cell/client.rb b/hotcell-client/lib/hot_cell/client.rb index b9c742f..aa69658 100644 --- a/hotcell-client/lib/hot_cell/client.rb +++ b/hotcell-client/lib/hot_cell/client.rb @@ -126,7 +126,7 @@ def wrap(inputs, outputs) # premise is that restoring is worse than carrying it: the cell may still hold the descriptor, undoing # it mid-request adds a failure path to the answer, and a caller that could not share the file could # not use this at all. Active Storage hands over tempfiles it then unlinks, so nothing survives there. - # docs/DEPLOYMENT.md tells anyone writing their own client to pass files they are willing to share. + # docs/client-api.md tells anyone writing their own client to pass files they are willing to share. def shared(io, mode) return io if HotCell.group.nil? diff --git a/hotcell-client/test/describe_survival_test.rb b/hotcell-client/test/describe_survival_test.rb index 25d72e1..0b1ca78 100644 --- a/hotcell-client/test/describe_survival_test.rb +++ b/hotcell-client/test/describe_survival_test.rb @@ -4,7 +4,7 @@ # `describe` is the one call a cell answers before the application is serving, and the README puts it in # `after_initialize`, where an exception is not a failed check but a Rails application that does not boot. -# The process answering runs untrusted content, and `docs/DESIGN.md` records the socket theft that lets a +# The process answering runs untrusted content, and `docs/design/worker-isolation.md` records the socket theft that lets a # compromised worker be the thing that answers — so nothing it sends may raise out of `describe`. class DescribeSurvivalTest < HotCellClientTest # Correctly framed responses whose contents this client cannot use. Each raised out of `describe` before diff --git a/hotcell-core/lib/hot_cell/codes.rb b/hotcell-core/lib/hot_cell/codes.rb index 33928ea..350576f 100644 --- a/hotcell-core/lib/hot_cell/codes.rb +++ b/hotcell-core/lib/hot_cell/codes.rb @@ -89,7 +89,7 @@ module Codes # It removes the supervisor as an instrument: a sibling's signal no longer travels through a wait status # into someone else's blob. It does nothing about a cell that has been compromised outright and answers # a connection itself — `from_wire` believes a `permanent` boolean off the wire, and a worker that stole - # `work.sock` writes whatever it likes. That is the socket-theft residual `docs/DESIGN.md` records, and + # `work.sock` writes whatever it likes. That is the socket-theft residual `docs/design/worker-isolation.md` records, and # it is not closed here. PERMANENT_BY_CAUSE = { FSIZE => true, diff --git a/hotcell-core/test/docs_test.rb b/hotcell-core/test/docs_test.rb new file mode 100644 index 0000000..1c440d4 --- /dev/null +++ b/hotcell-core/test/docs_test.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +require "test_helper" + +# The reference pages describe behavior, and these fail when a table there stops matching the code. +class DocsTest < HotCellTest + DOCS = File.expand_path("../../docs", __dir__) + + def test_the_codes_table_gives_each_code_its_split + documented = table("codes.md", "Code").reject { |code, *| code == "`killed`" }.to_h do |code, split, *| + [ unquote(code), split == "Permanent" ] + end + + assert_equal HotCell::Codes::PERMANENT.sort.to_h, documented.sort.to_h + end + + def test_the_causes_table_gives_each_kill_cause_its_split + documented = table("codes.md", "Cause").to_h { |cause, split, *| [ unquote(cause), split == "Permanent" ] } + + assert_equal HotCell::Codes::PERMANENT_BY_CAUSE.sort.to_h, documented.sort.to_h + end + + private + def unquote(cell) + cell.delete_prefix("`").delete_suffix("`") + end + + def table(page, first_header) + lines = File.readlines(File.join(DOCS, page), chomp: true) + header = lines.index { |line| line.start_with?("| #{first_header} |") } + flunk "#{page} has no table headed #{first_header.inspect}" if header.nil? + + lines.drop(header + 2).take_while { |line| line.start_with?("|") }.map do |line| + line.delete_prefix("|").delete_suffix("|").split(" | ").map(&:strip) + end + end +end diff --git a/hotcell-server/lib/hot_cell/log.rb b/hotcell-server/lib/hot_cell/log.rb index ecd6ec3..84d69ac 100644 --- a/hotcell-server/lib/hot_cell/log.rb +++ b/hotcell-server/lib/hot_cell/log.rb @@ -12,7 +12,7 @@ module HotCell # This is the channel for everything no response can carry: deadline kills, reaps that found a signal, # boot checks, and queue high-water. # - # Lines follow docs/LOGS.md: ECS field names, domain fields under the hotcell namespace. The fleet's + # Lines follow docs/observability.md: ECS field names, domain fields under the hotcell namespace. The fleet's # collector routes on service.name, takes the record timestamp from @timestamp, and severity from # log.level, so those three are load-bearing: renaming any of them silently drops or mislabels every # cell log line in production. diff --git a/hotcell-server/lib/hot_cell/operation.rb b/hotcell-server/lib/hot_cell/operation.rb index 1dcf74e..737bd70 100644 --- a/hotcell-server/lib/hot_cell/operation.rb +++ b/hotcell-server/lib/hot_cell/operation.rb @@ -57,7 +57,7 @@ def operation_name # class's own declaration, or of the nearest ancestor's when this class has none yet. That is what # lets a subclass narrow a single number, and what lets an operator give a shipped operation a # different budget from an operations file, after the operation loads, without editing the gem. The - # cell's own limits still clamp whatever is declared. docs/DEPLOYMENT.md, "Changing a shipped + # cell's own limits still clamp whatever is declared. docs/operation-api.md, "Changing a shipped # operation's limits". def limits(**values) return inherited_value(:@limits) || Limits.new if values.empty? diff --git a/hotcell-server/lib/hot_cell/slot.rb b/hotcell-server/lib/hot_cell/slot.rb index 9ae867f..9846b09 100644 --- a/hotcell-server/lib/hot_cell/slot.rb +++ b/hotcell-server/lib/hot_cell/slot.rb @@ -37,7 +37,7 @@ module HotCell # uid and `0700` is the owner's own mode, so a concurrent sibling — or a `setsid` descendant of a finished # request, which process groups do not contain — can still write into a home the moment it exists. The # slot directory itself can be renamed aside and replaced, and `chmod` follows what it finds. Both are the - # residuals `docs/DESIGN.md` records under worker isolation, and neither is closed here. + # residuals `docs/design/worker-isolation.md` records, and neither is closed here. # # There is one directory per request and not two. A request's staged inputs and outputs are named inside # `$HOME` rather than in a scratch directory of their own, because the two had the same lifetime and the diff --git a/hotcell-server/lib/hot_cell/supervisor.rb b/hotcell-server/lib/hot_cell/supervisor.rb index 29d8e61..312283b 100644 --- a/hotcell-server/lib/hot_cell/supervisor.rb +++ b/hotcell-server/lib/hot_cell/supervisor.rb @@ -281,7 +281,7 @@ def handle(source) # `Codes` already relies on when it refuses to attribute a signal to the input a worker was holding. # The premise is that handling TERM is not optional — it is how an orchestrator stops a cell without # killing requests in flight — and that denial of service against a cell is out of scope per - # docs/DESIGN.md. + # docs/design/threat-model.md. def trap_signals @signals, @signal_writer = IO.pipe trap("CHLD") { @signal_writer.write_nonblock "C", exception: false } @@ -527,9 +527,9 @@ def become_worker(supervisor_side, stderr_reader, stderr_writer) # capability and so survives `cap-drop ALL`, and neither the deadline kill nor the reap sweep reaches # it afterwards. It then runs untimed until `pids-limit` or the cgroup ends it. # - # It reaches further than that, so this is not the denial of service docs/DESIGN.md puts out of + # It reaches further than that, so this is not the denial of service docs/design/threat-model.md puts out of # scope. A stolen listener needs a live process to hold it, and this escape is the only way one - # outlives the reap — so it is what turns the socket theft under "Worker isolation" from a dead + # outlives the reap — so it is what turns the socket theft in docs/design/worker-isolation.md from a dead # socket path into a cell that intercepts every later request. # # The premise is that nothing in this process prevents it. A process group is the only bound the @@ -546,7 +546,7 @@ def become_worker(supervisor_side, stderr_reader, stderr_writer) # for it: a warning written from inside libvips is a `write(2)` in a C call Ruby cannot interrupt, so # the conversion would wait on the supervisor's scheduling — nearest exactly when the host is under # pressure and the supervisor is scheduled least. That trade is refused; the conversion path must - # never wait on the supervisor. What non-blocking loses instead is in docs/LOGS.md, and a test pins + # never wait on the supervisor. What non-blocking loses instead is in docs/observability.md, and a test pins # the flag so that a well-meaning fix has to argue with it. stderr_reader.close stderr_writer.fcntl Fcntl::F_SETFL, stderr_writer.fcntl(Fcntl::F_GETFL) | Fcntl::O_NONBLOCK diff --git a/hotcell-server/test/docs_test.rb b/hotcell-server/test/docs_test.rb new file mode 100644 index 0000000..ca5b1e7 --- /dev/null +++ b/hotcell-server/test/docs_test.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require "test_helper" + +# The reference pages describe behavior, and these fail when a table there stops matching the code. +class DocsTest < HotCellServerTest + DOCS = File.expand_path("../../docs", __dir__) + + def test_the_event_table_lists_every_event_at_its_level + documented = table("observability.md", "`event.action`", under: "Events").to_h { |event, level, *| [ unquote(event), level ] } + + assert_equal HotCell::Log::LEVELS.sort.to_h, documented.sort.to_h + end + + def test_the_cell_settings_tables_carry_the_defaults + scheduling = defaults_in("Scheduling settings") + limits = defaults_in("Request limits") + + assert_equal HotCell::Configuration::SCHEDULING, scheduling + assert_equal HotCell::Configuration::LIMITS, limits + end + + private + def defaults_in(heading) + table("cell-settings.md", "Setting", under: heading).to_h do |setting, default, *| + [ unquote(setting).to_sym, number(unquote(default)) ] + end + end + + def number(text) + text.end_with?("MB") ? Integer(text.delete_suffix("MB")) * 1024**2 : Integer(text) + end + + def unquote(cell) + cell.delete_prefix("`").delete_suffix("`") + end + + # The rows of the first table whose header starts with `first_header`, after the `under` heading if one is + # given. + def table(page, first_header, under: nil) + lines = File.readlines(File.join(DOCS, page), chomp: true) + lines = lines.drop_while { |line| !line.match?(/\A#+ #{Regexp.escape(under)}\z/) } if under + header = lines.index { |line| line.start_with?("| #{first_header} |") } + flunk "#{page} has no table headed #{first_header.inspect}" if header.nil? + + lines.drop(header + 2).take_while { |line| line.start_with?("|") }.map do |line| + line.delete_prefix("|").delete_suffix("|").split(" | ").map(&:strip) + end + end +end diff --git a/hotcell-server/test/log_test.rb b/hotcell-server/test/log_test.rb index 86ed9dc..c0b75d9 100644 --- a/hotcell-server/test/log_test.rb +++ b/hotcell-server/test/log_test.rb @@ -6,7 +6,7 @@ # The log sink is a pipe to the container runtime, and the supervisor writes to it inside the loop that # enforces every request's deadline. A runtime that stops draining must not be able to park that loop. # -# The line format is the schema in docs/LOGS.md. The fleet's log collector routes on service.name, +# The line format is the schema in docs/observability.md. The fleet's log collector routes on service.name, # takes the record timestamp from @timestamp, and severity from log.level, so those three are # load-bearing: a rename silently drops or mislabels every cell log line in production. class LogTest < HotCellServerTest diff --git a/rakelib/docs.rake b/rakelib/docs.rake new file mode 100644 index 0000000..c2e6522 --- /dev/null +++ b/rakelib/docs.rake @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +# The reference pages under docs/ carry OKF frontmatter (https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md): +# a `type`, a `title`, a one-line `description`, and the `sources` a page describes. The indexes are generated +# from the descriptions, and `sources` is what lets a change to the code name the page it may have made wrong. +module HotCellDocs + ROOT = "docs" + INDEX = "index.md" + MARKERS = /(\n).*?()/m + + Page = Struct.new(:path, :frontmatter) do + def title = frontmatter["title"] + def description = frontmatter["description"] + def sources = Array(frontmatter["sources"]) + def order = frontmatter["order"].is_a?(Integer) ? frontmatter["order"] : Float::INFINITY + + def describes?(file) + sources.any? { |source| file == source || file.start_with?("#{source.chomp("/")}/") } + end + end + + class << self + def pages + Dir[File.join(ROOT, "**", "*.md")].sort.reject { |path| File.basename(path) == INDEX }.map do |path| + Page.new(path, frontmatter(path)) + end + end + + def frontmatter(path) + text = File.read(path) + return {} unless text.start_with?("---\n") + + require "yaml" + YAML.safe_load(text[4...text.index("\n---\n", 4)]) || {} + end + + def indexes + Dir[File.join(ROOT, "**", INDEX)].sort + end + + # Each index lists the pages in its own directory. A page with an `order` comes first, in that order; the rest + # follow by path. + def listing(index) + directory = File.dirname(index) + own_pages = pages.select { |page| File.dirname(page.path) == directory } + .sort_by.with_index { |page, position| [ page.order, position ] } + rows = own_pages.map do |page| + "| [#{page.title}](#{page.path.delete_prefix("#{directory}/")}) | #{page.description} |" + end + + [ "| Page | Description |", "| --- | --- |", *rows ].join("\n") + end + + def regenerated(index) + File.read(index).sub(MARKERS) { "#{$1}\n#{listing(index)}\n\n#{$2}" } + end + + def problems + pages.flat_map { |page| page_problems(page) } + indexes.filter_map { |index| index_problem(index) } + end + + def page_problems(page) + %w[ type title description ].reject { |key| page.frontmatter[key].is_a?(String) && !page.frontmatter[key].empty? } + .map { |key| "#{page.path}: frontmatter has no #{key}" } + + (page.frontmatter.fetch("order", 0).is_a?(Integer) ? [] : [ "#{page.path}: order is not an integer" ]) + + page.sources.reject { |source| File.exist?(source) }.map { |source| "#{page.path}: source #{source} does not exist" } + end + + def index_problem(index) + if !File.read(index).match?(MARKERS) + "#{index}: no index markers" + elsif File.read(index) != regenerated(index) + "#{index}: out of date; run `rake docs:index`" + end + end + + # Pages whose sources changed since `base` while the page itself did not. + def stale(base) + changed = IO.popen([ "git", "diff", "--name-only", "#{base}...HEAD" ], &:read).lines(chomp: true) + raise "git diff against #{base} failed" unless $?.success? + + pages.reject { |page| changed.include?(page.path) }.filter_map do |page| + touched = changed.select { |file| page.describes?(file) } + [ page, touched ] if touched.any? + end + end + end +end + +namespace "docs" do + desc "Regenerate the page lists in docs/index.md and every other index.md under docs/" + task :index do + HotCellDocs.indexes.each { |index| File.write index, HotCellDocs.regenerated(index) } + end + + desc "Check the docs' frontmatter, that every source exists, and that the indexes are current" + task :check do + problems = HotCellDocs.problems + abort problems.join("\n") if problems.any? + + puts "docs: #{HotCellDocs.pages.size} pages, frontmatter, sources and indexes in order" + end + + # A reminder rather than a gate: most changes to a source do not change what its page says. + desc "List docs whose sources changed since BASE (default origin/master) while the doc did not" + task :stale do + base = ENV.fetch("BASE", "origin/master") + + HotCellDocs.stale(base).each do |page, touched| + message = "#{touched.join(", ")} changed; check that #{page.path} still describes it" + puts ENV["GITHUB_ACTIONS"] ? "::warning file=#{page.path}::#{message}" : "#{page.path}: #{message}" + end + end +end