diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 7bfa79c..ea35744 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -3,8 +3,8 @@ `ci.yml` runs on pull requests and pushes without production secrets. It covers backend tests, race tests, `go vet`, frontend lint/build, Docker image builds, Compose validation, and the deployment script test suite -(`deploy/scripts/deploy_scripts_test.sh`). That suite stubs `docker`, `curl`, and -`nginx` on `PATH`, so it needs no daemon or privileges — it exercises slot +(`deploy/scripts/deploy_scripts_test.sh`). That suite stubs `docker`, `curl` and +`nginx` on `PATH`, so it needs no daemon or privileges: it exercises slot selection, the transactional Nginx switch and its restore-on-failure paths, and the preflight checks. @@ -27,15 +27,14 @@ image SHA. `rollback.yml` is manual (`workflow_dispatch`) and switches Nginx back to the other slot, which is already running the previous release. It pulls no images, so it holds no `packages` permission. It shares the `delta-production-deploy` -concurrency group with `deploy.yml` so a rollback can never interleave with a +concurrency group with `deploy.yml`, so a rollback can never interleave with a deployment. `slot: auto` targets whichever slot is currently inactive; `blue` or -`green` names one explicitly. To recover an *older* image SHA instead, run -`deploy.yml` manually with that SHA — rollback only moves traffic between the two -slots that are already up. +`green` names one explicitly. Rollback only moves traffic between the two slots +already up; to recover an *older* image SHA, run `deploy.yml` manually with it. `pages.yml` publishes the committed Swagger spec as a static Swagger UI site on -GitHub Pages (). It is independent of -the CI -> Publish -> Deploy chain: it holds no `packages` permission, touches no -slot, and never runs on the self-hosted runner. Swagger UI's assets are vendored +GitHub Pages (). It is independent +of the CI -> Publish -> Deploy chain: no `packages` permission, no slot, never on +the self-hosted runner. Swagger UI's assets are vendored into the artifact at build time from a pinned `swagger-ui-dist`, so the published page loads nothing from a third-party CDN at runtime. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03c10d8..da47d69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,7 @@ jobs: run: npm run build # The Docker build regenerates the Swagger docs from the handler annotations, - # so the binary always serves a current spec -- but server/docs is also + # so the binary always serves a current spec, but server/docs is also # committed, and the Pages site publishes that committed copy. Nothing forced # the two to agree, so annotations could change without a follow-up # `swag init` and the published reference would drift from the real API with @@ -121,8 +121,8 @@ jobs: run: deploy/scripts/deploy_scripts_test.sh # The Python in scripts/ was previously untested. A full reseed cannot run - # here -- it needs the WordPress export zip, which is not in the repo, and a - # local Docker stack -- so this covers the parts that can fail silently: the + # here: it needs the WordPress export zip, which is not in the repo, and a + # local Docker stack. This covers the parts that can fail silently: the # scripts compiling at all, the seed generator's id=0 guarantee, and the # reseed script's refusal to destroy data unattended. scripts: @@ -142,7 +142,7 @@ jobs: # The non-TTY guard is the only thing standing between an automated # context and a destroyed database, so assert it stays a guard: this must # FAIL to proceed. Without a Docker daemon the run would abort in - # preflight anyway, so accept either refusal -- what is not acceptable is + # preflight anyway, so accept either refusal. What is not acceptable is # a clean exit, which would mean it sailed past the confirmation. - name: Refuses to destroy data unattended run: | @@ -171,7 +171,7 @@ jobs: fi # A mention of the mode in a comment must not be mistaken for setting - # it -- that regression would silently skip the preamble. + # it; that regression would silently skip the preamble. printf -- "-- NO_AUTO_VALUE_ON_ZERO handled upstream\nINSERT INTO articles (\`author_ids\`, \`comment_status\`) VALUES ('y','open');\n" > "$etl/articles.sql" python ./scripts/generate_wordpress_sql.py "$etl" "$out" if ! head -1 "$out/02-articles.sql" | grep -q '^SET sql_mode'; then diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index dee3df2..557be08 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -109,7 +109,7 @@ jobs: run: echo "value=$(git rev-parse HEAD:embeddings)" >> "$GITHUB_OUTPUT" # This image bakes in ~130MB of model weights, so building it is by far the - # slowest step here -- and it was rebuilt on every push to main, including + # slowest step here, and it was rebuilt on every push to main, including # the overwhelming majority that touch nothing but Go or TypeScript. A tag # that already exists is by definition built from identical content. - name: Check whether the embeddings image already exists diff --git a/.gitignore b/.gitignore index a8a1e69..cd634df 100644 --- a/.gitignore +++ b/.gitignore @@ -19,5 +19,5 @@ deploy/*.env ..env.un~ # Local DB dumps (contain cms_users password hashes and cms_sessions). -# Kept here so Borg backs them up -- /var/lib/docker volumes are NOT backed up. +# Kept here so Borg backs them up; /var/lib/docker volumes are NOT backed up. db-backups/ diff --git a/README.md b/README.md index 93c1933..7f88624 100644 --- a/README.md +++ b/README.md @@ -1,164 +1,99 @@ # Triangle CMS (Delta) -Headless CMS replacement for The Triangle, with: +Headless CMS replacement for The Triangle. + - `server/` Go API -- `frontend/` React frontend -- The observability stack — Prometheus, Loki, Promtail, Alertmanager, blackbox - and the Grafana dashboards — lives in - [`triangle-infrastructure`](https://github.com/DrexelTriangle/triangle-infrastructure). - The backend still exposes `GET /metrics`; scrape it directly in local dev. -- `scripts/` local setup helpers +- `frontend/` React dashboard +- `scripts/` setup and data helpers + +The observability stack (Prometheus, Loki, Promtail, Alertmanager, blackbox, +Grafana dashboards) lives in +[`triangle-infrastructure`](https://github.com/DrexelTriangle/triangle-infrastructure). +The backend still exposes `GET /metrics`; scrape it directly in local dev. -API docs and data models: https://github.com/DrexelTriangle/triangle-cms/wiki +API reference and data models: https://github.com/DrexelTriangle/triangle-cms/wiki ## Prerequisites -- Python 3.10+ -- Git -- Docker + Docker Compose -- Go 1.24+ -- Node.js 20+ + npm +Python 3.10+, Go 1.24+, Node.js 20+, Docker with Compose, Git. -## First-Time Setup (Recommended) +## First-time setup -Create a `triangle` directory first, then clone `triangle-cms` into it -(`triangle-cms` contains the setup script): +Clone into a `triangle/` parent directory; the setup script clones +`wordpress-etl` and `Scalene` as siblings and installs dependencies for all +three. ```bash -mkdir triangle -cd triangle +mkdir triangle && cd triangle git clone https://github.com/DrexelTriangle/triangle-cms.git cd triangle-cms -``` - -Then run: - -```bash python ./scripts/first_time_setup.py ``` -When run from inside `triangle/triangle-cms`, this installs dependencies for the current checkout and clones: -- `wordpress-etl` -- `Scalene` -as sibling directories inside `triangle/`. +Flags: `--target-dir` for a custom location, `--pull` to update already-cloned +repos, `--skip-embeddings` to skip `sentence-transformers` in `wordpress-etl`. -If you want a custom location, set `--target-dir` explicitly. +## First local run -This prepares and installs dependencies for: -- `triangle-cms` -- `wordpress-etl` -- `Scalene` +1. Create `.env` and set strong values: -Optional flags: -- `--pull` to update already-cloned repos (`git pull --ff-only`) -- `--skip-embeddings` to skip `sentence-transformers` install in `wordpress-etl` + ```bash + cp .env.example .env + ``` -## First Local Run +2. Generate ETL SQL (add `--generate-embeddings` for semantic search): -0. Create Docker env vars for required secrets: + ```bash + cd ../wordpress-etl && .venv/bin/python main.py + ``` -```bash -cp .env.example .env -``` + On Windows, `.venv/Scripts/python.exe main.py`. -Update `.env` with strong values before running Compose. +3. Copy it into the CMS bootstrap files: -1. Generate ETL SQL in `wordpress-etl`: + ```bash + cd ../triangle-cms && python ./scripts/generate_wordpress_sql.py + ``` -```bash -cd ../wordpress-etl -.venv/bin/python main.py -``` +4. Start the stack. Path A runs everything in Docker: -Windows PowerShell: + ```bash + python ./scripts/setup_containers.py + ``` -```powershell -cd ../wordpress-etl -.venv/Scripts/python.exe main.py -``` + Path B runs MariaDB in Docker and the backend locally: -For embeddings SQL: + ```bash + docker compose up -d mariadb + cd server && go run ./main.go + ``` -```bash -cd ../wordpress-etl -.venv/bin/python main.py --generate-embeddings -``` - -2. Copy ETL SQL into CMS bootstrap files: - -```bash -cd ../triangle-cms -python ./scripts/generate_wordpress_sql.py -``` - -3. Start services (Path A is recommended): - -Path A (recommended): full Docker stack (CMS + MariaDB): - -```bash -python ./scripts/setup_containers.py -``` - -This stack uses a shared Docker bridge network scoped to the Compose project, so services can resolve each other by container/service name (for example `mariadb`). - -The dev stack no longer runs Loki, Promtail or Grafana: their config files were -shared with Delta's stack and moved to `triangle-infrastructure` so there is one -source of truth. Use `docker compose logs` locally. - -Path B: Docker MariaDB + local Go backend: - -```bash -docker compose up -d mariadb -``` - -Then run backend locally: - -```bash -cd server -go run ./main.go -``` - -4. Verify API (works with either path): - -```bash -curl -k https://localhost:8080/v1/articles/christmas -``` - -5. Run the Triangle CMS frontend dashboard: - -```bash -cd frontend -npm run dev -- --port 5173 -``` + Compose services resolve each other by service name (`mariadb`). The dev + stack no longer runs Loki, Promtail or Grafana; their configs moved to + `triangle-infrastructure`. Use `docker compose logs`. -Frontend dashboard: `http://localhost:5173` +5. Verify, then start the frontends: -6. Run Scalene frontend: + ```bash + curl -k https://localhost:8080/v1/articles/christmas + cd frontend && npm run dev -- --port 5173 # dashboard on :5173 + cd ../Scalene && npm run dev -- --port 4321 # public site on :4321 + ``` -```bash -cd ../Scalene -npm run dev -- --port 4321 -``` - -Scalene: `http://localhost:4321` +> [!NOTE] +> Scalene only reads a local CMS on its `CMS-Testing` branch +> (`git switch CMS-Testing`). Otherwise it hits production WordPress. -## Common Tasks +## Common tasks -Reset DB + logs (fresh install): +Reset the database and logs: ```bash python ./scripts/setup_containers.py --reset-data ``` -Run backend locally (instead of Docker `cms` service): - -```bash -docker compose stop cms -cd server -go run ./main.go -``` - -If running backend locally against Docker MariaDB, create `server/.env`: +Run the backend locally against the Docker database: `docker compose stop cms`, +then create `server/.env`: ```env DB_NAME=triangle @@ -169,109 +104,64 @@ DB_PORT=3306 ACTIVITY_DB_PATH=./data/activity ``` -CMS activity events are stored in BadgerDB at `server/data/activity` by default for local runs, and in the `cms_activity_data` Docker volume for the Compose stack. - -Run Triangle CMS frontend dashboard: - -```bash -cd frontend -npm run dev -- --port 5173 -``` - -Frontend dashboard: `http://localhost:5173` - -> [!NOTE] -> If you want Scalene to use your local CMS instead of our production WordPress you need to be on the `CMS-Testing` branch -> -> Use `git switch CMS-Testing` to switch - -Run Scalene: - -```bash -cd ../Scalene -npm run dev -- --port 4321 -``` - -Scalene: `http://localhost:4321` +Activity events go to BadgerDB at `server/data/activity` locally, and to the +`cms_activity_data` volume under Compose. -## Rebuild the local DB from the ETL (Advanced) +## Rebuild the local DB from the ETL -One command for the whole chain — run the ETL (with embeddings), regenerate the -seed SQL, recreate the local database from it, verify, and restart the CMS: +Runs the ETL, regenerates the seed SQL, recreates the database, verifies, and +restarts the CMS: ```bash python ./scripts/reseed_from_etl.py -``` - -```bash -python ./scripts/reseed_from_etl.py --skip-etl # reuse the ETL's existing logs/sql +python ./scripts/reseed_from_etl.py --skip-etl # reuse existing ETL output python ./scripts/reseed_from_etl.py --no-embeddings # skip the slow model step python ./scripts/reseed_from_etl.py --yes # no confirmation prompt ``` **This is destructive.** The seed files are mounted into -`docker-entrypoint-initdb.d`, which MariaDB replays only on a first-time init, -so a genuine re-seed means deleting the `mariadb_data` volume. That takes local -`cms_users`, `cms_sessions`, `cms_settings`, polls and the media catalogue with -it — not just imported content. The script prints exactly what will be lost, -with current row counts, and makes you type `reseed`. It refuses to run without -a TTY unless `--yes` is passed. +`docker-entrypoint-initdb.d`, which MariaDB replays only on first init, so a +real reseed deletes the `mariadb_data` volume. That takes `cms_users`, +`cms_sessions`, `cms_settings`, polls and the media catalogue with it, not just +imported content. The script prints the row counts at stake and makes you type +`reseed`; it refuses to run without a TTY unless `--yes` is passed. `--no-embeddings` writes a placeholder that **drops** `article_embeddings`, so -related-articles on `/v1/articles/{slug}` comes back empty until you re-run with +related articles on `/v1/articles/{slug}` come back empty until you re-run with embeddings. -## ETL SQL Script (Advanced) - -Default usage: +For rebuilding production data from a fresh WordPress export, see +[`docs/ETL-REBUILD.md`](docs/ETL-REBUILD.md). -```bash -python ./scripts/generate_wordpress_sql.py -``` - -Optional override usage: +## ETL SQL generator ```bash python ./scripts/generate_wordpress_sql.py [source_sql_dir] [output_dir] +WP_ETL_SQL_DIR=... WP_ETL_OUT_DIR=... python ./scripts/generate_wordpress_sql.py ``` -Environment variable overrides: - -```bash -WP_ETL_SQL_DIR=../wordpress-etl/logs/sql WP_ETL_OUT_DIR=server/internal/database/wordpress_etl python ./scripts/generate_wordpress_sql.py -``` - -Generated files: -- `01-authors.sql` -- `02-articles.sql` -- `03-articles-authors.sql` -- `04-seo.sql` -- `05-article-embeddings.sql` (real file when available, placeholder otherwise) -- `06-taxonomy.sql` -- `07-poll-counts.sql` -- `08-comments.sql` (real file when available, empty table placeholder otherwise) - -## API Docs (Swagger) +Writes `01-authors.sql`, `02-articles.sql`, `03-articles-authors.sql`, +`04-seo.sql`, `05-article-embeddings.sql`, `06-taxonomy.sql`, +`07-poll-counts.sql`, `08-comments.sql`. Embeddings and comments fall back to +placeholders when the ETL did not produce them. -A read-only copy of the API reference is published to GitHub Pages at - — no server, no VPN. It renders the -`server/docs/swagger.json` committed on `main`, so it is only as current as the last -`swag init` that was committed, and "Try it out" is disabled (the spec's host is the -local dev server). +## API docs (Swagger) -The live, interactive Swagger UI is available at `https://localhost:8080/swagger/index.html` -when the server is running. +The interactive UI is at `https://localhost:8080/swagger/index.html` while the +server runs. A read-only copy is published to + from the `server/docs/swagger.json` +committed on `main`, so it is only as current as the last committed `swag init`, +and "Try it out" is disabled there. -The docs are generated automatically during the Docker build — no manual step needed. - -If running the server locally (Path B), regenerate the docs after changing any handler annotations: +The Docker build regenerates the docs. Running locally (Path B), regenerate them +after changing handler annotations: ```bash -cd server -swag init --parseDependency --parseInternal +cd server && swag init --parseDependency --parseInternal ``` -To add or update docs for an endpoint, add swaggo annotations directly above the handler function in `server/internal/handlers/handlers.go`. Example: +Document an endpoint with swaggo annotations above its handler in +`server/internal/handlers/handlers.go`: ```go // @Summary My endpoint @@ -282,18 +172,12 @@ To add or update docs for an endpoint, add swaggo annotations directly above the func MyHandler(...) { ``` -Tag order in the UI is controlled by the `@tag.name` lines in `main.go`. +Tag order in the UI comes from the `@tag.name` lines in `main.go`. ## Testing ```bash cd server go test ./... -``` - -Coverage: - -```bash -cd server go test -coverprofile=cover.out ./... ``` diff --git a/deploy/README.md b/deploy/README.md index 5651b20..301f96b 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,45 +1,43 @@ -# Triangle CMS Delta Deployment +# Triangle CMS Delta deployment -This directory describes the production deployment shape for Delta. It is split -from local development and from database/proxy infrastructure on purpose. +Production deployment shape for Delta, kept separate from local development and +from database/proxy infrastructure. -## Runtime Topology +## Runtime topology -Delta runs: +Delta runs host Nginx on port 80, a self-hosted GitHub Actions runner (labels +`drexel-vpn`, `delta`, `triangle-cms`), and four containers: -- Host Nginx on HTTP port 80. -- Self-hosted GitHub Actions runner with labels `drexel-vpn`, `delta`, and - `triangle-cms`. -- Blue frontend container on `127.0.0.1:8091`. -- Blue backend container on `127.0.0.1:8081`. -- Green frontend container on `127.0.0.1:8092`. -- Green backend container on `127.0.0.1:8082`. +| Slot | Frontend | Backend | +|---|---|---| +| blue | `127.0.0.1:8091` | `127.0.0.1:8081` | +| green | `127.0.0.1:8092` | `127.0.0.1:8082` | -Delta does not run MariaDB, MaxScale, Loki, or Promtail in the CMS -deployment Compose project. The backend connects to the external database/proxy -endpoint supplied by `DB_HOST` and `DB_PORT` in the host-only `cms.env`. +The CMS Compose project does not run MariaDB, MaxScale, Loki or Promtail. The +backend connects to the external database/proxy endpoint given by `DB_HOST` and +`DB_PORT` in the host-only `cms.env`. -Loki and Promtail do run on Delta, but in the separate `triangle-observability` -Compose project, which now lives in [`triangle-infrastructure`](https://github.com/DrexelTriangle/triangle-infrastructure). -Nothing in this repo starts or stops it: the deploy scripts pin `COMPOSE_FILE` -to `compose.cms.yml` so that a CMS deploy can never tear the log stack down, and +Loki and Promtail do run on Delta, in the separate `triangle-observability` +Compose project, which lives in +[`triangle-infrastructure`](https://github.com/DrexelTriangle/triangle-infrastructure). +Nothing here starts or stops it: the deploy scripts pin `COMPOSE_FILE` to +`compose.cms.yml` so a CMS deploy cannot tear the log stack down, and `restart: unless-stopped` keeps it up across deploys. ## Observability Prometheus, Loki, Promtail, Alertmanager, blackbox and the two read-only nginx -endpoints the central Grafana queries **moved to -[`triangle-infrastructure`](https://github.com/DrexelTriangle/triangle-infrastructure)** and are applied with -`ansible-playbook playbooks/observability.yml`. They are no longer deployed by -this repo's workflow. +endpoints central Grafana queries all **moved to +[`triangle-infrastructure`](https://github.com/DrexelTriangle/triangle-infrastructure)** +and are applied with `ansible-playbook playbooks/observability.yml`. This repo's +workflow no longer deploys them. -Two things about it still constrain the CMS, so they are recorded here rather -than only there: +Two things about that stack constrain the CMS: - **Prometheus joins this project's Compose network**, declared `external` on its side. The CMS stack must be up before the observability stack will start, and **renaming a service in `compose.cms.yml` breaks scraping in a different - repository.** `observability/prometheus/prometheus.delta.yml` over there + repository**: `observability/prometheus/prometheus.delta.yml` over there scrapes both slots by container name. - **The backend exposes unauthenticated Prometheus metrics on `GET /metrics`.** That is safe only because the host nginx proxies just `/v1` and `/swagger`, so @@ -51,120 +49,105 @@ than only there: `slot="green"`. The idle slot being up is normal and says nothing about which one serves traffic. -The local development stack in the repo-root `docker-compose.yml` keeps its own -Prometheus (`observability/prometheus/prometheus.dev.yml`). Its Loki, Promtail -and Grafana services were removed when their shared config files moved, so -`docker compose logs` is the local log story now. - +The local stack in the repo-root `docker-compose.yml` keeps its own Prometheus +(`observability/prometheus/prometheus.dev.yml`). Its Loki, Promtail and Grafana +services were removed when their shared configs moved, so `docker compose logs` +is the local log story. ### Reading blue/green logs in Grafana -Both slots run all the time. Only one receives traffic, and **nothing in the log -stream says which one** -- the idle slot goes on emitting healthchecks -indefinitely, so it is entirely possible to read the wrong slot's logs and -conclude the site is idle. The active slot is whatever Nginx currently points -at: +Both slots run all the time, only one receives traffic, and **nothing in the log +stream says which one**: the idle slot goes on emitting healthchecks +indefinitely, so it is easy to read the wrong slot's logs and conclude the site +is idle. The active slot is whatever Nginx points at: ``` sudo cat /etc/nginx/triangle-cms/active-upstreams.conf # set $triangle_cms_slot blue; ``` -Filter by slot in Grafana with the `compose_service` label -(`backend-blue`, `frontend-green`, ...) or `container` -(`triangle-cms-backend-blue-1`). Two consequences of how Compose names things: +Filter by slot with the `compose_service` label (`backend-blue`, +`frontend-green`) or `container` (`triangle-cms-backend-blue-1`). Two +consequences of how Compose names things: - Container names are reused across deploys, so labels do not churn and - cardinality stays flat. But the pre-deploy and post-deploy containers for a - slot land in the *same* stream, so a deploy boundary is only visible by - timestamp, not by label. + cardinality stays flat. But a slot's pre-deploy and post-deploy containers + land in the *same* stream, so a deploy boundary is visible only by timestamp. - A slot's logs are captured from the moment its container starts, including startup and crash output, even though discovery only refreshes every 15s. Promtail backfills from the beginning of a newly discovered container's log, so a container that dies during a deploy still gets its logs shipped. -Nginx serves whichever frontend slot is active and proxies `/v1`, `/swagger`, -and `/swagger/` to the matching backend slot. The initial Nginx config listens on -HTTP with `server_name _`, so it works through Delta's VPN IP or hostname before -a public domain exists. +Nginx serves whichever frontend slot is active and proxies `/v1`, `/swagger` and +`/swagger/` to the matching backend slot. The initial config listens on HTTP +with `server_name _`, so it works through Delta's VPN IP or hostname before a +public domain exists. -Later, when `cms.thetriangle.org` is ready, update the host Nginx site with that -`server_name`, configure HTTPS certificates, update `FRONTEND_ORIGIN` and -`OIDC_REDIRECT_URI` in `cms.env`, and update the GitHub environment variable -`DELTA_PUBLIC_BASE_URL`. The backend should remain in `CMS_SERVER_MODE=internal-http` -behind Nginx. +When `cms.thetriangle.org` is ready: set that `server_name`, configure HTTPS, +update `FRONTEND_ORIGIN` and `OIDC_REDIRECT_URI` in `cms.env`, and update the +GitHub environment variable `DELTA_PUBLIC_BASE_URL`. The backend stays in +`CMS_SERVER_MODE=internal-http` behind Nginx. ## Files - `compose.cms.yml` - Delta-only blue/green frontend/backend slots. - `cms.env.example` - sanitized variable-name-only production env template. -- `nginx/triangle-cms.conf` - **moved** to [`triangle-infrastructure`](https://github.com/DrexelTriangle/triangle-infrastructure) +- `nginx/triangle-cms.conf` - **moved** to + [`triangle-infrastructure`](https://github.com/DrexelTriangle/triangle-infrastructure) (`roles/delta_cms_host/`). Install it with `playbooks/delta-host.yml`. - `nginx/triangle-cms-active-upstreams.conf.example` - generated include seed. - `scripts/deploy.sh` - deploy exact SHA to inactive slot, switch, smoke test. -- `scripts/rollback.sh` - explicit rollback to the other slot or named slot. +- `scripts/rollback.sh` - explicit rollback to the other slot or a named slot. -## One-Time Server Bootstrap +## One-time server bootstrap -A GitHub workflow cannot safely install and register its own runner. Bootstrap is -a manual server task: +A workflow cannot safely install and register its own runner, so bootstrap is a +manual server task: -1. Install Docker, the Docker Compose plugin, Nginx, curl or wget, and flock. +1. Install Docker, the Compose plugin, Nginx, curl or wget, and flock. 2. Create a least-privilege local user for deployments. -3. Register a self-hosted GitHub Actions runner on Delta inside the Drexel VPN. - Apply the labels `drexel-vpn`, `delta`, and `triangle-cms`. +3. Register a self-hosted runner on Delta inside the Drexel VPN, with labels + `drexel-vpn`, `delta`, `triangle-cms`. 4. Allow the runner user to run Docker and reload/test Nginx. Prefer narrow - sudoers rules only for `/usr/sbin/nginx -t` and - `/usr/sbin/nginx -s reload`. -5. Place the host-only production env file at the path configured by - `DELTA_CMS_ENV_FILE`. Do not put it in git. + sudoers rules for `/usr/sbin/nginx -t` and `/usr/sbin/nginx -s reload` only. +5. Place the host-only production env file at `DELTA_CMS_ENV_FILE`. Not in git. 6. Install the host Nginx site with `triangle-infrastructure`'s - `playbooks/delta-host.yml` (see "Installing the Nginx site" below). -7. Create the narrow runtime-state directory and seed the active upstream - include: + `playbooks/delta-host.yml` (below). +7. Create the runtime-state directory and seed the active upstream include: ```bash - sudo install -d \ - -o triangle-runner \ - -g triangle-runner \ - -m 0750 \ + sudo install -d -o triangle-runner -g triangle-runner -m 0750 \ /etc/nginx/triangle-cms - sudo install \ - -o triangle-runner \ - -g triangle-runner \ - -m 0644 \ + sudo install -o triangle-runner -g triangle-runner -m 0644 \ deploy/nginx/triangle-cms-active-upstreams.conf.example \ /etc/nginx/triangle-cms/active-upstreams.conf ``` -8. Validate Nginx and reload it once during bootstrap. +8. Validate Nginx and reload it once. 9. Confirm the runner can pull GHCR images and run `docker compose`. Keep `/etc/nginx` root-owned and non-writable by the runner. Only -`/etc/nginx/triangle-cms` is writable runtime state for deployments, scoped to -the generated active upstream include. The directory should be owned by -`triangle-runner:triangle-runner` with mode `0750`; the active include should be -owned by `triangle-runner:triangle-runner` with mode `0644`. The host Nginx site -such as `/etc/nginx/sites-available/triangle-cms.conf` remains root-owned. +`/etc/nginx/triangle-cms` is runtime state, owned `triangle-runner:triangle-runner` +mode `0750`, with the active include at `0644`. The site itself +(`/etc/nginx/sites-available/triangle-cms.conf`) stays root-owned. ### Installing the Nginx site -**This is Ansible's job now.** The site and the runner-owned state directory are -`roles/delta_cms_host` in [`triangle-infrastructure`](https://github.com/DrexelTriangle/triangle-infrastructure): +**This is Ansible's job.** The site and the runner-owned state directory are +`roles/delta_cms_host` in +[`triangle-infrastructure`](https://github.com/DrexelTriangle/triangle-infrastructure): ```bash ansible-playbook playbooks/delta-host.yml --limit thetriangle-delta --check --diff ansible-playbook playbooks/delta-host.yml --limit thetriangle-delta ``` -The role creates `/etc/nginx/triangle-cms` (`0750`, -`triangle-runner:triangle-runner`) and **deliberately does not manage -`active-upstreams.conf` inside it** — that file is written by `deploy.sh` on +The role creates `/etc/nginx/triangle-cms` and **deliberately does not manage +`active-upstreams.conf` inside it**. That file is written by `deploy.sh` on every release and read back to determine the live slot, so anything that templates it silently reverts production to the other slot. -Seed the include once, by hand, from this repo's -`nginx/triangle-cms-active-upstreams.conf.example`: +Seed the include once, by hand: ```bash sudo install -o triangle-runner -g triangle-runner -m 0644 \ @@ -177,34 +160,34 @@ sudo nginx -t && sudo systemctl reload nginx ``` Nginx will not start without `active-upstreams.conf`, since the site `include`s -it unconditionally. A passing `nginx -t` *before* the site is enabled only -validates the stock config and proves nothing. - -The site sets `client_max_body_size 91m` on `/v1/` to sit just above the -backend's `MEDIA_MAX_UPLOAD_BYTES` (90 MiB default). Nginx's stock limit is 1m, -which rejects an ordinary phone photo with a 413 before the CMS ever sees it, so -a deploy that skips re-copying this file leaves media uploads broken while the -app looks correctly configured. Raise both together, never just one. - -The 90 MiB figure is sized to the migrated corpus, which contains unresized -camera originals up to ~77 MiB (largest: `2025/07/BZ9A5771.jpg`). The hard -ceiling above it is Cloudflare's **100 MB** request-body limit on the tunnel -fronting Delta; a body that passes Nginx and the backend but exceeds that dies -at the edge with an error the CMS never sees, so do not raise the pair past -~95 MiB without moving media uploads off the tunnel. - -The backend streams uploads to a temp file rather than buffering them in RAM -(only the first 8 MiB stays in memory), so a large upload costs container disk, -not memory. +it unconditionally. A passing `nginx -t` *before* the site is enabled validates +only the stock config and proves nothing. + +The site sets `client_max_body_size 91m` on `/v1/`, just above the backend's +`MEDIA_MAX_UPLOAD_BYTES` (90 MiB default). Nginx's stock limit is 1m, which +rejects an ordinary phone photo with a 413 before the CMS sees it, so a deploy +that skips re-copying this file leaves uploads broken while the app looks +correctly configured. Raise both together, never one. + +90 MiB is sized to the migrated corpus, which holds unresized camera originals +up to ~77 MiB (largest: `2025/07/BZ9A5771.jpg`). The ceiling above it is +Cloudflare's **100 MB** request-body limit on the tunnel fronting Delta; a body +that passes Nginx and the backend but exceeds that dies at the edge with an +error the CMS never sees. Do not raise the pair past ~95 MiB without moving +uploads off the tunnel. + +The backend streams uploads to a temp file rather than buffering them (only the +first 8 MiB stays in memory), so a large upload costs container disk, not +memory. ### Media serving `location /wp-content/` reads the migrated WordPress corpus straight off CephFS. -It has no dependency on the containers, the runner, or the database, so it can be -brought up on its own before the rest of the stack exists. `/` and `/v1/` return -502 until a slot is deployed; that is expected and does not affect media. +It depends on no container, runner or database, so it can be brought up before +the rest of the stack exists. `/` and `/v1/` return 502 until a slot is +deployed; that is expected and does not affect media. -Verify the mount and that the Nginx worker user can traverse to it: +Verify the mount and that the Nginx worker can traverse to it: ```bash mountpoint /mnt/cephfs @@ -213,24 +196,24 @@ sudo -u www-data ls /mnt/cephfs/media/wp-content/uploads >/dev/null && echo ok A failure there is almost always missing execute permission on a path component (`sudo chmod o+x /mnt/cephfs /mnt/cephfs/media`), not the Nginx config. On -RHEL-family hosts SELinux blocks the read separately; check `ausearch -m avc -ts -recent` and set `httpd_read_user_content`. +RHEL-family hosts SELinux blocks the read separately: check +`ausearch -m avc -ts recent` and set `httpd_read_user_content`. -Smoke test with a real file: +Smoke test with a real file, expecting `200` and +`Cache-Control: public, max-age=2592000, immutable`: ```bash find /mnt/cephfs/media/wp-content/uploads -name '*.jpg' | head -1 curl -I http://localhost/wp-content/uploads/YYYY/MM/name.jpg ``` -Expect `200` with `Cache-Control: public, max-age=2592000, immutable`. - -### Making the media tree writable (uploads) +### Making the media tree writable -The checks above only prove Nginx can *read*. `POST /v1/media` also has to +Those checks only prove Nginx can *read*. `POST /v1/media` also has to **write**, and the rsynced corpus arrives owned by whoever ran the rsync (`tadmin`), mode 755, while the backend container runs as uid **10001**. Nothing -in the read path notices, so uploads fail long after media serving looks healthy: +in the read path notices, so uploads fail long after media serving looks +healthy: ```bash docker exec triangle-cms-backend-blue-1 \ @@ -238,8 +221,8 @@ docker exec triangle-cms-backend-blue-1 \ ``` If that says `Permission denied`, grant the container uid write on upload -directories. CephFS is mounted with `acl`, so this is additive -- ownership and -the migrated files are untouched, and Nginx keeps reading as before. The mount +directories. CephFS is mounted with `acl`, so this is additive: ownership and +the migrated files are untouched, and Nginx keeps reading as before. A mount supporting ACLs does not mean the tools are installed; Ubuntu server images generally lack them: @@ -249,14 +232,14 @@ sudo find /mnt/cephfs/media/wp-content/uploads -type d \ -exec setfacl -m u:10001:rwx -m d:u:10001:rwx {} + ``` -The `d:` (default) entry is what makes each new `YYYY/MM` directory inherit the -grant, so this does not need repeating every month. +The `d:` (default) entry makes each new `YYYY/MM` directory inherit the grant, +so this does not need repeating every month. -**But a later chmod silently disables it.** A chmod on an ACL'd directory sets -the **ACL mask** from the mode's group bits, and the mask caps every named -entry: `chmod 755` leaves `user:10001:rwx` printed but `#effective:r-x`. An -rsync of the corpus run with `--chmod=D755` does this to every directory it -touches (see `docs/HANDOVER.md` — that command now uses `D775`). +**A later chmod silently disables it.** A chmod on an ACL'd directory sets the +**ACL mask** from the mode's group bits, and the mask caps every named entry: +`chmod 755` leaves `user:10001:rwx` printed but `#effective:r-x`. An rsync run +with `--chmod=D755` does this to every directory it touches (see +`docs/HANDOVER.md`; that command now uses `D775`). Nothing notices at the time. Uploads keep working until the 1st of the next month, when the handler first has to create a new `YYYY/MM`, and then every @@ -273,15 +256,14 @@ creates the month. `triangle-infrastructure`'s `delta_cms_host` role repairs both on every playbook run. Without the `acl` package, setgid does the same job in plain POSIX, at the cost -of changing group ownership rather than adding a grant beside it: +of changing group ownership rather than adding a grant beside it. `101` is the +container's gid; the setgid bit is what new directories inherit: ```bash sudo find /mnt/cephfs/media/wp-content/uploads -type d -exec chgrp 101 {} + sudo find /mnt/cephfs/media/wp-content/uploads -type d -exec chmod 2775 {} + ``` -`101` is the container's gid; the setgid bit is what new directories inherit. - The failure is easy to misread. `MkdirAll` returns nil for a directory that already exists, and every migrated `YYYY/MM` directory does exist, so a permission problem surfaces as `failed to store upload` rather than `failed to @@ -291,8 +273,8 @@ create upload directory`. Check the backend log for the underlying `error=`. Serving the files is independent of *listing* them. The CMS media page reads a `media` table, which starts empty: the rsynced corpus is on disk but unknown to -the database. After the media rsync completes, populate it once from the CMS -(Media -> Reindex) or directly: +the database. After the rsync, populate it once from the CMS (Media -> Reindex) +or directly: ```bash curl -X POST https://localhost/v1/media/index # admin session required; returns 202 @@ -300,34 +282,33 @@ curl https://localhost/v1/media/index # poll progress ``` It walks `MEDIA_ROOT/wp-content/uploads`, skips WordPress's generated `-WxH` -thumbnails, and inserts a row per original. It is idempotent and safe to re-run — -already-indexed files are skipped and any alt text set in the CMS is preserved — -so re-run it after any later out-of-band rsync. Uploads through the CMS index -themselves and need no reindex. +thumbnails, and inserts a row per original. It is idempotent: already-indexed +files are skipped and any alt text set in the CMS is preserved, so re-run it +after any later out-of-band rsync. Uploads through the CMS index themselves. **The index runs in the background.** `POST` returns `202` immediately and `GET` reports `{running, progress:{walked, scanned, added, skipped}, error}`; a second -`POST` while one is in flight returns `409`. This is not cosmetic: the real corpus -is ~145k filesystem entries and the walk takes minutes, while Nginx cuts an idle -upstream read at 60s and Cloudflare at ~100s. A synchronous version was cancelled -by those proxies every time and could never finish. A run is capped at two hours -so a wedged filesystem cannot leave the job stuck "running" forever. +`POST` while one is in flight returns `409`. The real corpus is ~145k filesystem +entries and the walk takes minutes, while Nginx cuts an idle upstream read at +60s and Cloudflare at ~100s, so a synchronous version could never finish. A run +is capped at two hours so a wedged filesystem cannot leave the job stuck. -Progress is reported per entry *walked* rather than per file indexed, because the -corpus is mostly derivatives that are skipped without a stat — counting only -indexed files would look frozen for long stretches. +Progress counts entries *walked* rather than files indexed: the corpus is mostly +skipped derivatives, so counting indexed files would look frozen for long +stretches. ### The public photo gallery -`/v1/gallery`, which the public site's `/photo` page reads, serves only images an +`/v1/gallery`, which the public `/photo` page reads, serves only images an editor has marked (Media -> open an image -> "Show on the photo gallery", or the -"Photo gallery" filter to review the current set). The library itself is every -file on the mount — house ads, comic strips, crossword scans — so an unfiltered -gallery is a dump of the upload directory rather than the photo desk's work. +"Photo gallery" filter to review the set). The library is every file on the +mount, house ads and comic strips and crossword scans included, so an unfiltered +gallery would be a dump of the upload directory rather than the photo desk's +work. Reindexing never sets the flag. WordPress kept the same selection as the -`include_in_gallery` attachment meta, so seed it once per cutover from the legacy -database: +`include_in_gallery` attachment meta, so seed it once per cutover from the +legacy database: ```bash python ./scripts/backfill_gallery_flags.py \ @@ -338,71 +319,61 @@ python ./scripts/backfill_gallery_flags.py \ Passwords come from `WP_DB_PASSWORD` / `CMS_DB_PASSWORD` or a prompt. Drop `--dry-run` to apply. By default the CMS ends up matching WordPress exactly, which also *clears* marks made in the CMS since the last run; once editors are -curating in the CMS, use `--additive` so it only ever adds. Run it after the -media reindex — it matches on file path, and images the library has not seen yet -are reported and skipped. +curating in the CMS, use `--additive`. Run it after the media reindex: it +matches on file path, and images the library has not seen are reported and +skipped. ### Disk -Blue/green keeps two frontend and two backend images resident, plus whatever -prior tags have not been reaped. Delta's root filesystem is small (15 GB), so -prune before it fills: +Blue/green keeps two frontend and two backend images resident, plus unreaped +prior tags. Prune before Delta's root filesystem fills: ```bash docker image prune -af --filter 'until=168h' ``` -## Required Host Environment - -Copy `cms.env.example` to the private host env path and fill it with real values. -The file must contain the exact immutable image tag for the active deployment: - -- `CMS_IMAGE_TAG` -- `CMS_BACKEND_IMAGE` -- `CMS_FRONTEND_IMAGE` -- `DB_NAME` -- `DB_USER` -- `DB_PASSWORD` -- `DB_HOST` -- `DB_PORT` -- `OIDC_ISSUER_URL` -- `OIDC_CLIENT_ID` -- `OIDC_CLIENT_SECRET` -- `FRONTEND_ORIGIN` -- `OIDC_REDIRECT_URI` -- `CMS_SESSION_TTL_SECONDS` -- `CMS_REBUILD_TAXONOMY_COUNTS_ON_STARTUP` -- `AKISMET_API_KEY` - optional; leave empty to disable comment spam filtering. -- `AKISMET_BLOG_URL` - full public site URL Akismet should associate with - comment checks. Required when `AKISMET_API_KEY` is set. +## Required host environment + +Copy `cms.env.example` to the private host env path and fill in real values. It +must carry the exact immutable image tag for the active deployment. + +`CMS_IMAGE_TAG`, `CMS_BACKEND_IMAGE`, `CMS_FRONTEND_IMAGE`, `DB_NAME`, +`DB_USER`, `DB_PASSWORD`, `DB_HOST`, `DB_PORT`, `OIDC_ISSUER_URL`, +`OIDC_CLIENT_ID`, `OIDC_CLIENT_SECRET`, `FRONTEND_ORIGIN`, `OIDC_REDIRECT_URI`, +`CMS_SESSION_TTL_SECONDS`, `CMS_REBUILD_TAXONOMY_COUNTS_ON_STARTUP`, plus: + +- `AKISMET_API_KEY` - optional; empty disables comment spam filtering. +- `AKISMET_BLOG_URL` - public site URL Akismet associates with comment checks. + Required when `AKISMET_API_KEY` is set. - `MEDIA_HOST_PATH` - host path to the CephFS media tree, bind-mounted into the backend. Defaults to `/mnt/cephfs/media`. - `MEDIA_ROOT` - the same tree as seen *inside* the container. Leave at `/mnt/cephfs/media` unless the bind-mount target changes. -- `MEDIA_BASE_URL` - public origin that serves `/wp-content/`, used to build - media URLs returned by the upload endpoint. Empty yields relative URLs. -- `MEDIA_MAX_UPLOAD_BYTES` - per-file upload cap in bytes. Empty uses the - 90 MiB default. Must stay at or below Nginx's `client_max_body_size`. +- `MEDIA_BASE_URL` - public origin serving `/wp-content/`, used to build media + URLs returned by the upload endpoint. Empty yields relative URLs. +- `MEDIA_MAX_UPLOAD_BYTES` - per-file upload cap in bytes; empty uses the 90 MiB + default. Must stay at or below Nginx's `client_max_body_size`. -Keep `CMS_REBUILD_TAXONOMY_COUNTS_ON_STARTUP=false` in production. Rebuild -taxonomy through the admin endpoint after deploys when needed. +Keep `CMS_REBUILD_TAXONOMY_COUNTS_ON_STARTUP=false` in production; rebuild +through the admin endpoint after deploys when needed. -New users are created as editors. The very first user to log in to an empty +New users are created as editors. The first user to log in to an empty `cms_users` table is bootstrapped as an admin; promote anyone else from the users screen. -## Required GitHub Production Environment Variables +## Required GitHub environment variables -Configure these as GitHub Environment variables for `production`: +Set these on the `production` GitHub Environment: - `DELTA_CMS_ENV_FILE` - absolute path to the host-only `cms.env`. -- `DELTA_NGINX_ACTIVE_INCLUDE` - usually `/etc/nginx/triangle-cms/active-upstreams.conf`. +- `DELTA_NGINX_ACTIVE_INCLUDE` - usually + `/etc/nginx/triangle-cms/active-upstreams.conf`. - `DELTA_PUBLIC_BASE_URL` - initial HTTP VPN URL or hostname for smoke tests. -Production database passwords, OIDC secrets, runner registration tokens, -certificates, and server addresses must not be exposed to pull-request workflows. -The deploy workflow runs only on the labelled self-hosted runner and uses the -host env file. +Database passwords, OIDC secrets, runner registration tokens, certificates and +server addresses must not be exposed to pull-request workflows. The deploy +workflow runs only on the labelled self-hosted runner and uses the host env +file. ## Deployment @@ -412,70 +383,50 @@ Images are immutable and tagged only with the full commit SHA: - `ghcr.io/drexeltriangle/triangle-cms-frontend:` Automatic publish runs only after a successful CI workflow for a trusted push to -`main`. It publishes backend and frontend images tagged with that exact commit -SHA. Manual publish is intentionally unsupported. - -The deploy workflow checks out trusted deployment code from the protected default -branch. The image SHA is data only: it is never used as an Actions checkout ref, -script path, Compose-file source, env-file source, or executable source. +`main`, and tags both images with that exact SHA. Manual publish is +intentionally unsupported. -The deploy workflow runs: +The deploy workflow checks out deployment code from the protected default +branch. The image SHA is data only: never an Actions checkout ref, script path, +Compose-file source, env-file source, or executable source. ```bash deploy/scripts/deploy.sh ``` -The script: +The script acquires an exclusive `flock`, runs preflight checks, reads the +active slot from the Nginx include, pulls the exact SHA images, starts only the +inactive slot, waits for backend `/v1/health/db` and frontend `/healthz`, writes +the active include atomically, runs `nginx -t` and a graceful reload, then runs +public smoke tests through Nginx. It switches back automatically if post-switch +smoke tests fail, and keeps the previous slot running for fast rollback. It +never runs `docker compose down -v` and never deletes persistent data. -- Acquires an exclusive `flock`. -- Runs deployment preflight checks before pulling images or starting containers. -- Reads the active slot from the Nginx include. -- Pulls the exact frontend/backend SHA images. -- Starts only the inactive frontend/backend services. -- Waits for backend `/v1/health/db` and frontend `/healthz`. -- Writes the active Nginx include atomically. -- Runs `nginx -t` and gracefully reloads Nginx. -- Runs public smoke tests through Nginx. -- Switches back automatically if post-switch smoke tests fail. -- Keeps the previous slot running for fast rollback. - -It never runs `docker compose down -v` and never deletes persistent data. - -The deployment preflight fails before pulling images, starting containers, or -switching Nginx if the active include directory is missing or not writable, an -existing active include is not readable and writable, the active slot is not -`blue` or `green`, `cms.env` is missing or unreadable, or Nginx validation/reload -privileges are not available. +Preflight fails before pulling images, starting containers or switching Nginx if +the active include directory is missing or not writable, an existing include is +not readable and writable, the active slot is not `blue` or `green`, `cms.env` +is missing or unreadable, or Nginx validation/reload privileges are unavailable. ## Rollback -Rollback switches Nginx back to the previous running slot: +Switches Nginx back to the previous running slot, optionally by name: ```bash deploy/scripts/rollback.sh -``` - -You can also name a target slot: - -```bash deploy/scripts/rollback.sh blue -deploy/scripts/rollback.sh green ``` -For recovery to an older image SHA, manually run the deploy workflow with that -full SHA. The backend and frontend images for that SHA must already exist in -GHCR. Manual deployment starts the inactive slot with those immutable images and -then switches traffic after health checks. +To recover an older image SHA, run the deploy workflow manually with that SHA; +its backend and frontend images must already exist in GHCR. That starts the +inactive slot on those immutable images and switches after health checks. -## Stateful and Rollback Notes +## Stateful and rollback notes -The backend runs additive, idempotent startup schema operations such as -`CREATE TABLE IF NOT EXISTS`, `ADD COLUMN IF NOT EXISTS`, and one guarded SEO -backfill. Rollback is safe only while database changes stay backward-compatible. -Do not deploy destructive migrations without a backup and a tested restoration -plan. +The backend runs additive, idempotent startup schema operations (`CREATE TABLE +IF NOT EXISTS`, `ADD COLUMN IF NOT EXISTS`, one guarded SEO backfill). Rollback +is safe only while database changes stay backward-compatible. Do not deploy +destructive migrations without a backup and a tested restore. -Activity/audit state is in MariaDB. Article edit leases and IP rate-limit -counters remain in process memory; they can reset during a release switch. Shared -locking/rate limiting should move to MariaDB or Redis later, but that is outside -this CI/CD implementation. +Activity and audit state is in MariaDB. Article edit leases and IP rate-limit +counters live in process memory and can reset during a release switch; shared +locking and rate limiting should move to MariaDB or Redis eventually. diff --git a/deploy/cms.env.example b/deploy/cms.env.example index b4af0ac..3bb6d5b 100644 --- a/deploy/cms.env.example +++ b/deploy/cms.env.example @@ -15,7 +15,7 @@ CMS_SESSION_TTL_SECONDS= CMS_REBUILD_TAXONOMY_COUNTS_ON_STARTUP= # Slack classified moderation. The webhook posts the notification; the signing # secret authorizes the Approve/Reject clicks that come back. Setting only one -# of the two leaves the loop broken -- the backend warns at startup which half +# of the two leaves the loop broken; the backend warns at startup which half # is missing. SLACK_CLASSIFIEDS_QUEUE_URL is optional and only adds a link back # to the CMS queue. SLACK_WEBHOOK_URL= @@ -37,12 +37,12 @@ MEDIA_MAX_UPLOAD_BYTES= # embedding reconciler exits at startup. # # EMBED_MODEL must match the WordPress ETL's --embedding-model, and changing it -# is a schema change if the new model is not 384-dimensional -- see +# is a schema change if the new model is not 384-dimensional; see # server/internal/database/schema/article_embeddings.sql. EMBEDDINGS_URL=http://embeddings:8000 EMBED_MODEL=BAAI/bge-small-en-v1.5 # Cores the sidecar may use, out of the host's 6. Uncapped, embedding a freshly # reseeded corpus pins nearly the whole machine for over an hour and starves the -# backends. Raising this speeds up that backfill and nothing else -- ordinary +# backends. Raising this speeds up that backfill and nothing else: ordinary # search only embeds one short query at a time, and caches the result. EMBED_CPUS=4 diff --git a/deploy/compose.cms.yml b/deploy/compose.cms.yml index 048cbee..1c03dd1 100644 --- a/deploy/compose.cms.yml +++ b/deploy/compose.cms.yml @@ -47,7 +47,7 @@ x-backend-base: &backend-base # (91m) and below Cloudflare's 100 MB tunnel limit; raise all of them # together or the smallest one silently wins. MEDIA_MAX_UPLOAD_BYTES: ${MEDIA_MAX_UPLOAD_BYTES:-94371840} - # Semantic search. Shared by both slots -- the sidecar is stateless, so there + # Semantic search. Shared by both slots: the sidecar is stateless, so there # is no reason to run one per slot. Leave empty to run lexical-only search: # the backend then skips query embedding and its reconciler exits at start. EMBEDDINGS_URL: ${EMBEDDINGS_URL-http://embeddings:8000} @@ -80,7 +80,7 @@ x-frontend-base: &frontend-base services: # One shared instance, not one per slot: it holds no state, so both slots can # use it and a deploy never has to wait for a model to load. Not exposed to the - # host -- only the backends talk to it, over the internal network. + # host; only the backends talk to it, over the internal network. embeddings: # Tagged by content, not by commit: CMS_EMBEDDINGS_TAG is the git tree hash # of embeddings/, so it only changes when the sidecar itself changes. Tagging @@ -98,8 +98,8 @@ services: # without OMP_NUM_THREADS, onnxruntime still spawns one thread per visible # core and thrashes against that ceiling instead of respecting it. # - # This trades backfill throughput for headroom -- roughly 2 articles/sec - # becomes 1.4 -- which is the right way round: the backfill is background + # This trades backfill throughput for headroom (roughly 2 articles/sec + # becomes 1.4) which is the right way round: the backfill is background # work that converges on its own, and nothing waits on it. cpus: ${EMBED_CPUS:-4} environment: diff --git a/deploy/scripts/common.sh b/deploy/scripts/common.sh index 2f31b9c..13eb5f9 100755 --- a/deploy/scripts/common.sh +++ b/deploy/scripts/common.sh @@ -246,7 +246,7 @@ wait_for_url() { } # wait_for_embeddings polls the container's health state rather than an HTTP -# endpoint, because the sidecar is deliberately not published to the host -- only +# endpoint, because the sidecar is deliberately not published to the host: only # the backends reach it, over the compose network. Its healthcheck 503s until the # model has finished loading, so "healthy" here means it can actually answer. wait_for_embeddings() { diff --git a/deploy/scripts/deploy.sh b/deploy/scripts/deploy.sh index 53a3f88..d6c8009 100755 --- a/deploy/scripts/deploy.sh +++ b/deploy/scripts/deploy.sh @@ -22,7 +22,7 @@ export CMS_IMAGE_TAG # The sidecar image is tagged by content: the git tree hash of embeddings/, # which changes only when something in that directory does. Publish skips the # build when that tag already exists, and Compose leaves the running container -# alone when the tag is unchanged -- so a Go-only commit no longer rebuilds a +# alone when the tag is unchanged, so a Go-only commit no longer rebuilds a # ~130MB model image in CI, nor reloads the model on the host for ~60s. # # Derived from the trusted default-branch checkout rather than from @@ -55,7 +55,7 @@ echo "deploying ${CMS_IMAGE_TAG} to inactive slot: ${next_slot}" # The embedding sidecar is shared by both slots rather than duplicated per slot: # it is stateless, so a second copy would only cost memory on a host that has # little to spare. That means it is not part of the blue/green swap and has to be -# brought up separately -- the slot services below are started with --no-deps. +# brought up separately; the slot services below are started with --no-deps. # # Deliberately never fatal. Search degrades to lexical-only when the sidecar is # missing or still loading its model, so a sidecar problem must not block or roll diff --git a/deploy/scripts/deploy_scripts_test.sh b/deploy/scripts/deploy_scripts_test.sh index 83d57df..0699c53 100755 --- a/deploy/scripts/deploy_scripts_test.sh +++ b/deploy/scripts/deploy_scripts_test.sh @@ -56,7 +56,7 @@ make_case() { PUBLIC_HEALTH_TIMEOUT=0 # The sidecar wait polls container health, which never turns healthy against # the fake compose in these tests. Left at its 240s production default it made - # every deploy case sit out the full timeout -- the suite took 8 minutes. + # every deploy case sit out the full timeout; the suite took 8 minutes. EMBEDDINGS_HEALTH_TIMEOUT=0 DEPLOY_TEST_MODE=1 NGINX_TEST_CMD='exit "${FAKE_NGINX_TEST_STATUS:-0}"' diff --git a/docker-compose.yml b/docker-compose.yml index 8e0bfd0..ac442de 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,7 +35,7 @@ services: restart: unless-stopped # Capped for the same reason as production (see deploy/compose.cms.yml): # uncapped, embedding a freshly seeded corpus saturates every core on the - # machine. Both settings are needed and should match -- cpus is the ceiling, + # machine. Both settings are needed and should match: cpus is the ceiling, # OMP_NUM_THREADS stops onnxruntime spawning a thread per core regardless. cpus: ${EMBED_CPUS:-4} environment: @@ -50,7 +50,7 @@ services: retries: 5 start_period: 60s # Stateless by design: no volume, nothing to back up. Not published to the - # host either -- only the CMS talks to it, over the internal network. + # host either; only the CMS talks to it, over the internal network. networks: - triangle_net diff --git a/docs/ETL-REBUILD.md b/docs/ETL-REBUILD.md index 66d6db7..b9b763f 100644 --- a/docs/ETL-REBUILD.md +++ b/docs/ETL-REBUILD.md @@ -1,55 +1,51 @@ -# Destructive ETL Rebuild — Primer +# Destructive ETL rebuild -How to rebuild the `triangle` database from scratch out of a fresh WordPress -export. This is the "nuke and repave" path: `DROP DATABASE` → load six -ETL-generated SQL files → restore the things the ETL does not produce. +Rebuilding the `triangle` database from a fresh WordPress export: `DROP +DATABASE`, load six ETL-generated SQL files, restore what the ETL does not +produce. Last verified end-to-end **2026-08-01** (9780 articles onto DB1). --- -## 1. What this is, and when you actually need it +## 1. When you need it The ETL (`wordpress-etl`) turns a WordPress WXR export into six SQL files under -`logs/sql/`. Those files each carry their own `CREATE TABLE`, so loading them -means dropping and recreating the database. There is no incremental mode and no -migration tool. +`logs/sql/`. Each carries its own `CREATE TABLE`, so loading them means dropping +and recreating the database. There is no incremental mode and no migration tool. -Reach for a full rebuild when: +Rebuild when: -- article **content** must change wholesale — image URL canonicalization, - byline fixes, category/tag handling, un-dropping comics, etc. All of these are - baked into the data at ETL time, so an env-var change alone does nothing. +- article **content** must change wholesale: image URL canonicalization, byline + fixes, category/tag handling, un-dropping comics. All of it is baked into the + data at ETL time, so an env-var change alone does nothing. - you are pulling a newer WordPress export. -Do **not** rebuild for CMS-side schema changes. The Go backend applies those -itself at startup via `EnsureArticlesSchema` (`server/internal/database/users.go`) -with idempotent `ADD COLUMN IF NOT EXISTS`. +Do **not** rebuild for CMS-side schema changes. The backend applies those at +startup via `EnsureArticlesSchema` (`server/internal/database/users.go`) with +idempotent `ADD COLUMN IF NOT EXISTS`. ## 2. What it destroys -The ETL only owns six tables: `articles`, `authors`, `articles_authors`, `seo`, -`article_embeddings`, `comments`. - -Everything else in the database is CMS-side and is **created empty at runtime** -by the Go backend. `DROP DATABASE` therefore wipes real editorial state: +The ETL owns six tables: `articles`, `authors`, `articles_authors`, `seo`, +`article_embeddings`, `comments`. Everything else is CMS-side and **created +empty at runtime**, so `DROP DATABASE` wipes real editorial state: | Lost | Consequence | | --- | --- | -| `site_taxonomy` | **Every section page on the public site 404s.** Not "counts read 0" — Scalene resolves section routes from `/v1/taxonomy`, and an empty response means `/entertainment` etc. render the 404 page. Sections are config, not test data. | +| `site_taxonomy` | **Every section page on the public site 404s.** Scalene resolves section routes from `/v1/taxonomy`, so an empty response renders `/entertainment` as the 404 page. Sections are config, not test data. | | `media.in_gallery` | `/photo` goes empty. The gallery is a hand-curated 25-image set; nothing re-seeds it. | | `cms_users`, `cms_sessions` | Everyone is logged out. Auto-promote recreates the first user on next login. | -| `cms_settings` | Resets to defaults — breaking-news banner text is gone. | +| `cms_settings` | Resets to defaults; breaking-news banner text is gone. | | `cms_activity`, `cms_poll_counts` | Gone. | -Take a dump first, always. DB1 is still the only live copy — there is no DB2. +Take a dump first, always. DB1 is still the only live copy; there is no DB2. ``` mariadb-dump --single-transaction --routines --events --databases triangle | gzip > pre-reset.sql.gz ``` -Pull a copy off-host. You will need this dump again in step 6 to restore -`site_taxonomy`. +Pull a copy off-host. Step 6 needs it to restore `site_taxonomy`. ## 3. Prerequisites @@ -57,25 +53,24 @@ Pull a copy off-host. You will need this dump again in step 6 to restore `wordpress-etl/Data/wp-export.zip` (path hardcoded at `Utils/Constants.py`). Members are matched by case-sensitive filename prefix: -- `wp-posts*.xml` — posts. An "All content" export is safe here; the extractor - drops anything that isn't `post_type == "post"`. -- `wp-guestAuths*.xml` — a WXR export of the Co-Authors Plus `guest-author` post - type. **Do not reuse the all-content export for this half** — the guest-author - parser does not filter by post type and would run every post through the - author path. +- `wp-posts*.xml`: posts. An "All content" export is safe; the extractor drops + anything that is not `post_type == "post"`. +- `wp-guestAuths*.xml`: a WXR export of the Co-Authors Plus `guest-author` post + type. **Do not reuse the all-content export here.** The guest-author parser + does not filter by post type and would run every post through the author path. -Also confirm before starting: +Also confirm: -- Media is synced to Ceph (see step 7 — do it *after* the load, but know the gap - exists). -- You know which `MEDIA_BASE_URL` you want baked into the data. It must be a - public hostname, never a private IP: the ETL writes it into `photo_url` **and** - into inline `` tags in article bodies, both of which render to browsers. +- Media is synced to Ceph. Do it *after* the load (step 7), but know the gap + exists. +- Which `MEDIA_BASE_URL` you want baked in. It must be a public hostname, never + a private IP: the ETL writes it into `photo_url` **and** into inline `` + tags in article bodies, both of which render to browsers. ## 4. Run the ETL -There is a real `--headless` now (ETL commit `2975d05`). No TUI, no scratchpad -driver — the older notes describing a stub driver are obsolete. +`--headless` is real as of ETL commit `2975d05`. Older notes describing a stub +TUI driver are obsolete. ``` cd wordpress-etl @@ -84,7 +79,7 @@ MEDIA_BASE_URL=https://delta.thetriangle.org \ ``` Embeddings take about 65 seconds for ~10k articles. `--headless` requires -`--best-guess`. Note `triangle-cms/scripts/reseed_from_etl.py` does *not* pass +`--best-guess`. `triangle-cms/scripts/reseed_from_etl.py` does *not* pass `--headless`, so it still opens the TUI, and it is local-stack only. Output lands in `wordpress-etl/logs/sql/`: @@ -95,23 +90,23 @@ articles.sql authors.sql articles_authors.sql seo.sql comments.sql article_ ### Prompt traps -`--headless` claims it cannot prompt. That is not quite true. It covers author -*matching*, but duplicate-record **field conflicts** still hard-exit the run. -Resolved conflicts are cached in `logs/auth_conflicts.json`. +`--headless` covers author *matching*, but duplicate-record **field conflicts** +still hard-exit the run. Resolved conflicts are cached in +`logs/auth_conflicts.json`. -Historically both ID-keyed caches (`auth_conflicts.json` and -`logs/article-sanitizer/article_author_resolution_cache.json`) rotted between -exports, because **WordPress renumbers *and reuses* author IDs**. A stale ID -either orphaned the link or silently credited an article to the wrong person. -Fixed upstream in PRs #54/#56/#57: a cached decision now records *which person* -wins, and the ID is always re-looked-up from the current export. If you ever -touch that code again, keep that invariant — never write a cached ID. +Both ID-keyed caches (`auth_conflicts.json` and +`logs/article-sanitizer/article_author_resolution_cache.json`) used to rot +between exports, because **WordPress renumbers *and reuses* author IDs**. A +stale ID either orphaned the link or silently credited an article to the wrong +person. Fixed in PRs #54/#56/#57: a cached decision records *which person* wins, +and the ID is re-looked-up from the current export. Keep that invariant if you +touch the code: never write a cached ID. ## 5. Load into DB1 DB1 is `10.248.40.154`; `tadmin` has NOPASSWD sudo there. Delta cannot ssh to -DB1 (host key unverified, and `triangle_user` is granted only from `.183`/`.168`), -so load from the workstation. +DB1 (host key unverified, and `triangle_user` is granted only from +`.183`/`.168`), so load from the workstation. ``` ssh tadmin@10.248.40.154 'sudo -n mariadb --default-character-set=utf8mb4 -e "DROP DATABASE triangle; CREATE DATABASE triangle;"' @@ -121,7 +116,7 @@ for f in authors articles articles_authors seo article_embeddings comments end ``` -Load in that order — `articles_authors` has FK-shaped dependencies on the first +Load in that order: `articles_authors` has FK-shaped dependencies on the first two. ### `--default-character-set=utf8mb4` is not optional @@ -132,7 +127,7 @@ character: `“` (`E2 80 9C`) is stored as `“`. This cost a full reload on 2026-08-01. **The obvious check cannot detect it.** Reading the column back through a bare -client converts it a second time and prints a correct-looking `“` — the two +client converts it a second time and prints a correct-looking `“`; the two errors cancel. Verify with `SELECT HEX(...)` and expect `E2809C`. Likewise, `LIKE "%â€%"` typed at a shell gets mangled in transit and returns 0 rows, which reads as "no corruption". @@ -143,11 +138,11 @@ A partial load looks like a successful one. The classic failure is the authors file aborting halfway on a duplicate key while every other table loads fine. Do not pipe loader output through `head`. -(`article_embeddings.sql` legitimately lacks the `NO_AUTO_VALUE_ON_ZERO` -preamble the other five carry — its PK is a bare `BIGINT`, not `AUTO_INCREMENT`, -so the `id=0` row stores literally. Don't "fix" it.) +`article_embeddings.sql` legitimately lacks the `NO_AUTO_VALUE_ON_ZERO` preamble +the other five carry: its PK is a bare `BIGINT`, not `AUTO_INCREMENT`, so the +`id=0` row stores literally. Do not "fix" it. -## 6. Restore what the ETL doesn't produce +## 6. Restore what the ETL does not produce **Order matters: restart the backend first.** `site_taxonomy` is created at runtime by the Go code, so restoring into a freshly dropped database fails with @@ -155,50 +150,55 @@ runtime by the Go code, so restoring into a freshly dropped database fails with 1. Restart **both** blue and green backends. Green serves; blue is the rollback target and must see the new schema too. -2. Restore sections from the pre-reset dump: +2. Restore sections from the pre-reset dump, loading it the same way as above. + Only 7 of the ~35 rows are `kind='section'`, but restore all of them. + ``` zcat pre-reset.sql.gz | awk '/^INSERT INTO `site_taxonomy`/,/;$/' > taxonomy.sql ``` - Load it the same way as above. Only 7 of the ~35 rows are `kind='section'`, - but restore all of them. -3. Backfill parent sections. The ETL writes whatever categories WordPress had, - which for a subsection article is the subsection alone — so the row carries - "Welcome Week" with no "Special Editions" above it. Section *listing* still - finds those articles (a section query expands to its subsections), but - anything reading `articles.categories` directly, including the public site's - per-article section labels, sees an unparented subsection. Must run after - step 2: it reads the section tree out of `site_taxonomy`. + +3. Backfill parent sections. Must run after step 2: it reads the section tree + out of `site_taxonomy`. + ``` python ./scripts/backfill_parent_sections.py \ --cms-dsn 'triangle_user@tcp(10.248.40.154)/triangle' --dry-run ``` + + The ETL writes whatever categories WordPress had, which for a subsection + article is the subsection alone, so the row carries "Welcome Week" with no + "Special Editions" above it. Section *listing* still finds those articles (a + section query expands to its subsections), but anything reading + `articles.categories` directly, including the public site's per-article + section labels, sees an unparented subsection. + Via MaxScale add `--skip-ssl`: the proxy does not offer TLS and MariaDB - clients 11.4+ require it by default. Delta has no MariaDB client package — + clients 11.4+ require it by default. Delta has no MariaDB client package; `docker run --rm -i -v /tmp:/tmp mariadb:11.7 mariadb "$@"` as a shim on - `PATH` works, and the `-v /tmp:/tmp` matters so the script's 0600 option file + `PATH` works, and `-v /tmp:/tmp` matters so the script's 0600 option file resolves inside the container. - Additive and idempotent — it only appends a missing parent, never removes or - reorders — so re-running is a no-op and `--dry-run` first costs nothing. It - refuses any row whose categories text opens with `[` but will not parse as - JSON, reporting the ids rather than risk corrupting them. + The script is additive and idempotent, only ever appending a missing parent, + so re-running is a no-op and `--dry-run` first costs nothing. It refuses any + row whose categories text opens with `[` but will not parse as JSON, + reporting the ids rather than risking corruption. **Read the dry run before applying.** It only adds a parent the article is not already matched by, using the same fuzzy substring rule the read path - uses — so an article tagged `Arts & Entertainment` does *not* also get - `Entertainment`, because `%entertainment%` already matches it. First run on - DB1 (2026-08-02) rewrote **638 of 10058** rows: mostly `Comics` → adds - `Comics & Puzzles` (329), plus `Music`/`Cooking` → `Entertainment`, - `Public Safety` → `News`, `Men's Basketball` → `Sports`. -4. Restart the backend again to rebuild taxonomy counts. Note this is *not* - needed on account of step 3: counts bucket by section **plus its - subsections**, so a subsection-only article was already counted — verified on - 2026-08-02, `comics-puzzles` read 331 both before and after. The backfill is - count-neutral; it changes which sections an article *names*, not the totals. -5. Re-flag the photo gallery — 25 rows in `media.in_gallery`. Source of truth is - `https://cms.thetriangle.org/wp-json/triangle/v1/gallery` (disappears when - WordPress is retired); `scripts/backfill_gallery_flags.py` is the durable - route but has never been run against the real WP DB. + uses, so an article tagged `Arts & Entertainment` does *not* also get + `Entertainment`. First run on DB1 (2026-08-02) rewrote **638 of 10058** rows: + mostly `Comics` → `Comics & Puzzles` (329), plus `Music`/`Cooking` → + `Entertainment`, `Public Safety` → `News`, `Men's Basketball` → `Sports`. + +4. Restart the backend again to rebuild taxonomy counts. This is *not* needed on + account of step 3: counts bucket by section **plus its subsections**, so a + subsection-only article was already counted. Verified 2026-08-02, + `comics-puzzles` read 331 both before and after. The backfill changes which + sections an article *names*, not the totals. +5. Re-flag the photo gallery, 25 rows in `media.in_gallery`. Source of truth is + `https://cms.thetriangle.org/wp-json/triangle/v1/gallery` (which disappears + when WordPress is retired); `scripts/backfill_gallery_flags.py` is the + durable route but has never been run against the real WP DB. 6. Log back in; re-enter any breaking-news text. ### Never re-slug a section to fix a count @@ -206,8 +206,8 @@ runtime by the Go code, so restoring into a freshly dropped database fails with Section slugs are hardcoded in **both** repos (Scalene's `Header.astro`, `SideMenu.astro`, `index.astro`, `siteConfig.ts`; the CMS's `handlers.go` homepage block). Changing `comics-puzzles` → `comics` on 2026-08-01 returned -400s on the live site. The correct fix for an under-matching section is to add a -**subsection** whose slug matches the real category and leave the section slug +400s on the live site. The fix for an under-matching section is to add a +**subsection** whose slug matches the real category, leaving the section slug alone. ## 7. Post-load verification @@ -215,23 +215,23 @@ alone. Run all of these. Each has produced a real defect at least once. **Media gap.** A fresh export references images added to WP after the last -rsync. Test the filesystem, not curl — Cloudflare caches 404s: +rsync. Test the filesystem, not curl; Cloudflare caches 404s: ``` while read p; do [ -f "/mnt/cephfs/media$p" ] || echo "$p"; done ``` -On 2026-08-01, 24 of 6323 featured images were absent, and 22 of those are -absent on WordPress itself (irreducible). Sync must be **pushed** from the WP VM +On 2026-08-01, 24 of 6323 featured images were absent, 22 of them absent on +WordPress itself (irreducible). The sync must be **pushed** from the WP VM `10.248.40.141`; Delta cannot pull. rsync exiting **23** with "failed to set -times on .../2026/08" is one directory's mtime, not a data failure — ignore it. +times on .../2026/08" is one directory's mtime, not a data failure. Newly synced files can 404 publicly for up to 4 hours while the pre-sync edge -404 ages out. `?v=` returns 200 immediately; that confirms the file is fine. +404 ages out. `?v=` returning 200 confirms the file is fine. **Upload ACL mask.** The sync can leave media *readable* and uploads broken. A chmod on an ACL'd directory rewrites the ACL mask, capping the backend's -`user:10001:rwx` grant to `#effective:r-x`; the symptom is delayed until the 1st +`user:10001:rwx` grant to `#effective:r-x`. The symptom is delayed until the 1st of the next month, when `POST /v1/media` starts returning 500 on every upload. Check the mask, not the entry: @@ -240,7 +240,7 @@ getfacl -pc /mnt/cephfs/media/wp-content/uploads/2026 | grep -E '10001|mask::' # want: user:10001:rwx (no "#effective:") and mask::rwx ``` -**Orphaned author links** — byline present in `articles.authors` but the join +**Orphaned author links.** Byline present in `articles.authors` but the join returns nothing, so the API emits `authors: null`: ```sql @@ -249,7 +249,7 @@ LEFT JOIN authors au ON au.id = aa.author_id WHERE au.id IS NULL; ``` -**Mis-credited articles** — the row is valid, so no integrity check catches it: +**Mis-credited articles.** The row is valid, so no integrity check catches it: ```sql SELECT ar.id FROM articles ar @@ -258,22 +258,22 @@ JOIN authors au ON au.id = aa.author_id WHERE ar.authors NOT LIKE CONCAT('%', au.display_name, '%'); ``` -**Encoding** — `SELECT HEX(text) ...`, expect `E2809C` for smart quotes. +**Encoding.** `SELECT HEX(text) ...`, expect `E2809C` for smart quotes. -**Photo URLs** — sample 15–20 and check they return 200. Note the API field is +**Photo URLs.** Sample 15-20 and check they return 200. The API field is `featured_image`; the DB column is `photo_url`. Easy false alarm. -**Section pages** — hit `/v1/taxonomy` and load a couple of section routes. +**Section pages.** Hit `/v1/taxonomy` and load a couple of section routes. `dev.thetriangle.org` sits behind Cloudflare Access, so page-level checks must happen in a browser, not from curl. -### Expected residue that is *not* a bug +### Expected residue -- ~1200 `www.thetriangle.org` URLs — article cross-links in body text. -- 2 `therectangle.org` URLs — also article links (the defunct predecessor - domain), not media. +- ~1200 `www.thetriangle.org` URLs: article cross-links in body text. +- 2 `therectangle.org` URLs: also article links to the defunct predecessor + domain, not media. - `entertainment` counts far fewer than its 2543 `Arts & Entertainment` - articles: listing resolves slug → canonical title, counting buckets by + articles. Listing resolves slug → canonical title; counting buckets by `CanonicalizeSlug(category)`. Different keys, so a section can list correctly while its count is wrong. Cosmetic. @@ -284,7 +284,7 @@ happen in a browser, not from curl. | ETL repo | `wordpress-etl`, export at `Data/wp-export.zip` | | ETL output | `logs/sql/{articles,authors,articles_authors,seo,comments,article_embeddings}.sql` | | DB1 | `10.248.40.154`, `tadmin` NOPASSWD sudo, load from workstation | -| Delta (app host) | `tadmin@10.248.40.168` — reaches DB via MaxScale `10.248.40.183:4006` | +| Delta (app host) | `tadmin@10.248.40.168`, reaches DB via MaxScale `10.248.40.183:4006` | | WP VM (media source) | `tadmin@10.248.40.141`, docroot `/var/www/html/thetriangle.org/wp-content/uploads/` | | Media on Delta | `/mnt/cephfs/media/wp-content/uploads/` | -| Deploy | merge to `main` → auto-deploys in ~3 min | +| Deploy | merge to `main`, auto-deploys in ~3 min | diff --git a/docs/HANDOVER.md b/docs/HANDOVER.md index ad1f2d4..1014b72 100644 --- a/docs/HANDOVER.md +++ b/docs/HANDOVER.md @@ -1,21 +1,20 @@ -# Triangle CMS — Delta handover +# Triangle CMS: Delta handover -State of the Delta deployment as of **2026-08-01**, what to watch out for, and -what is still open. Written for someone taking over operation of the system. +State of the Delta deployment as of **2026-08-01**, and what is still open. Companion docs: the [wiki](https://github.com/DrexelTriangle/triangle-cms/wiki) has the API reference and data models; `README.md` covers local development. --- -## 1. What is running where +## 1. What runs where | Host | Address | Role | |---|---|---| | `thetriangle-delta` (VM 105) | `10.248.40.168` | CMS backend + frontend (blue/green), host nginx, CephFS media, GitHub Actions runner | | `THETRIANGLE-DB1-LXC` (CT 108) | `10.248.40.154` | MariaDB 11.8 primary | | `THETRIANGLE-MAXSCALE` (CT 109) | `10.248.40.183` | MaxScale 24.02 proxy, port **4006** | -| `thetriangle-wordpress` (VM 100) | `10.248.40.141` | Legacy WordPress — source of media and exports | +| `thetriangle-wordpress` (VM 100) | `10.248.40.141` | Legacy WordPress, source of media and exports | Public entry point is **https://delta.thetriangle.org** via Cloudflare Tunnel. The backend reaches the database through MaxScale (`DB_HOST=10.248.40.183`, @@ -26,21 +25,20 @@ GHCR images into the idle slot and flips `active-upstreams.conf`. Check the live slot with `sudo cat /etc/nginx/triangle-cms/active-upstreams.conf`. Access is `ssh tadmin@` with key auth. **`tadmin` has `(ALL) NOPASSWD: ALL` -on all three hosts**, so root operations need no password — verify with -`sudo -n -l`. The nginx-only NOPASSWD entries on Delta -(`nginx -t`, `nginx -s reload`) belong to the **`triangle-runner`** service -account, not to `tadmin`; do not confuse the two. +on all three hosts**; verify with `sudo -n -l`. The nginx-only NOPASSWD entries +on Delta (`nginx -t`, `nginx -s reload`) belong to the **`triangle-runner`** +service account, not to `tadmin`. -A password is still required for SSH *password* login (`PasswordAuthentication` -is on), the Proxmox VM console, and `su -` to root — that last one is root's own -credential, not `tadmin`'s. +A password is still required for SSH *password* login +(`PasswordAuthentication` is on), the Proxmox VM console, and `su -` to root. +That last one is root's own credential, not `tadmin`'s. --- ## 2. Current data state -Loaded 2026-08-01 from a WordPress export taken the same day, and verified -checksum-identical to the ETL output (see §4). +Loaded 2026-08-01 from a WordPress export taken the same day, verified +checksum-identical to the ETL output (§4). | | | |---|---| @@ -55,36 +53,32 @@ Integrity: 0 orphan links, 0 duplicate links, 0 articles whose byline disagrees with the linked author, 0 mojibake, 0 `photo_url` sentinels. All 6,757 featured images point at `delta.thetriangle.org`. -`cms_users` starts empty by design — the first OIDC login creates and promotes +`cms_users` starts empty by design; the first OIDC login creates and promotes the account (`CMS_AUTO_PROMOTE_ALL_ADMINS=true`). -### Things that look like bugs but are not +### Not bugs -- **2,623 published articles have no byline.** They genuinely have no author in - WordPress, mostly pre-2019. This is the source data, not a migration failure. -- **22 featured images 404.** They are absent from the WordPress origin too — - they 404 on the live WP site. Nothing to recover. -- **`site_taxonomy` counts and section listings both come from the same - matcher**, so they agree. If you add a section, its slug must match the - category text in the articles (see §5). +- **2,623 published articles have no byline.** They have no author in + WordPress either, mostly pre-2019. +- **22 featured images 404.** They 404 on the live WP site too. Nothing to + recover. +- **`site_taxonomy` counts and section listings use the same matcher**, so they + agree. A new section's slug must match the category text in the articles (§5). --- ## 3. Rebuilding the data (ETL → reseed) -Everything below runs from a `wordpress-etl` checkout on a workstation, not on -the servers. +Runs from a `wordpress-etl` checkout on a workstation, not on the servers. ### 3.1 Getting an export -WordPress does not produce a zip — you build one from **two separate exports**: +WordPress does not produce a zip. Build one from **two separate exports**: -1. **Tools → Export → All content** (or Posts) → save as `wp-posts.xml` -2. **Tools → Export →** the Co-Authors Plus **Guest Authors** post type → save +1. **Tools → Export → All content** (or Posts), saved as `wp-posts.xml` +2. **Tools → Export →** the Co-Authors Plus **Guest Authors** post type, saved as `wp-guestAuths.xml` -Then: - ```bash mkdir -p wp-export mv .xml wp-export/wp-posts.xml @@ -94,9 +88,9 @@ mv wp-export.zip wordpress-etl/Data/wp-export.zip ``` The path is hardcoded (`Utils/Constants.py`). Members are matched by filename -prefix and the match is **case-sensitive** — `wp-guestAuths`, capital A. +prefix, **case-sensitively**: `wp-guestAuths`, capital A. -Do **not** reuse the All-content export for both members: the guest-author +Do **not** reuse the All-content export for both members. The guest-author parser does not filter by post type and would run every post through the author path. @@ -108,11 +102,11 @@ MEDIA_BASE_URL=https://delta.thetriangle.org \ .venv/bin/python main.py --generate-embeddings --best-guess --headless ``` -Takes a few minutes; embeddings are ~1 minute for 10k articles. `--headless` +A few minutes total; embeddings are ~1 minute for 10k articles. `--headless` requires `--best-guess`. `MEDIA_BASE_URL` is baked into `photo_url` **and** into inline `` tags in -article bodies. It only changes on a re-run — editing the env var without +article bodies. It only changes on a re-run: editing the env var without reseeding leaves every image pointing at the old host. ### 3.3 Loading into the database @@ -131,18 +125,18 @@ utf8mb4_uca1400_ai_ci;` first is safe. > **`--default-character-set=utf8mb4` is not optional.** DB1's client charset > defaults to **latin1**, so a bare `mariadb` double-encodes every multibyte -> character — `“` is stored as `“` across the whole corpus. Worse, reading it -> back through the same bare client converts it a second time and prints a +> character: `“` is stored as `“` across the whole corpus. Reading it back +> through the same bare client converts it a second time and prints a > correct-looking `“`, so the obvious check passes. Verify with > `SELECT HEX(...)` and expect `E2809C`. Then: -1. **Restart the backends** — `site_taxonomy` and the other CMS tables are - created at runtime by the Go code, so restoring taxonomy before this fails - with `Table 'triangle.site_taxonomy' doesn't exist`. -2. **Restore `site_taxonomy`** from a dump (it is CMS config, not ETL output — - a reseed wipes it, and empty taxonomy makes every section page 404). +1. **Restart the backends.** `site_taxonomy` and the other CMS tables are + created at runtime by the Go code, so restoring taxonomy first fails with + `Table 'triangle.site_taxonomy' doesn't exist`. +2. **Restore `site_taxonomy`** from a dump. It is CMS config, not ETL output; a + reseed wipes it, and empty taxonomy 404s every section page. 3. **Restart again** so the counts rebuild. ```bash @@ -154,7 +148,7 @@ ssh tadmin@10.248.40.168 'docker restart triangle-cms-backend-blue-1 triangle-cm ### 3.4 Always check for errors `mariadb` prints nothing on success, and a failure in one file does not stop the -others. **Grep the output for `^ERROR` — do not pipe it through `head`.** A +others. **Grep the output for `^ERROR`; do not pipe it through `head`.** A failed `authors` load with everything else succeeding looks almost normal, and leaves every link pointing at authors that were never inserted. @@ -175,12 +169,10 @@ SELECT BIT_XOR(CRC32(CONCAT_WS(char(1), col1, COALESCE(col2,char(2)), ...))) FRO limit. Sentinel-encode NULLs so they cannot collide with empty strings. Drop the scratch database afterwards. -Expect `articles` to have **four columns the ETL does not emit** — -`archived_at`, `focus_keyword`, `meta_description`, `seo_title` — added at -startup by `EnsureArticlesSchema` and backfilled from the Yoast data in `seo`. -Exclude them from the comparison. - -Useful integrity queries: +`articles` has **four columns the ETL does not emit**: `archived_at`, +`focus_keyword`, `meta_description`, `seo_title`, added at startup by +`EnsureArticlesSchema` and backfilled from the Yoast data in `seo`. Exclude them +from the comparison. ```sql -- links pointing at an author that does not exist @@ -197,32 +189,31 @@ SELECT COUNT(DISTINCT ar.id) FROM articles ar --- -## 5. Gotchas worth knowing +## 5. Gotchas **The ETL keeps decision caches under `logs/`.** `auth_conflicts.json` (author merges) and `article-sanitizer/article_author_resolution_cache.json` (byline → author) persist answers across runs so unattended runs do not stop on prompts. Both used to store WordPress author **ids**, which are renumbered *and reused* -between exports — that silently credited articles to the wrong person. They now -re-resolve by name and heal themselves, but **if bylines ever look wrong after a -fresh export, these caches are the first place to look.** +between exports, silently crediting articles to the wrong person. They now +re-resolve by name and heal themselves, but **if bylines look wrong after a +fresh export, check these caches first.** **Never rename a section slug.** `/comics-puzzles` is hardcoded in Scalene (`Header.astro`, `SideMenu.astro`, `index.astro`, `siteConfig.ts`) *and* in the -CMS (`handlers.go`, homepage blocks). Renaming the slug 404s the section page -and empties its homepage block. To make a section match different category text, -add a **subsection** whose slug matches instead — a section matches itself OR -any of its children. +CMS (`handlers.go`, homepage blocks). Renaming it 404s the section page and +empties its homepage block. To make a section match different category text, add +a **subsection** whose slug matches: a section matches itself or any child. **Cloudflare caches 404s for up to 4 hours.** A newly added image can 404 -publicly while the origin serves it fine. Diagnose with -`curl -sI` and read `cf-cache-status` / `age`; `?v=` bypasses it. Always -check the origin (`ssh delta` → `curl -sI http://localhost/wp-content/...`) -before concluding a file is missing. +publicly while the origin serves it fine. Read `cf-cache-status` / `age` from +`curl -sI`; `?v=` bypasses it. Check the origin +(`ssh delta` → `curl -sI http://localhost/wp-content/...`) before concluding a +file is missing. -**Media lives on CephFS and is synced from WordPress by hand.** New images -added to WP after the last sync will 404. The rsync must be **pushed from** -`10.248.40.141` — Delta holds no key to pull: +**Media lives on CephFS and is synced from WordPress by hand.** Images added to +WP after the last sync will 404. The rsync must be **pushed from** +`10.248.40.141`; Delta holds no key to pull: ```bash ssh tadmin@10.248.40.141 'rsync -a --no-owner --no-group --chmod=D775,F664 \ @@ -231,21 +222,20 @@ ssh tadmin@10.248.40.141 'rsync -a --no-owner --no-group --chmod=D775,F664 \ tadmin@10.248.40.168:/mnt/cephfs/media/wp-content/uploads/2026/' ``` -**`D775`, not `D755` — the group bit is load-bearing.** The CMS backend runs as -uid 10001 and owns none of this tree, so its write access comes from a POSIX ACL +**`D775`, not `D755`: the group bit is load-bearing.** The backend runs as uid +10001 and owns none of this tree, so its write access comes from a POSIX ACL (`user:10001:rwx` plus the `default:` entries new directories inherit). A chmod on an ACL'd directory sets the **ACL mask** from the mode's group bits, so -`D755` silently clamps that entry to `#effective:r-x`. `getfacl` still prints +`D755` clamps that entry to `#effective:r-x`. `getfacl` still prints `user:10001:rwx` and `ls` shows only a `+`, so it reads as correct. Uploads then keep working until the 1st of the next month, when the handler -first has to create a new `YYYY/MM` directory and can't — `POST /v1/media` +first has to create a new `YYYY/MM` directory and cannot: `POST /v1/media` returns 500 on **every** upload with `mkdir .../uploads/YYYY/MM: permission -denied` in the backend log. That is exactly the 2026-09-01 outage. +denied` in the backend log. That was the 2026-09-01 outage. -`D775` keeps the mask at `rwx` and matches the `0775` the CMS itself creates -month directories with. Verify after any sync — the mask, not the entry, is -what decides: +`D775` keeps the mask at `rwx` and matches the `0775` the CMS creates month +directories with. Verify after any sync, reading the mask rather than the entry: ```bash getfacl -pc /mnt/cephfs/media/wp-content/uploads/2026 | grep -E '10001|mask::' @@ -253,10 +243,10 @@ getfacl -pc /mnt/cephfs/media/wp-content/uploads/2026 | grep -E '10001|mask::' ``` `ansible-playbook playbooks/delta-host.yml` in `triangle-infrastructure` repairs -a clamped mask, but only on the next run — the fix belongs in the command above. +a clamped mask, but only on its next run; the fix belongs in the rsync above. -It exits **23** with "failed to set times" on one directory. That is a -directory mtime owned by another uid, not a data failure — ignore it. +rsync exits **23** with "failed to set times" on one directory. That is a +directory mtime owned by another uid, not a data failure. Ignore it. **Merged PRs do not always contain what you think.** Twice during this work a PR was auto-closed as "merged" against another branch's merge commit, leaving later @@ -270,54 +260,40 @@ git fetch origin && git merge-base --is-ancestor origin/main && echo IN MA ## 6. Open items -### 6.1 No automated backups, and DB1 is the only copy — **highest priority** +### 6.1 No automated backups, and DB1 is the only copy (highest priority) There is no replica (DB2 was never provisioned) and no scheduled dump: no cron -entry, no systemd timer. The only backups are manual dumps in -`~tadmin/` on DB1, taken by hand. Losing DB1 loses everything since the last one. +entry, no systemd timer. The only backups are manual dumps in `~tadmin/` on DB1. +Losing DB1 loses everything since the last one. A nightly dump is a few lines; a replica is the real fix. -### 6.2 Delta's root filesystem — RESOLVED 2026-08-02 +### 6.2 Delta's root filesystem: RESOLVED 2026-08-02 -`/` was 15 GB with ~4.7 GB free (67%), which is tight for blue/green: it keeps -two slots plus old tags, and 54 images were on disk with only 4 in use. - -**Done — the volume was grown.** The volume group turned out to be only half -allocated (`VSize <30.00g`, `VFree 15.00g`), so 15 GB of unused extents were -already attached to the VM. No Proxmox change and no reboot were needed; ext4 -resizes while mounted: +`/` was 15 GB with 4.7 GB free, tight for blue/green. The volume group was only +half allocated, so 15 GB of unused extents were already attached to the VM and +ext4 resized while mounted, with no Proxmox change and no reboot: ```bash sudo lvextend -l +100%FREE /dev/ubuntu-vg/ubuntu-lv sudo resize2fs /dev/ubuntu-vg/ubuntu-lv ``` -| | before | after | -|---|---|---| -| LV size | 15.00 GiB | 30.00 GiB | -| VG free extents | 15.00 GiB | 0 | -| `/` free | 4.7 GB (67% used) | 19 GB (34% used) | +`/` is now 30 GiB with 19 GB free (34% used); all four containers stayed healthy +throughout. **The VG is now fully allocated**, so growing further means adding a +disk in Proxmox and `pvcreate`/`vgextend` first. -Filesystem state `clean` afterwards, all four containers stayed healthy, and -`/healthz`, `/v1/health/db` and `/v1/homepage` all returned 200 throughout. - -**The VG is now fully allocated**, so this trick is spent — growing further -means adding a disk in Proxmox and `pvcreate`/`vgextend` first. - -Still worth doing as hygiene, though no longer urgent at 19 GB free: prune old -images on a weekly timer, or as a step at the end of a successful deploy. +Still worth doing as hygiene: prune old images weekly, or at the end of a +successful deploy. ```bash docker image prune -af --filter 'until=168h' # ~1.2 GB reclaimable today ``` -`tadmin` has `NOPASSWD: ALL` on Delta, so this runs over SSH with key auth. +### 6.3 CephFS is mounted with cluster-admin credentials on the CI box -### 6.3 CephFS is mounted with full cluster-admin credentials on the box that runs CI - -**What is set up.** `/mnt/cephfs` is mounted from `/etc/fstab` using the -**`client.admin`** CephX identity — the Ceph cluster's root-equivalent account: +`/mnt/cephfs` is mounted from `/etc/fstab` using the **`client.admin`** CephX +identity, the Ceph cluster's root-equivalent account: ``` admin@.cephfs-pve3=/ /mnt/cephfs ceph _netdev,nofail,noatime, @@ -325,30 +301,28 @@ admin@.cephfs-pve3=/ /mnt/cephfs ceph _netdev,nofail,noatime, ``` The secret is at `/etc/ceph/admin.keyring`, mode `0600 root:root`. The same host -also runs `actions.runner.DrexelTriangle-triangle-cms.thetriangle-delta.service` -as user `triangle-runner`. +runs `actions.runner.DrexelTriangle-triangle-cms.thetriangle-delta.service` as +user `triangle-runner`. -**Why this matters.** `client.admin` is not scoped to our media directory — it -grants administrative control of the **entire Proxmox Ceph cluster**: every -pool, every other VM's disks, all 7.1 TB. The media tree we actually use is one -subdirectory of it. +`client.admin` is not scoped to our media directory. It grants administrative +control of the **entire Proxmox Ceph cluster**: every pool, every other VM's +disks, all 7.1 TB. The media tree we use is one subdirectory of it. -The runner cannot read the keyring directly (it is `0600 root`) and its sudo is -limited to `nginx -t` / `nginx -s reload`. **But `triangle-runner` is in the -`docker` group, and docker group membership is root-equivalent** — any process -running as that user can start a container that bind-mounts `/` and read the -keyring as root. So the practical chain is: +The runner cannot read the keyring directly and its sudo is limited to +`nginx -t` / `nginx -s reload`. **But `triangle-runner` is in the `docker` group, +and docker group membership is root-equivalent**: any process running as that +user can start a container that bind-mounts `/` and read the keyring as root. > code running in a GitHub Actions job → `triangle-runner` → `docker` group → > root on Delta → `/etc/ceph/admin.keyring` → **admin of the whole Ceph cluster** The first link is ordinary CI. Any workflow change, compromised dependency, or misconfigured `pull_request` trigger that executes attacker-controlled code -reaches cluster admin. Nothing here is currently known to be compromised — this -is about blast radius being far larger than the job requires. +reaches cluster admin. Nothing here is known to be compromised; the problem is +blast radius far larger than the job requires. -**Suggested fix.** Ask the Proxmox administrator for a CephX identity scoped to -the media path, and mount with that instead. Roughly: +**Fix.** Ask the Proxmox administrator for a CephX identity scoped to the media +path and mount with that instead: ```bash ceph fs authorize cephfs-pve3 client.triangle-media /media rw @@ -356,30 +330,27 @@ ceph fs authorize cephfs-pve3 client.triangle-media /media rw which yields caps along the lines of `mon 'allow r'`, `mds 'allow rw path=/media'`, `osd 'allow rw tag cephfs data=cephfs-pve3'`. -Then write that key to `/etc/ceph/triangle-media.keyring` (`0600 root:root`), -change the fstab source from `admin@...` to `triangle-media@...`, point -`secretfile=` at the new file, remount, and **remove -`/etc/ceph/admin.keyring` from Delta entirely**. - -After that, the worst case for a compromised runner is read/write to the media -directory it already serves, instead of the whole cluster. +Write that key to `/etc/ceph/triangle-media.keyring` (`0600 root:root`), change +the fstab source from `admin@...` to `triangle-media@...`, point `secretfile=` +at the new file, remount, and **remove `/etc/ceph/admin.keyring` from Delta**. -Worth doing alongside: move the runner off the box that holds storage -credentials, or drop `triangle-runner` from the `docker` group and give it a -narrower deploy path. +The worst case for a compromised runner is then read/write on the media +directory it already serves. Worth doing alongside: move the runner off the box +holding storage credentials, or drop `triangle-runner` from the `docker` group +and give it a narrower deploy path. ### 6.4 DB host addresses sit in a DHCP pool `10.248.40.154` and `10.248.40.183` came from Drexel's dynamic pool; the old -leases lapse around 2026-08-05. Both containers are pinned static at the -Proxmox *and* in-container layers, so they will not lose their addresses — but -nothing stops the DHCP server handing the same address to a different device -later. DHCP is central Drexel (server `10.254.5.41`); Proxmox is not the DHCP -server, so no reservation can be made from our side. - -**This risk was reviewed and deliberately accepted** — other production VMs run -on pool addresses and have never collided. Revisit only on unexplained -intermittent DB connection failures; first diagnostic is +leases lapse around 2026-08-05. Both containers are pinned static at the Proxmox +*and* in-container layers, so they will not lose their addresses, but nothing +stops the DHCP server handing the same address to a different device later. DHCP +is central Drexel (server `10.254.5.41`), so no reservation can be made from our +side. + +**This risk was reviewed and accepted**; other production VMs run on pool +addresses and have never collided. Revisit on unexplained intermittent DB +connection failures. First diagnostic is `arping -I vmbr0 -c 2 10.248.40.154` from the Proxmox host (expect MAC `bc:24:11:e5:cc:58`). diff --git a/embeddings/app.py b/embeddings/app.py index a7c568e..9a8eeb6 100644 --- a/embeddings/app.py +++ b/embeddings/app.py @@ -6,8 +6,8 @@ meaningless, so both go through this one service and it reports which model produced them. -It is deliberately stateless. Nothing here is a source of truth -- the vectors -live in MariaDB -- so it needs no volume, no backup, and no reconciliation. If +It is deliberately stateless. Nothing here is a source of truth (the vectors +live in MariaDB) so it needs no volume, no backup, and no reconciliation. If it restarts, or is missing entirely, the CMS degrades to lexical search. """ @@ -27,7 +27,7 @@ # BGE is an asymmetric retrieval model: it was trained with short queries # prefixed and documents bare. Embedding a query without this prefix quietly -# costs a chunk of retrieval quality -- it still returns vectors, just worse +# costs a chunk of retrieval quality: it still returns vectors, just worse # ones, which is the kind of bug that never surfaces as an error. QUERY_PREFIX = os.getenv( "EMBED_QUERY_PREFIX", @@ -96,7 +96,7 @@ def embed(request: EmbedRequest) -> EmbedResponse: # Normalize explicitly rather than trusting the model wrapper's default. # MariaDB ranks these with euclidean distance, which only agrees with cosine - # similarity on unit vectors -- the previous ETL skipped this, so magnitude + # similarity on unit vectors. The previous ETL skipped this, so magnitude # leaked into every "related articles" ranking. norms = np.linalg.norm(vectors, axis=1, keepdims=True) vectors = vectors / np.clip(norms, 1e-12, None) diff --git a/frontend/README.md b/frontend/README.md index 8c2d760..403291f 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -1,73 +1,20 @@ -# React + TypeScript + Vite +# Triangle CMS dashboard -This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules. +React + TypeScript + Vite. The editor dashboard: articles, media, taxonomy, +polls, classifieds, settings. -Currently, two official plugins are available: +Setup and running it against a local backend are in the +[repo README](../README.md). -- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react) uses [Babel](https://babeljs.io/) (or [oxc](https://oxc.rs) when used in [rolldown-vite](https://vite.dev/guide/rolldown)) for Fast Refresh -- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh - -## React Compiler - -The React Compiler is not enabled on this template because of its impact on dev & build performances. To add it, see [this documentation](https://react.dev/learn/react-compiler/installation). - -## Expanding the ESLint configuration - -If you are developing a production application, we recommend updating the configuration to enable type-aware lint rules: - -```js -export default defineConfig([ - globalIgnores(["dist"]), - { - files: ["**/*.{ts,tsx}"], - extends: [ - // Other configs... - - // Remove tseslint.configs.recommended and replace with this - tseslint.configs.recommendedTypeChecked, - // Alternatively, use this for stricter rules - tseslint.configs.strictTypeChecked, - // Optionally, add this for stylistic rules - tseslint.configs.stylisticTypeChecked, - - // Other configs... - ], - languageOptions: { - parserOptions: { - project: ["./tsconfig.node.json", "./tsconfig.app.json"], - tsconfigRootDir: import.meta.dirname, - }, - // other options... - }, - }, -]); +```bash +npm run dev -- --port 5173 # dev server (strict port 5173) +npm run build # tsc -b + vite build +npm run lint +npm test # vitest ``` -You can also install [eslint-plugin-react-x](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-x) and [eslint-plugin-react-dom](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-dom) for React-specific lint rules: - -```js -// eslint.config.js -import reactX from "eslint-plugin-react-x"; -import reactDom from "eslint-plugin-react-dom"; - -export default defineConfig([ - globalIgnores(["dist"]), - { - files: ["**/*.{ts,tsx}"], - extends: [ - // Other configs... - // Enable lint rules for React - reactX.configs["recommended-typescript"], - // Enable lint rules for React DOM - reactDom.configs.recommended, - ], - languageOptions: { - parserOptions: { - project: ["./tsconfig.node.json", "./tsconfig.app.json"], - tsconfigRootDir: import.meta.dirname, - }, - // other options... - }, - }, -]); -``` +API calls go through `useApiFetch`, which prefixes `VITE_API_BASE_URL`. That is +empty by default, so requests are same-origin and the dev server proxies `/v1` +to `https://localhost:8080` with `secure: false`, which is what lets the local +backend keep its self-signed certificate. `VITE_AUTH_BASE_URL` and +`VITE_PUBLIC_SITE_URL` are the other two overrides (see `src/auth/urls.ts`). diff --git a/frontend/nginx.conf b/frontend/nginx.conf index 076409e..4092fe1 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -40,7 +40,7 @@ server { } # index.html carries the references to those hashed filenames, so it is the - # one file that must never be cached -- a stale copy points the browser at + # one file that must never be cached: a stale copy points the browser at # assets the last deploy removed. location = /index.html { add_header Cache-Control "no-cache"; diff --git a/frontend/src/auth/adminOnlyNoticeContext.ts b/frontend/src/auth/adminOnlyNoticeContext.ts index 293b600..94cdfa5 100644 --- a/frontend/src/auth/adminOnlyNoticeContext.ts +++ b/frontend/src/auth/adminOnlyNoticeContext.ts @@ -11,7 +11,7 @@ export const AdminOnlyNoticeContext = createContext /** * Access to the shared "admin only" dialog. * - * Unlike useSessionAuth this does not throw when no provider is mounted -- it + * Unlike useSessionAuth this does not throw when no provider is mounted; it * degrades to a no-op. useApiFetch calls it on every request, and a missing * provider should not take down a page (or a test) that never triggers a 403. */ diff --git a/frontend/src/auth/urls.ts b/frontend/src/auth/urls.ts index 7bb25ce..4a74643 100644 --- a/frontend/src/auth/urls.ts +++ b/frontend/src/auth/urls.ts @@ -21,7 +21,7 @@ export function publicSiteUrl() { // The public permalink for a slug. The URL is fully determined by the slug, so // it can be handed out (newsletter, social scheduling) before the article is -// published -- it 404s until then, and resolves the moment it goes live. +// published. It 404s until then, and resolves the moment it goes live. export function articleUrl(slug: string) { return `${publicSiteUrl()}/article/${encodeURIComponent(slug)}` } diff --git a/frontend/src/components/MediaPicker.tsx b/frontend/src/components/MediaPicker.tsx index d3e4608..fdb4c73 100644 --- a/frontend/src/components/MediaPicker.tsx +++ b/frontend/src/components/MediaPicker.tsx @@ -23,7 +23,7 @@ type MediaPickerProps = { // When set, the picker also accepts a bare image URL. Only the featured-image // field wants this: an article's photo_url is served verbatim, so it can point // at an image that was never in our library. Body attachments have no such - // escape hatch by design -- they get sideloaded so articles never hotlink. + // escape hatch by design: they get sideloaded so articles never hotlink. onUseUrl?: (url: string) => void initialUrl?: string } @@ -147,7 +147,7 @@ function MediaPicker({ onSelect, onClose, title = "Insert image", onUseUrl, init }, [sentinel, hasMore, isLoading, isLoadingMore, items.length]) // Alt text saved from a tile belongs to the library record, so the grid has to - // show the new value straight away -- otherwise the "No alt text" warning + // show the new value straight away; otherwise the "No alt text" warning // stays up on an image that now has some. const updateItem = useCallback((id: number, altText: string) => { setItems((prev) => prev.map((item) => (item.id === id ? { ...item, alt_text: altText } : item))) @@ -344,7 +344,7 @@ type AltTextFieldProps = { * Inline alt-text editor for one tile. * * It writes to the library record (PATCH /v1/media/{id}), not to this article, - * so an image described once is described everywhere it is used -- the same + * so an image described once is described everywhere it is used, the same * contract the picker already relies on when it hands alt_text to the editor. * * Its own error state rather than the picker's banner: a failed save belongs diff --git a/frontend/src/components/TrixEditor.css b/frontend/src/components/TrixEditor.css index 48f480a..d934f28 100644 --- a/frontend/src/components/TrixEditor.css +++ b/frontend/src/components/TrixEditor.css @@ -285,7 +285,7 @@ trix-editor.trix-content .attachment__caption .attachment__size { /* An image with no caption yet. Trix marks the empty
with data-trix-placeholder ("Add a caption…") but ships no CSS for it, so the - element collapses to zero height -- invisible, and with nothing to click. + element collapses to zero height: invisible, and with nothing to click. Rendering the placeholder gives the author both the prompt and the target. */ trix-editor.trix-content .attachment__caption[data-trix-placeholder]:empty::before { content: attr(data-trix-placeholder); @@ -447,8 +447,8 @@ trix-editor.trix-content .attachment__toolbar .attachment__metadata-container { /* Remove. Trix draws this as a white circle with a 2px `highlight` (system accent) border and hides the text label behind text-indent, showing an ✕ via - a ::before background image. Keep the icon -- undoing the text-indent would - leave the button empty -- and drop only the standalone-circle chrome, which + a ::before background image. Keep the icon (undoing the text-indent would + leave the button empty) and drop only the standalone-circle chrome, which looks wrong now that it sits inside a grouped toolbar. */ trix-editor.trix-content .attachment__toolbar .trix-button--remove { position: relative; @@ -476,8 +476,8 @@ trix-editor.trix-content .attachment.attachment--preview.attachment--dragging { cursor: grabbing; } -/* Per-image alignment. No longer settable from the editor -- the public site - sizes article images at width:100%, which leaves nothing for a float to do -- +/* Per-image alignment. No longer settable from the editor, since the public + site sizes article images at width:100%, which leaves nothing for a float to do, but alignment already present on migrated WordPress content is preserved through a save, so it is still reflected here to match what will publish. @@ -486,7 +486,7 @@ trix-editor.trix-content .attachment.attachment--preview.attachment--dragging { as far as Trix's MutationObserver is concerned: it re-parses the document, re-renders it and restores its own selection, which yanks the caret (and the focus) back into the article from wherever the author had moved it. Doing - that on every trix-change -- i.e. on every keystroke -- made the editor + that on every trix-change, i.e. on every keystroke, made the editor un-leavable while an image was selected. */ trix-editor.trix-content .attachment--preview[data-trix-attributes*='"align":"left"'] { text-align: left; diff --git a/frontend/src/components/TrixEditor.tsx b/frontend/src/components/TrixEditor.tsx index 80ed961..4c3de6c 100644 --- a/frontend/src/components/TrixEditor.tsx +++ b/frontend/src/components/TrixEditor.tsx @@ -13,8 +13,8 @@ import { // Leave the caption area of an image empty until it has a real caption. Trix's // default is to fill it with the filename and file size, which reads as a -// caption the author didn't write, and -- because the slot is then never empty -// -- suppresses the "Add a caption…" placeholder that tells them it is editable. +// caption the author didn't write, and (because the slot is then never empty) +// suppresses the "Add a caption…" placeholder that tells them it is editable. // Non-previewable file attachments are unaffected: Trix forces the name on for // those, and a file stub with no label would be nothing at all. if (typeof window !== "undefined" && window.Trix) { @@ -35,8 +35,8 @@ if (typeof window !== "undefined" && window.Trix) { // ["caption", "presentation"] and removeProhibitedAttributes drops everything // else as soon as the piece is built, so `align` only ever lived on the live // figure as a class our own effect re-applied. That class was enough to make - // alignment look preserved until Trix re-rendered the figure from the piece - // -- which editing a caption does -- at which point the alignment silently + // alignment look preserved until Trix re-rendered the figure from the piece, + // which editing a caption does, at which point the alignment silently // vanished from both the editor and the saved article. if (!window.Trix.AttachmentPiece.permittedAttributes.includes("align")) { window.Trix.AttachmentPiece.permittedAttributes.push("align"); @@ -97,7 +97,7 @@ function TrixEditor({ value, onChange }: TrixEditorProps) { const editorRef = useRef(null); const wrapperRef = useRef(null); // Every HTML string we have emitted since the last load from outside, so we - // never call loadHTML on our own output -- which would reset the document and + // never call loadHTML on our own output, which would reset the document and // drop the caret at the top of the article mid-edit. // // This has to be a set of everything emitted, not just the most recent one. @@ -173,7 +173,7 @@ function TrixEditor({ value, onChange }: TrixEditorProps) { // Trix decides an attachment is no longer being edited by watching the // document's selection: on the first selectionchange whose range is not the // attachment's own, it stops editing, tears the caption field down and - // re-renders -- and that render puts the selection, and with it the browser's + // re-renders, and that render puts the selection, and with it the browser's // focus, back inside the article. While an image sat in that state the check // was still armed after the author had clicked away, and Firefox raises // selectionchange on the document for typing in a plain (Chrome @@ -185,13 +185,13 @@ function TrixEditor({ value, onChange }: TrixEditorProps) { // asking Trix to stop editing the attachment runs the same teardown and steals // the focus just as thoroughly. A selection left inside a contenteditable // nobody is focused on has nothing to draw and nothing to move, so clearing it - // costs the author nothing -- Trix remembers the caret separately and puts it - // back when the editor is focused again -- and it leaves Trix reading no range + // costs the author nothing (Trix remembers the caret separately and puts it + // back when the editor is focused again) and it leaves Trix reading no range // at all rather than a stale one, which is what makes the check fall through. // // focusout rather than trix-blur: while a caption is open the focus is on the // caption field, not on the editor element, so the editor never blurs and - // trix-blur never comes -- and that is exactly the state this is here for. + // trix-blur never comes, and that is exactly the state this is here for. // Where the focus went is read a tick later, since focusout fires before it // has landed. Measured against the wrapper, not the editor, so that stepping // into our own chrome (the alt-text dialog, the media picker) is not treated @@ -228,7 +228,7 @@ function TrixEditor({ value, onChange }: TrixEditorProps) { // mutation inside Trix's contenteditable: its MutationObserver re-parsed the // document, re-rendered it, and restored its own selection with it. Doing // that on every trix-change meant every keystroke in an image caption pulled - // the caret -- and the browser's focus -- back into the article, so clicking + // the caret, and the browser's focus, back into the article, so clicking // out to the author or section search box and typing put the first letter in // the search box and the rest back in the story. @@ -288,7 +288,7 @@ function TrixEditor({ value, onChange }: TrixEditorProps) { const { url } = JSON.parse(xhr.responseText) as { url: string }; // href only for non-images. Trix wraps an attachment carrying an href // in an , which for a previewable image swallows every click on the - // figure -- including the caption field. A file stub, by contrast, has + // figure, including the caption field. A file stub, by contrast, has // nothing to edit and a download link is the whole point of it. attachment.setAttributes( file.type.startsWith("image/") ? { url } : { url, href: url }, @@ -421,7 +421,7 @@ function TrixEditor({ value, onChange }: TrixEditorProps) { } openBlockForImage(); - // Deliberately no href -- see uploadFile. The saved article is a plain + // Deliberately no href; see uploadFile. The saved article is a plain //
either way, so the link would only ever have existed inside // the editor, where it fights with selecting and captioning the image. const attachment = new Trix.Attachment({ @@ -474,7 +474,7 @@ function TrixEditor({ value, onChange }: TrixEditorProps) { // Article-scoped by design: the attachment carries no library id (nothing // survives a reload but the URL), and an image can legitimately want a // different description in a different story. Setting the library's own alt - // text -- the value every future insertion starts from -- is the picker's job. + // text, the value every future insertion starts from, is the picker's job. const saveAltText = useCallback((attachmentId: number, alt: string) => { setAltEditor(null); const editor = editorRef.current; @@ -505,7 +505,7 @@ function TrixEditor({ value, onChange }: TrixEditorProps) { // which overrides anything an author sets, so both were controls that appeared // to work in the editor and changed nothing on the published page. Reordering // is kept because it does survive. Alignment already stored on legacy - // WordPress content is still preserved through a save -- it just can no longer + // WordPress content is still preserved through a save; it just can no longer // be set from here. useEffect(() => { const editor = editorRef.current; @@ -572,13 +572,13 @@ function TrixEditor({ value, onChange }: TrixEditorProps) { return null; }; - // Where the dragged block would land, as a boundary index -- or null if it + // Where the dragged block would land, as a boundary index, or null if it // cannot go anywhere from here. // // Only boundaries clear of the dragged block itself are candidates: the two // touching it leave the document exactly as it was. They used to be offered, // and since an image block is as tall as the image, the pointer spent the - // whole of a short drag inside them -- the indicator appeared, tracked the + // whole of a short drag inside them: the indicator appeared, tracked the // pointer, and then the drop did nothing. Moving an image at all meant // dragging clear past its own block and half of the next one, with the // indicator claiming otherwise the entire way. That is the bug this shape @@ -638,8 +638,8 @@ function TrixEditor({ value, onChange }: TrixEditorProps) { }; // Re-resolve a figure by id. The element captured when a gesture started may - // already be detached: selecting an attachment -- which Trix does on the very - // mousedown that starts a drag -- re-renders it, so an identical figure has + // already be detached: selecting an attachment, which Trix does on the very + // mousedown that starts a drag, re-renders it, so an identical figure has // taken its place. Anything that asks "which block is this image in?" has to // go through here or it gets the answer for a corpse. const liveFigure = (figure: HTMLElement): HTMLElement | null => { @@ -659,7 +659,7 @@ function TrixEditor({ value, onChange }: TrixEditorProps) { : -1; }; - // Put an attachment into Trix's "being edited" state -- the state that + // Put an attachment into Trix's "being edited" state, the one that // shows its caption field and its toolbar. Trix only enters it from a // mousedown of its own, which is no help either when that mousedown's // selection gets reset (see onUp) or after a move has replaced the element. @@ -679,8 +679,8 @@ function TrixEditor({ value, onChange }: TrixEditorProps) { // Move the figure's whole block to sit before or after the target block. // // This reorders Trix's serialized HTML and reloads it, rather than splicing - // the live document through Trix's mutation APIs. The obvious approach -- - // remove the attachment, then insert it at a captured DOM Range -- cannot + // the live document through Trix's mutation APIs. The obvious approach, + // removing the attachment then inserting it at a captured DOM Range, cannot // work: the Attachment objects on editor.getDocument() are plain models // with no remove() of their own (only the ManagedAttachment passed to // trix-attachment-add has one), and any removal re-renders the blocks the @@ -721,7 +721,7 @@ function TrixEditor({ value, onChange }: TrixEditorProps) { // Swap the document in under a recorded undo entry, rather than through // loadHTML. loadHTML routes to Editor#loadSnapshot, which replaces the - // whole UndoManager -- so a move was not merely un-undoable, it silently + // whole UndoManager, so a move was not merely un-undoable: it silently // threw away every undo step the author had built up before it. // recordUndoEntry snapshots the current document and selection onto the // stack, and Composition#setDocument then mutates without disturbing it. @@ -738,8 +738,8 @@ function TrixEditor({ value, onChange }: TrixEditorProps) { editor.editor.composition.setDocument(movedDocument); // Re-select the image at its new home. Without this every nudge costs the - // author the selection -- and with it the toolbar they are clicking -- - // so moving an image three blocks would mean three round trips to it. + // author the selection, and with it the toolbar they are clicking, so + // moving an image three blocks would mean three round trips to it. // // Two frames, not one. One frame is enough for the element to exist, but // not for Trix to have finished rendering the reloaded document, and @@ -840,12 +840,12 @@ function TrixEditor({ value, onChange }: TrixEditorProps) { // This is the whole reason the gesture is on pointer events: with plain // mouse events, anything that starts a native drag session mid-gesture // (see the dragstart guard below) makes the browser stop delivering - // mousemove and -- fatally -- mouseup, so the release was never seen and + // mousemove and, fatally, mouseup, so the release was never seen and // nothing was ever committed. Capture is retargeted to the wrapper // rather than the figure because Trix re-renders the figure during a // drag; capturing on an element that then leaves the document drops the // capture with it. Capture is taken here and not on pointerdown so that - // a plain click still reaches Trix untouched -- capturing at pointerdown + // a plain click still reaches Trix untouched: capturing at pointerdown // retargets the compatibility mousedown too, which is the event Trix // selects the attachment from. try { @@ -859,7 +859,7 @@ function TrixEditor({ value, onChange }: TrixEditorProps) { liveFigure(figure)?.classList.add("attachment--dragging"); document.body.style.cursor = "grabbing"; // Drop whatever text selection the un-prevented pointerdown started, so - // the drag doesn't paint a selection highlight across the article -- and + // the drag doesn't paint a selection highlight across the article, and // so there is no selection left for the browser to want to drag. window.getSelection()?.removeAllRanges(); scroller = scrollContainer(); @@ -904,7 +904,7 @@ function TrixEditor({ value, onChange }: TrixEditorProps) { dropIndicator.style.display = "none"; }; - // The pointer was taken away mid-gesture -- a touch turning into a scroll, + // The pointer was taken away mid-gesture: a touch turning into a scroll, // the window losing the device. Abandon the move rather than committing to // wherever the indicator happened to be. const onCancel = (cancelEvent: PointerEvent) => { @@ -917,7 +917,7 @@ function TrixEditor({ value, onChange }: TrixEditorProps) { if (upEvent.pointerId !== pointerId) return; activeGestureCleanup = null; // Read before cleanup, which clears `dragging` to stop the auto-scroll - // loop -- reading after it would make every drop look like a click. + // loop. Reading after it would make every drop look like a click. const wasDragging = dragging; const releasedOn = dropTarget; cleanup(); @@ -928,7 +928,7 @@ function TrixEditor({ value, onChange }: TrixEditorProps) { // Re-assert Trix's own attachment selection. Trix makes it on // mousedown, but when the click is what focuses the editor in the // first place, the focus that follows resets the selection and takes - // the attachment toolbar back down with it -- so the first click on + // the attachment toolbar back down with it, so the first click on // an image in a freshly loaded editor appeared to do nothing, and it // took a second click to get at Remove or the move buttons. Trix // ignores this when the attachment is already the one being edited. @@ -951,9 +951,9 @@ function TrixEditor({ value, onChange }: TrixEditorProps) { //