feat: media library, production DB tier, and CI gaps

.github/workflows/README.md

@@ -2,7 +2,11 @@
 
 `ci.yml` runs on pull requests and pushes without production secrets. It covers
 backend tests, race tests, `go vet`, frontend lint/build, Docker image builds,
-and Compose validation.
+Compose validation, and the deployment script test suite
+(`deploy/scripts/deploy_scripts_test.sh`). That suite stubs `docker`, `curl`, and
+`nginx` on `PATH`, so it needs no daemon or privileges — it exercises slot
+selection, the transactional Nginx switch and its restore-on-failure paths, and
+the preflight checks.
 
 `publish.yml` runs only from a successful `CI` workflow run on `main` that was
 triggered by a trusted push. It validates
@@ -14,3 +18,12 @@ VPN. Automatic deployments use the trusted publish run `head_sha`; manual
 deployments accept an already-published image SHA as data only. Deployment code
 is always checked out from the protected default branch, never from the supplied
 image SHA.
+
+`rollback.yml` is manual (`workflow_dispatch`) and switches Nginx back to the
+other slot, which is already running the previous release. It pulls no images, so
+it holds no `packages` permission. It shares the `delta-production-deploy`
+concurrency group with `deploy.yml` so a rollback can never interleave with a
+deployment. `slot: auto` targets whichever slot is currently inactive; `blue` or
+`green` names one explicitly. To recover an *older* image SHA instead, run
+`deploy.yml` manually with that SHA — rollback only moves traffic between the two
+slots that are already up.

.github/workflows/ci.yml

@@ -78,6 +78,16 @@ jobs:
           cache-from: type=gha,scope=frontend
           cache-to: type=gha,mode=max,scope=frontend
 
+  # The deployment scripts are the least reversible code in the repo, so their
+  # test suite runs on every PR. It stubs docker/curl/nginx on PATH and needs no
+  # daemon, database, or privileges.
+  deploy-scripts:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Deploy script tests
+        run: deploy/scripts/deploy_scripts_test.sh
+
   compose-validation:
     runs-on: ubuntu-latest
     steps:

.github/workflows/rollback.yml

@@ -0,0 +1,68 @@
+name: Rollback Delta
+
+# Manual only. Rollback switches Nginx back to the other slot, which is already
+# running the previous release, so it pulls no images and needs no registry
+# access. Without this workflow the only way to invoke rollback.sh is an SSH
+# session into Delta from inside the VPN — during an incident, which is exactly
+# when that is hardest.
+on:
+  workflow_dispatch:
+    inputs:
+      slot:
+        description: 'Slot to activate ("auto" picks the currently inactive one)'
+        required: true
+        default: auto
+        type: choice
+        options:
+          - auto
+          - blue
+          - green
+
+# Deliberately the SAME group as Deploy Delta, so a rollback can never interleave
+# with a deployment. deploy.sh/rollback.sh also take an exclusive flock on the
+# host, so this is belt and braces.
+concurrency:
+  group: delta-production-deploy
+  cancel-in-progress: false
+
+# No packages:read — rollback never pulls an image.
+permissions:
+  contents: read
+
+jobs:
+  rollback:
+    runs-on:
+      - self-hosted
+      - drexel-vpn
+      - delta
+      - triangle-cms
+    environment: production
+    steps:
+      - name: Checkout trusted deployment code
+        uses: actions/checkout@v4
+        with:
+          # Same trust boundary as deploy.yml: deployment code always comes from
+          # the protected default branch.
+          ref: ${{ github.event.repository.default_branch }}
+
+      - name: Switch Nginx to the target slot
+        env:
+          ENV_FILE: ${{ vars.DELTA_CMS_ENV_FILE }}
+          NGINX_ACTIVE_INCLUDE: ${{ vars.DELTA_NGINX_ACTIVE_INCLUDE }}
+          PUBLIC_BASE_URL: ${{ vars.DELTA_PUBLIC_BASE_URL }}
+          # Read into the environment rather than interpolated into the script
+          # body, so the input is data and never shell syntax.
+          SLOT: ${{ inputs.slot }}
+        run: |
+          case "${SLOT}" in
+            auto)
+              deploy/scripts/rollback.sh
+              ;;
+            blue|green)
+              deploy/scripts/rollback.sh "${SLOT}"
+              ;;
+            *)
+              echo "invalid slot: ${SLOT}" >&2
+              exit 2
+              ;;
+          esac

deploy/README.md

@@ -82,6 +82,96 @@ the generated active upstream include. The directory should be owned by
 owned by `triangle-runner:triangle-runner` with mode `0644`. The host Nginx site
 such as `/etc/nginx/sites-available/triangle-cms.conf` remains root-owned.
 
+### Installing the Nginx site
+
+Steps 6-8 above, concretely. The repo is not checked out on Delta, so copy the
+two files over first (from a workstation, at the repo root):
+
+```bash
+scp deploy/nginx/triangle-cms.conf \
+    deploy/nginx/triangle-cms-active-upstreams.conf.example \
+    <user>@<delta>:/tmp/
+```
+
+Then on Delta:
+
+```bash
+sudo cp /tmp/triangle-cms.conf /etc/nginx/sites-available/triangle-cms.conf
+sudo ln -sf /etc/nginx/sites-available/triangle-cms.conf /etc/nginx/sites-enabled/
+
+sudo install -d -o triangle-runner -g triangle-runner -m 0750 /etc/nginx/triangle-cms
+sudo install -o triangle-runner -g triangle-runner -m 0644 \
+  /tmp/triangle-cms-active-upstreams.conf.example \
+  /etc/nginx/triangle-cms/active-upstreams.conf
+
+# The stock default site also matches `server_name _` and can win the vhost pick.
+sudo rm -f /etc/nginx/sites-enabled/default
+
+sudo nginx -t && sudo systemctl reload nginx
+```
+
+Nginx will not start without `active-upstreams.conf`, since the site `include`s
+it unconditionally. A passing `nginx -t` *before* the site is enabled only
+validates the stock config and proves nothing.
+
+### Media serving
+
+`location /wp-content/` reads the migrated WordPress corpus straight off CephFS.
+It has no dependency on the containers, the runner, or the database, so it can be
+brought up on its own before the rest of the stack exists. `/` and `/v1/` return
+502 until a slot is deployed; that is expected and does not affect media.
+
+Verify the mount and that the Nginx worker user can traverse to it:
+
+```bash
+mountpoint /mnt/cephfs
+sudo -u www-data ls /mnt/cephfs/media/wp-content/uploads >/dev/null && echo ok
+```
+
+A failure there is almost always missing execute permission on a path component
+(`sudo chmod o+x /mnt/cephfs /mnt/cephfs/media`), not the Nginx config. On
+RHEL-family hosts SELinux blocks the read separately; check `ausearch -m avc -ts
+recent` and set `httpd_read_user_content`.
+
+Smoke test with a real file:
+
+```bash
+find /mnt/cephfs/media/wp-content/uploads -name '*.jpg' | head -1
+curl -I http://localhost/wp-content/uploads/YYYY/MM/name.jpg
+```
+
+Expect `200` with `Cache-Control: public, max-age=2592000, immutable`.
+
+### Media library
+
+Serving the files is independent of *listing* them. The CMS media page reads a
+`media` table, which starts empty: the rsynced corpus is on disk but unknown to
+the database. After the media rsync completes, populate it once from the CMS
+(Media -> Reindex) or directly:
+
+```bash
+curl -X POST https://localhost/v1/media/index   # admin session required
+```
+
+It walks `MEDIA_ROOT/wp-content/uploads`, skips WordPress's generated `-WxH`
+thumbnails, and inserts a row per original. It is idempotent and safe to re-run —
+already-indexed files are skipped and any alt text set in the CMS is preserved —
+so re-run it after any later out-of-band rsync. Uploads through the CMS index
+themselves and need no reindex.
+
+Note this walks the whole tree, so on a large corpus over CephFS the first run
+takes a while; run it once at cutover rather than on a schedule.
+
+### Disk
+
+Blue/green keeps two frontend and two backend images resident, plus whatever
+prior tags have not been reaped. Delta's root filesystem is small (15 GB), so
+prune before it fills:
+
+```bash
+docker image prune -af --filter 'until=168h'
+```
+
 ## Required Host Environment
 
 Copy `cms.env.example` to the private host env path and fill it with real values.
@@ -103,6 +193,12 @@ The file must contain the exact immutable image tag for the active deployment:
 - `CMS_SESSION_TTL_SECONDS`
 - `CMS_AUTO_PROMOTE_ALL_ADMINS`
 - `CMS_REBUILD_TAXONOMY_COUNTS_ON_STARTUP`
+- `MEDIA_HOST_PATH` - host path to the CephFS media tree, bind-mounted into the
+  backend. Defaults to `/mnt/cephfs/media`.
+- `MEDIA_ROOT` - the same tree as seen *inside* the container. Leave at
+  `/mnt/cephfs/media` unless the bind-mount target changes.
+- `MEDIA_BASE_URL` - public origin that serves `/wp-content/`, used to build
+  media URLs returned by the upload endpoint. Empty yields relative URLs.
 
 Keep `CMS_AUTO_PROMOTE_ALL_ADMINS=false` and
 `CMS_REBUILD_TAXONOMY_COUNTS_ON_STARTUP=false` in production. Rebuild taxonomy

deploy/cms.env.dryrun.example

@@ -0,0 +1,62 @@
+# Triangle CMS — Delta DRY RUN env template. Copy to the host-only cms.env,
+# fill the <...> placeholders, and keep it out of git.
+#
+#   scp deploy/cms.env.dryrun.example tadmin@10.248.40.168:/tmp/
+#   # on Delta: fill it in, then `chmod 600` it at its final path
+#
+# Values already filled below are specific to the dry run on Delta
+# (10.248.40.168, HTTP, no TLS, throwaway local database). Every one of them
+# changes at production cutover — see cms.env.example and README.md.
+
+# --- Images ------------------------------------------------------------------
+# Deployments use immutable full-commit-SHA tags; there is no `latest`. Set this
+# to the SHA you want to run. If nothing has been published to GHCR yet, build
+# locally on Delta instead and point these at the local image names.
+CMS_IMAGE_TAG=<full-commit-sha>
+CMS_BACKEND_IMAGE=ghcr.io/drexeltriangle/triangle-cms-backend
+CMS_FRONTEND_IMAGE=ghcr.io/drexeltriangle/triangle-cms-frontend
+
+# --- Database ----------------------------------------------------------------
+# Points at the throwaway node from compose.mariadb-dev.yml, reachable by
+# service name over triangle_net. At cutover this becomes the MaxScale endpoint
+# (port 4006) and MARIADB_ROOT_PASSWORD disappears entirely.
+DB_NAME=triangle
+DB_USER=triangle_user
+DB_PASSWORD=<generate-a-strong-password>
+DB_HOST=mariadb-dev
+DB_PORT=3306
+MARIADB_ROOT_PASSWORD=<generate-a-different-strong-password>
+
+# --- OIDC --------------------------------------------------------------------
+# The backend will NOT start without these; there are no defaults. The redirect
+# URI must be registered verbatim with the identity provider or login fails at
+# the callback. Register this exact HTTP/IP form for the dry run.
+OIDC_ISSUER_URL=<issuer-url>
+OIDC_CLIENT_ID=<client-id>
+OIDC_CLIENT_SECRET=<client-secret>
+OIDC_REDIRECT_URI=http://10.248.40.168/v1/auth/callback
+
+# --- Frontend ----------------------------------------------------------------
+# Origin the browser actually uses. Must match what Nginx serves or CORS and
+# cookies break. No trailing slash.
+FRONTEND_ORIGIN=http://10.248.40.168
+
+# --- Session / behaviour -----------------------------------------------------
+CMS_SESSION_TTL_SECONDS=604800
+# Dry run only: lets any admin-role SSO user in without manual promotion, so you
+# can actually get past the login wall. MUST be false in production.
+CMS_AUTO_PROMOTE_ALL_ADMINS=true
+# MUST stay false against the init_schema.sql seed. The rebuild reads an
+# articles.categories column (server/internal/database/taxonomy.go) that the
+# seed dump does not have, and the resulting "Unknown column 'categories'"
+# error is FATAL -- the backend crash-loops and never becomes healthy.
+CMS_REBUILD_TAXONOMY_COUNTS_ON_STARTUP=false
+
+# --- Media -------------------------------------------------------------------
+# Host Nginx already serves /wp-content/ from this tree (verified working).
+# MEDIA_BASE_URL is used to build URLs returned by the upload endpoint, and must
+# match the base the ETL used when it wrote photo_url into the seed data --
+# otherwise uploads and seeded articles disagree about where images live.
+MEDIA_HOST_PATH=/mnt/cephfs/media
+MEDIA_ROOT=/mnt/cephfs/media
+MEDIA_BASE_URL=http://10.248.40.168

deploy/cms.env.example

@@ -14,3 +14,6 @@ OIDC_REDIRECT_URI=
 CMS_SESSION_TTL_SECONDS=
 CMS_AUTO_PROMOTE_ALL_ADMINS=
 CMS_REBUILD_TAXONOMY_COUNTS_ON_STARTUP=
+MEDIA_HOST_PATH=
+MEDIA_ROOT=
+MEDIA_BASE_URL=

deploy/compose.cms.yml

@@ -29,6 +29,14 @@ x-backend-base: &backend-base
     CMS_SESSION_TTL_SECONDS: ${CMS_SESSION_TTL_SECONDS:-604800}
     CMS_AUTO_PROMOTE_ALL_ADMINS: ${CMS_AUTO_PROMOTE_ALL_ADMINS:-false}
     CMS_REBUILD_TAXONOMY_COUNTS_ON_STARTUP: ${CMS_REBUILD_TAXONOMY_COUNTS_ON_STARTUP:-false}
+    # Media: legacy WP uploads migrated to CephFS. The upload endpoint writes new
+    # assets under MEDIA_ROOT; MEDIA_BASE_URL is the public host that serves them.
+    MEDIA_ROOT: ${MEDIA_ROOT:-/mnt/cephfs/media}
+    MEDIA_BASE_URL: ${MEDIA_BASE_URL:-}
+  volumes:
+    # CephFS media tree (host). rw so the upload endpoint can store new files;
+    # host Nginx serves the same tree read-only (see deploy/nginx/triangle-cms.conf).
+    - ${MEDIA_HOST_PATH:-/mnt/cephfs/media}:/mnt/cephfs/media:rw
   healthcheck:
     test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080/v1/health/db || exit 1"]
     interval: 10s

deploy/compose.mariadb-dev.yml

@@ -0,0 +1,66 @@
+# Triangle CMS — THROWAWAY MariaDB for a Delta dry run. NOT for production.
+#
+# Production runs a primary + read replica behind MaxScale on dedicated hosts
+# (compose.mariadb-primary.yml / compose.mariadb-replica.yml). Those are tuned
+# for dedicated 8 GB boxes and will OOM on Delta, which has ~3.8 GB total and is
+# also running both CMS slots. This file exists purely so the CMS can be brought
+# up end-to-end before that hardware is provisioned.
+#
+# It is an OVERLAY on compose.cms.yml, so the database joins the same
+# `triangle_net` bridge and the backend can reach it by service name:
+#
+#   docker compose -f compose.cms.yml -f compose.mariadb-dev.yml \
+#     --env-file cms.env up -d
+#
+# With that, cms.env sets DB_HOST=mariadb-dev and DB_PORT=3306.
+#
+# Tear this down when the real DB host lands — repoint DB_HOST at MaxScale and
+# drop the second -f flag. The named volume below is deliberately distinct from
+# the production volume names so it can be removed without ambiguity:
+#
+#   docker compose -f compose.cms.yml -f compose.mariadb-dev.yml down
+#   docker volume rm triangle-cms_mariadb_dev_data
+
+services:
+  mariadb-dev:
+    image: mariadb:11.7
+    restart: unless-stopped
+    command: ["--log-error=/var/lib/mysql/error.log"]
+    environment:
+      MARIADB_ROOT_PASSWORD: ${MARIADB_ROOT_PASSWORD:?MARIADB_ROOT_PASSWORD is required}
+      # Unlike the production primary, this node DOES create the app database and
+      # user on first init — there is no replication or provisioning step here.
+      MARIADB_DATABASE: ${DB_NAME:?DB_NAME is required}
+      MARIADB_USER: ${DB_USER:?DB_USER is required}
+      MARIADB_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required}
+    volumes:
+      - mariadb_dev_data:/var/lib/mysql
+      - ./mariadb/dev.cnf:/etc/mysql/conf.d/dev.cnf:ro,z
+    # Loopback only. Nothing outside Delta should reach this, and the backend
+    # talks to it over triangle_net rather than through the host.
+    ports:
+      - "127.0.0.1:${MARIADB_PORT_FORWARD:-3306}:3306"
+    healthcheck:
+      test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+      start_period: 30s
+    networks:
+      - triangle_net
+
+  # Production cannot express this -- the database is external there, so the
+  # backend just retries until the endpoint answers. Here the DB is a sibling
+  # service, so wait for it and keep the startup logs clean.
+  backend-blue:
+    depends_on:
+      mariadb-dev:
+        condition: service_healthy
+
+  backend-green:
+    depends_on:
+      mariadb-dev:
+        condition: service_healthy
+
+volumes:
+  mariadb_dev_data: