Refactor the base tethys image to uv

.github/security/.grype.yaml

@@ -12,4 +12,7 @@ ignore:
   - package:
       name: python
       type: binary
-    reason: "Pinned to an older CPython until conda-build grows support for the latest version."
+    reason: "Pinned to an older CPython until conda-build grows support for the latest version."
+  - package:
+      name: pyo3
+    reason: "pyo3 0.25.1 is vendored in py-rattler 0.25.0 (conda's Rust solver) from the tethys-core base image. Not app runtime code and not controllable from this repo's dependencies. Revisit when the base image ships py-rattler built against pyo3 >=0.29."

.github/workflows/build_and_push_dev_image.yml

@@ -24,7 +24,6 @@ jobs:
       matrix:
         platform:
           - linux/amd64
-          - linux/arm64
 
     steps:
       - name: Prepare
@@ -147,6 +146,9 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - scan   # <- only push/tag if scan job as a whole succeeded
+    outputs:
+      # e.g. displays the new image tag dev_abc1234
+      image_version: ${{ steps.meta.outputs.version }}
 
     steps:
       - name: Download digests
@@ -184,3 +186,12 @@ jobs:
       - name: Inspect image
         run: |
           docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
+
+  # Auto-deploy merges to main onto the staging server (nrds-staging.ciroh.org)
+  deploy-staging:
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    needs: merge
+    uses: ./.github/workflows/deploy.yml
+    with:
+      environment: staging
+      image_tag: ${{ needs.merge.outputs.image_version }}

.github/workflows/build_and_push_prod_image.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [linux/amd64, linux/arm64]
+        platform: [linux/amd64]
 
     steps:
       - uses: actions/checkout@v4
@@ -171,3 +171,11 @@ jobs:
         run: |
           docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
           docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:latest
+
+  # Deploy the released version to the production server (nrds.ciroh.org)
+  deploy-prod:
+    needs: merge
+    uses: ./.github/workflows/deploy.yml
+    with:
+      environment: prod
+      image_tag: ${{ github.ref_name }}

.github/workflows/deploy.yml

@@ -0,0 +1,151 @@
+name: deploy
+
+on:
+  # Called by the dev/prod image workflows after an image is published
+  workflow_call:
+    inputs:
+      environment:
+        description: "Target environment (staging or prod)"
+        required: true
+        type: string
+      image_tag:
+        description: "Image tag to deploy (e.g. dev_abc1234 or v1.2.0)"
+        required: true
+        type: string
+
+  # Manual deploy / rollback: pick any previously published tag
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: "Target environment"
+        required: true
+        type: choice
+        options:
+          - staging
+          - prod
+      image_tag:
+        description: "Image tag to deploy (e.g. dev_abc1234 or v1.2.0)"
+        required: true
+        type: string
+
+env:
+  REGISTRY_IMAGE: awiciroh/tethys-nrds
+  CONTAINER_NAME: tethys-nrds
+  # Host directory backing TETHYS_PERSIST. The container writes collected static here as
+  # uid 1000 (= ec2-user on Amazon Linux) and the host nginx serves ${PERSIST_DIR}/static
+  # directly -- the image has no internal nginx of its own.
+  PERSIST_DIR: /srv/nrds/persist
+  # Host port stays 8080 so the existing nginx proxy_pass needs no change; only the
+  # container port moved (uvicorn listens on 8000, the old image's nginx was on 80).
+  HOST_BIND: 127.0.0.1:8080
+  CONTAINER_PORT: 8000
+
+jobs:
+  deploy:
+    # Runner label must match the environment name (staging / prod)
+    runs-on:
+      - self-hosted
+      - ${{ inputs.environment }}
+    environment: ${{ inputs.environment }}
+
+    # Set once for the whole job so both the provision and web containers get identical
+    # config. `docker run -e NAME` (no value) forwards the value from this environment,
+    # which keeps secrets out of the command line and avoids all shell quoting issues.
+    env:
+      TETHYS_SECRET_KEY: ${{ secrets.TETHYS_SECRET_KEY }}
+      PORTAL_SUPERUSER_NAME: ${{ vars.PORTAL_SUPERUSER_NAME || 'admin' }}
+      PORTAL_SUPERUSER_PASSWORD: ${{ secrets.PORTAL_SUPERUSER_PASSWORD }}
+      # Comma-separated. portal-config.sh merges these into ALLOWED_HOSTS and derives
+      # CSRF_TRUSTED_ORIGINS as https://<host> for each non-IP entry.
+      PORTAL_ALLOWED_HOSTS: ${{ vars.PORTAL_ALLOWED_HOSTS }}
+      # Per-environment GA measurement ID; unset => conf/portal-config.d/10-analytics.sh no-ops
+      GOOGLE_ANALYTICS_GTAG_PROPERTY_ID: ${{ vars.GA_TRACKING_ID }}
+      TETHYS_DB_ENGINE: django.db.backends.sqlite3
+      DOCKER_ENV_FLAGS: >-
+        -e TETHYS_SECRET_KEY
+        -e PORTAL_SUPERUSER_NAME
+        -e PORTAL_SUPERUSER_PASSWORD
+        -e PORTAL_ALLOWED_HOSTS
+        -e GOOGLE_ANALYTICS_GTAG_PROPERTY_ID
+        -e TETHYS_DB_ENGINE
+
+    steps:
+      - name: Check prerequisites
+        run: |
+          if [ -z "$TETHYS_SECRET_KEY" ]; then
+            echo "::error::TETHYS_SECRET_KEY secret is not set for the '${{ inputs.environment }}' environment."
+            echo "Generate one with: python3 -c 'import secrets; print(secrets.token_urlsafe(50))'"
+            exit 1
+          fi
+          if [ -z "$PORTAL_ALLOWED_HOSTS" ]; then
+            echo "::error::PORTAL_ALLOWED_HOSTS variable is not set for the '${{ inputs.environment }}' environment."
+            echo "Expected a comma-separated list, e.g. nrds.ciroh.org,localhost,127.0.0.1"
+            exit 1
+          fi
+          if [ ! -d "$PERSIST_DIR" ]; then
+            echo "::error::$PERSIST_DIR does not exist on this runner (one-time host setup)."
+            echo "Run on the instance:  sudo mkdir -p $PERSIST_DIR && sudo chown -R 1000:1000 /srv/nrds"
+            exit 1
+          fi
+          if [ ! -w "$PERSIST_DIR" ]; then
+            echo "::error::$PERSIST_DIR is not writable by the runner user (needs uid 1000)."
+            echo "Run on the instance:  sudo chown -R 1000:1000 /srv/nrds"
+            exit 1
+          fi
+
+      - name: Pull image
+        run: docker pull "$REGISTRY_IMAGE:${{ inputs.image_tag }}"
+
+      # One-shot: migrate the DB, collect static into PERSIST_DIR, create the superuser.
+      # The web container does none of this -- serve.sh only renders config and serves.
+      - name: Provision
+        run: |
+          docker run --rm \
+            $DOCKER_ENV_FLAGS \
+            -v "$PERSIST_DIR:/home/tethys/persist" \
+            "$REGISTRY_IMAGE:${{ inputs.image_tag }}" \
+            /usr/local/bin/bootstrap-provision.sh
+
+      - name: Replace container
+        run: |
+          docker rm -f "$CONTAINER_NAME" 2>/dev/null || true
+          docker run -d \
+            --name "$CONTAINER_NAME" \
+            --restart unless-stopped \
+            -p "$HOST_BIND:$CONTAINER_PORT" \
+            $DOCKER_ENV_FLAGS \
+            -v "$PERSIST_DIR:/home/tethys/persist" \
+            "$REGISTRY_IMAGE:${{ inputs.image_tag }}"
+
+      - name: Health check
+        run: |
+          echo "Waiting for uvicorn to accept requests..."
+          for i in $(seq 1 30); do
+            if curl -fsSL -o /dev/null "http://$HOST_BIND/"; then
+              echo "App is up after ~$((i * 5))s"
+              exit 0
+            fi
+            sleep 5
+          done
+          echo "App did not respond within 150s; container logs:"
+          docker logs --tail 100 "$CONTAINER_NAME"
+          exit 1
+
+      # Static is served by the host nginx off the bind mount, so verify it actually
+      # landed there -- a missing STATIC_ROOT would leave the UI unstyled/broken.
+      - name: Verify collected static
+        run: |
+          count=$(find "$PERSIST_DIR/static" -type f 2>/dev/null | wc -l)
+          echo "static files present: $count"
+          if [ "$count" -lt 10 ]; then
+            echo "::error::$PERSIST_DIR/static looks empty -- collectstatic did not run as expected."
+            exit 1
+          fi
+
+      - name: Remove unused images
+        run: docker image prune -f
+
+      - name: Summary
+        run: |
+          echo "Deployed \`$REGISTRY_IMAGE:${{ inputs.image_tag }}\` to **${{ inputs.environment }}**" \
+            >> "$GITHUB_STEP_SUMMARY"

Dockerfile

@@ -1,87 +1,64 @@
-
-FROM tethysplatform/tethys-core:dev-py3.12-dj5.2 
-
-
-###################
-# BUILD ARGUMENTS #
-###################
-
-ARG MICRO_TETHYS=true \
-    MAMBA_DOCKERFILE_ACTIVATE=1
-
-
-#########################
-# ADD APPLICATION FILES #
-#########################
-COPY . ${TETHYS_HOME}/apps/nrds
-COPY run.sh ${TETHYS_HOME}/run.sh
-
-###############
-# ENVIRONMENT #
-###############
-ENV TETHYS_DB_ENGINE=django.db.backends.sqlite3
-ENV SKIP_DB_SETUP=True
-ENV TETHYS_DB_NAME=
-ENV TETHYS_DB_USERNAME=
-ENV TETHYS_DB_PASSWORD=
-ENV TETHYS_DB_HOST=
-ENV TETHYS_DB_PORT=
-ENV ENABLE_OPEN_PORTAL=True
-ENV MULTIPLE_APP_MODE=False
-ENV STANDALONE_APP=nrds
-ENV PORTAL_SUPERUSER_NAME=admin
-ENV PORTAL_SUPERUSER_PASSWORD=pass
-ENV PROJ_LIB=/opt/conda/envs/tethys/share/proj
-
-ENV NVM_DIR=/usr/local/nvm
-ENV NODE_VERSION=24.4.1
-ENV NODE_VERSION_DIR=${NVM_DIR}/versions/node/v${NODE_VERSION}
-ENV NODE_PATH=${NODE_VERSION_DIR}/lib/node_modules
-ENV PATH=${NODE_VERSION_DIR}/bin:$PATH
-ENV NPM=${NODE_VERSION_DIR}/bin/npm
-ENV PDM="/root/.local/bin/pdm"
-ENV APP_SRC_ROOT=${TETHYS_HOME}/apps/nrds
-
-ENV DEV_REACT_CONFIG="${APP_SRC_ROOT}/reactapp/config/development.env"
-ENV PROD_REACT_CONFIG="${APP_SRC_ROOT}/reactapp/config/production.env"
-ENV TETHYS_DEBUG_MODE="false"
-ENV TETHYS_APP_PACKAGE=nrds
-ENV TETHYS_APP_ROOT_URL="/"
-ENV TETHYS_LOADER_DELAY=500
-ENV TETHYS_PORTAL_HOST=""
-
-# SETUP
-RUN mkdir -p ${NVM_DIR} \
-    && curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | /bin/bash \
-    && . ${NVM_DIR}/nvm.sh \
-    && nvm install ${NODE_VERSION} \
-    && nvm alias default ${NODE_VERSION} \
-    && nvm use default \
-    && npm install -g npm@latest \
-    && ls -la ${NODE_VERSION_DIR} \
-    && ls -la ${NODE_VERSION_DIR}/lib \
-    && pip install --user pdm \
-    && ${PDM} self update \
-    && cd ${APP_SRC_ROOT} \ 
-    && git config --global --add safe.directory '*' \
-    && git update-index --assume-unchanged
-
-
-RUN mv ${DEV_REACT_CONFIG} ${PROD_REACT_CONFIG} \
-  && sed -i "s#TETHYS_DEBUG_MODE.*#TETHYS_DEBUG_MODE = ${TETHYS_DEBUG_MODE}#g" ${PROD_REACT_CONFIG} \
-  && sed -i "s#TETHYS_LOADER_DELAY.*#TETHYS_LOADER_DELAY = ${TETHYS_LOADER_DELAY}#g" ${PROD_REACT_CONFIG} \
-  && sed -i "s#TETHYS_PORTAL_HOST.*#TETHYS_PORTAL_HOST = ${TETHYS_PORTAL_HOST}#g" ${PROD_REACT_CONFIG} \
-  && sed -i "s#TETHYS_APP_ROOT_URL.*#TETHYS_APP_ROOT_URL = ${TETHYS_APP_ROOT_URL}#g" ${PROD_REACT_CONFIG}
-
-RUN cd ${APP_SRC_ROOT} \
-    && ${NPM} install \
-    && ${NPM} run build \
-    && rm -rf node_modules \
-    && ${PDM} install --no-editable --production
-
-ADD salt/ /srv/salt/
-
-CMD bash run.sh
-
-HEALTHCHECK --start-period=30s --retries=12 \
-    CMD ./liveness-probe.sh
+# syntax=docker/dockerfile:1
+#
+# PROTOTYPE: NRDS portal image built on tethys-uvx (uv venv, no conda/salt/nginx/supervisor).
+# Served by uvicorn on ${TETHYS_PORT} (8000) as uid 1000. Static files are NOT served by this
+# container -- a reverse proxy serves them from STATIC_ROOT on the mounted persist volume.
+#
+# Base tags are pinned to a sha: tethys-uvx is young and its script/env contract still moves.
+
+###############################################################################
+# builder - React build + install the app into the base venv
+###############################################################################
+FROM ghcr.io/aquaveo/tethys-uvx:builder-a3148d5 AS builder
+
+# React build-time config (was ENV in the conda-image Dockerfile)
+ARG TETHYS_DEBUG_MODE=false
+ARG TETHYS_LOADER_DELAY=500
+ARG TETHYS_PORTAL_HOST=""
+ARG TETHYS_APP_ROOT_URL="/"
+
+WORKDIR /build
+
+# npm deps first so the (slow) install layer caches independently of app source
+COPY package.json package-lock.json ./
+RUN npm ci
+
+COPY . .
+
+# The repo ships development.env; production values are substituted in here.
+RUN mv reactapp/config/development.env reactapp/config/production.env \
+  && sed -i "s#TETHYS_DEBUG_MODE.*#TETHYS_DEBUG_MODE = ${TETHYS_DEBUG_MODE}#g" reactapp/config/production.env \
+  && sed -i "s#TETHYS_LOADER_DELAY.*#TETHYS_LOADER_DELAY = ${TETHYS_LOADER_DELAY}#g" reactapp/config/production.env \
+  && sed -i "s#TETHYS_PORTAL_HOST.*#TETHYS_PORTAL_HOST = ${TETHYS_PORTAL_HOST}#g" reactapp/config/production.env \
+  && sed -i "s#TETHYS_APP_ROOT_URL.*#TETHYS_APP_ROOT_URL = ${TETHYS_APP_ROOT_URL}#g" reactapp/config/production.env \
+  && npm run build
+
+# Install the app (frontend bundle included via package-data) plus:
+#   django-analytical - not in the base image; required to render the GA tag
+#   urllib3>=2        - overrides the base's urllib3<2 pin (keeps our CVE fix)
+# constraints.txt holds pandas<3 (pandas 3.x is untested against controllers/data_utils).
+# setuptools-scm reads the version from .git, which is owned by another uid in the build context.
+RUN git config --global --add safe.directory '*' \
+  && uv pip install --no-cache -c conf/constraints.txt . django-analytical "urllib3>=2" \
+  && chmod -R a+rX /opt/conda
+
+###############################################################################
+# runtime - slim base + the app-augmented venv
+###############################################################################
+FROM ghcr.io/aquaveo/tethys-uvx:runtime-base-a3148d5
+
+# Interpreter and venv must land at the same paths (pyvenv.cfg hardcodes /opt/python)
+COPY --from=builder /opt/python /opt/python
+COPY --from=builder /opt/conda  /opt/conda
+
+# Declarative portal settings (replaces salt/patches.sls) + per-environment config hooks
+COPY --chown=1000:1000 conf/portal_config.yml /config/portal_config.yml
+COPY --chown=1000:1000 conf/portal-config.d/ /opt/portal/portal-config.d/
+
+# Provision wrapper that can bootstrap an empty DB (see the script for why)
+COPY --chmod=0755 conf/bootstrap-provision.sh /usr/local/bin/bootstrap-provision.sh
+
+HEALTHCHECK --start-period=60s --interval=30s --retries=3 \
+    CMD curl -fsS -o /dev/null http://127.0.0.1:8000/ || exit 1
+
+# CMD (serve.sh) is inherited from the base

conf/bootstrap-provision.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Wrapper around the base image's provision.sh that can bootstrap an EMPTY database.
+#
+# Why this exists: Tethys builds its URLconf from rows in `tethys_apps_tethysapp`, and Django
+# runs system checks (which load the URLconf) before `migrate` executes. On a brand-new DB that
+# table doesn't exist yet, so plain `tethys db migrate` -- and therefore provision.sh -- dies with
+# "no such table: tethys_apps_tethysapp". The old conda image never hit this because its sqlite DB
+# was migrated at image-build time. We break the cycle with one checks-free migration, after which
+# provision.sh runs normally (migrate is idempotent).
+#
+# Upstream candidate: tethys-uvx's provision.sh arguably should do this itself.
+
+/usr/local/bin/portal-config.sh
+
+MANAGE="$(python -c 'import os, tethys_portal; print(os.path.join(os.path.dirname(tethys_portal.__file__), "manage.py"))')"
+
+if ! python "$MANAGE" migrate --check --skip-checks >/dev/null 2>&1; then
+  echo "Bootstrapping database (migrate --skip-checks)"
+  python "$MANAGE" migrate --skip-checks
+fi
+
+exec /usr/local/bin/provision.sh

conf/constraints.txt

@@ -0,0 +1,2 @@
+# Resolution constraints for the portal image build (uv pip install -c).
+pandas<3