Rework DataStream loading, map layers, and visual system

.github/security/.grype.yaml

@@ -12,4 +12,7 @@ ignore:
   - package:
       name: python
       type: binary
-    reason: "Pinned to an older CPython until conda-build grows support for the latest version."
+    reason: "Pinned to an older CPython until conda-build grows support for the latest version."
+  - package:
+      name: pyo3
+    reason: "pyo3 0.25.1 is vendored in py-rattler 0.25.0 (conda's Rust solver) from the tethys-core base image. Not app runtime code and not controllable from this repo's dependencies. Revisit when the base image ships py-rattler built against pyo3 >=0.29."

.github/workflows/build_and_push_dev_image.yml

@@ -147,6 +147,9 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - scan   # <- only push/tag if scan job as a whole succeeded
+    outputs:
+      # e.g. displays the new image tag dev_abc1234
+      image_version: ${{ steps.meta.outputs.version }}
 
     steps:
       - name: Download digests
@@ -184,3 +187,12 @@ jobs:
       - name: Inspect image
         run: |
           docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
+
+  # Auto-deploy merges to main onto the staging server (nrds-staging.ciroh.org)
+  deploy-staging:
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    needs: merge
+    uses: ./.github/workflows/deploy.yml
+    with:
+      environment: staging
+      image_tag: ${{ needs.merge.outputs.image_version }}

.github/workflows/build_and_push_prod_image.yml

@@ -171,3 +171,11 @@ jobs:
         run: |
           docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
           docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:latest
+
+  # Deploy the released version to the production server (nrds.ciroh.org)
+  deploy-prod:
+    needs: merge
+    uses: ./.github/workflows/deploy.yml
+    with:
+      environment: prod
+      image_tag: ${{ github.ref_name }}

.github/workflows/deploy.yml

@@ -0,0 +1,85 @@
+name: deploy
+
+on:
+  # Called by the dev/prod image workflows after an image is published
+  workflow_call:
+    inputs:
+      environment:
+        description: "Target environment (staging or prod)"
+        required: true
+        type: string
+      image_tag:
+        description: "Image tag to deploy (e.g. dev_abc1234 or v1.2.0)"
+        required: true
+        type: string
+
+  # Manual deploy / rollback: pick any previously published tag
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: "Target environment"
+        required: true
+        type: choice
+        options:
+          - staging
+          - prod
+      image_tag:
+        description: "Image tag to deploy (e.g. dev_abc1234 or v1.2.0)"
+        required: true
+        type: string
+
+env:
+  REGISTRY_IMAGE: awiciroh/tethys-nrds
+  CONTAINER_NAME: tethys-nrds
+
+jobs:
+  deploy:
+    # Runner label must match the environment name (staging / prod)
+    runs-on:
+      - self-hosted
+      - ${{ inputs.environment }}
+    environment: ${{ inputs.environment }}
+
+    steps:
+      - name: Pull image
+        run: docker pull "$REGISTRY_IMAGE:${{ inputs.image_tag }}"
+
+      - name: Replace container
+        run: |
+          # Google Analytics: only injected when the environment defines GA_TRACKING_ID
+          GA_ARGS=()
+          if [ -n "${{ vars.GA_TRACKING_ID }}" ]; then
+            GA_ARGS=(-e "GOOGLE_ANALYTICS_GTAG_PROPERTY_ID=${{ vars.GA_TRACKING_ID }}")
+          fi
+
+          docker rm -f "$CONTAINER_NAME" 2>/dev/null || true
+          docker run -d \
+            --name "$CONTAINER_NAME" \
+            --restart unless-stopped \
+            -p 127.0.0.1:8080:80 \
+            -e NGINX_PORT=80 \
+            -e SKIP_DB_SETUP=false \
+            -e ALLOWED_HOSTS='${{ vars.ALLOWED_HOSTS }}' \
+            -e CSRF_TRUSTED_ORIGINS='${{ vars.CSRF_TRUSTED_ORIGINS }}' \
+            "${GA_ARGS[@]}" \
+            "$REGISTRY_IMAGE:${{ inputs.image_tag }}"
+
+      - name: Health check
+        run: |
+          echo "Waiting for app to come up (first boot runs salt setup)..."
+          for i in $(seq 1 30); do
+            if curl -fsSL -o /dev/null http://127.0.0.1:8080/; then
+              echo "App is up after ~$((i * 5))s"
+              exit 0
+            fi
+            sleep 5
+          done
+          echo "App did not respond within 150s; container logs:"
+          docker logs --tail 100 "$CONTAINER_NAME"
+          exit 1
+
+      - name: Remove unused images
+        run: docker image prune -f
+
+      - name: Summary
+        run: echo "Deployed \`$REGISTRY_IMAGE:${{ inputs.image_tag }}\` to **${{ inputs.environment }}**" >> "$GITHUB_STEP_SUMMARY"

.gitignore

@@ -43,6 +43,10 @@ reactapp/config/production.env
 coverage
 reactapp/wdyr.js
 
+docs
+.impeccable
+.context
+DESIGN.md
 
 # singularity
 

Dockerfile

@@ -34,7 +34,7 @@ ENV PORTAL_SUPERUSER_PASSWORD=pass
 ENV PROJ_LIB=/opt/conda/envs/tethys/share/proj
 
 ENV NVM_DIR=/usr/local/nvm
-ENV NODE_VERSION=24.4.1
+ENV NODE_VERSION=24.18.0
 ENV NODE_VERSION_DIR=${NVM_DIR}/versions/node/v${NODE_VERSION}
 ENV NODE_PATH=${NODE_VERSION_DIR}/lib/node_modules
 ENV PATH=${NODE_VERSION_DIR}/bin:$PATH
@@ -77,7 +77,10 @@ RUN cd ${APP_SRC_ROOT} \
     && ${NPM} install \
     && ${NPM} run build \
     && rm -rf node_modules \
-    && ${PDM} install --no-editable --production
+    && ${PDM} install --no-editable --production \
+    # node is only needed to build the frontend; remove it so node CVEs
+    # don't flag the runtime image in security scans
+    && rm -rf ${NVM_DIR}
 
 ADD salt/ /srv/salt/
 

package.json

@@ -142,7 +142,7 @@
       "^(?!.*\\.(js|jsx|mjs|cjs|ts|tsx|css|json)$)": "<rootDir>/reactapp/config/tests/transforms/fileTransform.js"
     },
     "transformIgnorePatterns": [
-      "[/\\\\]node_modules[/\\\\](?!(ol|ol-mapbox-style|plotly.js|react-plotly.js)[/\\\\]).+\\.(js|jsx|mjs|cjs|ts|tsx)$",
+      "[/\\\\]node_modules[/\\\\](?!(ol|ol-mapbox-style|plotly.js|react-plotly.js|d3-[a-z-]+|internmap|delaunator|robust-predicates)[/\\\\]).+\\.(js|jsx|mjs|cjs|ts|tsx)$",
       "^.+\\.module\\.(css|sass|scss)$"
     ],
     "moduleFileExtensions": [