Fix proxy trust and attest RFMS runtime identity

.github/workflows/deploy.yml

@@ -18,6 +18,8 @@ jobs:
     if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
     steps:
       - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.workflow_run.head_sha || github.sha }}
 
       - name: Install Ansible
         run: |
@@ -36,10 +38,13 @@ jobs:
 
       - name: Smoke test SSH connection
         run: |
-          ssh -i ~/.ssh/id_rsa_racknerd deploy@198.23.137.16 "echo SSH_OK"
+          ssh -i ~/.ssh/id_rsa_racknerd -o IdentitiesOnly=yes deploy@198.23.137.16 "echo SSH_OK"
 
       - name: Run Ansible playbook
-        run: |
-          ansible-playbook -i inventory/hosts.yml ansible-playbook.yml -vv
         env:
-          ANSIBLE_HOST_KEY_CHECKING: false
+          DEPLOY_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
+        run: |
+          [[ "$DEPLOY_SHA" =~ ^[0-9a-f]{40}$ ]]
+          ANSIBLE_HOST_KEY_CHECKING=false ansible-playbook \
+            -i inventory/hosts.yml ansible-playbook.yml \
+            --extra-vars "deploy_sha=$DEPLOY_SHA" -vv

Dockerfile

@@ -23,6 +23,9 @@ RUN npm run build
 # Stage 2: Production runtime
 FROM node:22-slim
 
+ARG APP_VERSION=unknown
+ARG COMMIT_SHA=unknown
+
 # Install runtime dependencies
 RUN apt-get update && apt-get install -y \
     curl \
@@ -47,6 +50,8 @@ RUN npm ci --omit=dev -w server
 
 ENV NODE_ENV=production
 ENV PORT=8080
+ENV APP_VERSION=${APP_VERSION}
+ENV COMMIT_SHA=${COMMIT_SHA}
 
 RUN chown -R node:node /app
 USER node

ansible-playbook.yml

@@ -10,6 +10,13 @@
     public_origin: 'https://fmc.reidar.tech'
 
   tasks:
+    - name: Validate exact deployment commit
+      assert:
+        that:
+          - deploy_sha is defined
+          - deploy_sha is match('^[0-9a-f]{40}$')
+        fail_msg: 'deploy_sha must be the exact 40-character CI-tested commit.'
+
     - name: Ensure app directory exists
       file:
         path: '{{ app_dir }}'
@@ -18,23 +25,58 @@
         group: deploy
         mode: '0755'
 
+    - name: Inspect current production container for rollback
+      community.docker.docker_container_info:
+        name: '{{ container_name }}'
+      register: previous_container_info
+
+    - name: Record rollback image
+      set_fact:
+        previous_image_id: '{{ previous_container_info.container.Image }}'
+        previous_image_ref: '{{ previous_container_info.container.Config.Image }}'
+      when: previous_container_info.exists
+
     - name: Clone or pull repository
       git:
         repo: '{{ repo_url }}'
         dest: '{{ app_dir }}'
-        version: main
+        version: '{{ deploy_sha }}'
         force: true
       register: git_result
 
+    - name: Verify checkout and read application version
+      block:
+        - name: Assert checkout matches tested commit
+          assert:
+            that:
+              - git_result.after == deploy_sha
+
+        - name: Read server package metadata
+          slurp:
+            src: '{{ app_dir }}/server/package.json'
+          register: server_package
+
+        - name: Set immutable build identity
+          set_fact:
+            app_version: '{{ (server_package.content | b64decode | from_json).version }}'
+            image_ref: '{{ container_name }}:sha-{{ deploy_sha }}'
+
     - name: Build Docker image
       community.docker.docker_image:
-        name: '{{ container_name }}'
+        name: '{{ image_ref }}'
         source: build
         build:
           path: '{{ app_dir }}'
+          args:
+            APP_VERSION: '{{ app_version }}'
+            COMMIT_SHA: '{{ deploy_sha }}'
         force_source: true
       register: build_result
 
+    - name: Record exact image identity
+      set_fact:
+        image_id: '{{ build_result.image.Id }}'
+
     - name: Stop any stale canary container
       community.docker.docker_container:
         name: '{{ container_name }}_canary'
@@ -43,7 +85,7 @@
     - name: Start canary container for health check
       community.docker.docker_container:
         name: '{{ container_name }}_canary'
-        image: '{{ container_name }}'
+        image: '{{ image_ref }}'
         state: started
         ports:
           - '127.0.0.1:8083:{{ container_port }}'
@@ -52,13 +94,25 @@
           PORT: '{{ container_port }}'
           AIRCRAFT_ADAPTER: 'mock'
           WS_ALLOWED_ORIGINS: '{{ public_origin }}'
+          IMAGE_ID: '{{ image_id }}'
+          IMAGE_REF: '{{ image_ref }}'
         healthcheck:
           test: ['CMD-SHELL', 'curl -f http://localhost:{{ container_port }}/health || exit 1']
           interval: 10s
           timeout: 5s
           retries: 5
           start_period: 5s
 
+    - name: Verify canary uses the built image
+      community.docker.docker_container_info:
+        name: '{{ container_name }}_canary'
+      register: canary_container_info
+
+    - name: Assert canary Docker image identity
+      assert:
+        that:
+          - canary_container_info.container.Image == image_id
+
     - name: Wait for canary health check
       block:
         - name: Poll canary health endpoint
@@ -67,7 +121,12 @@
             return_content: true
             timeout: 5
           register: canary_health
-          until: canary_health is succeeded and canary_health.json.status == "ok"
+          until:
+            - canary_health is succeeded
+            - canary_health.json.status == "ok"
+            - canary_health.json.build.commit == deploy_sha
+            - canary_health.json.build.imageId == image_id
+            - canary_health.json.build.imageRef == image_ref
           retries: 12
           delay: 5
       rescue:
@@ -107,7 +166,7 @@
         - name: Start production container
           community.docker.docker_container:
             name: '{{ container_name }}'
-            image: '{{ container_name }}'
+            image: '{{ image_ref }}'
             state: started
             restart_policy: unless-stopped
             ports:
@@ -124,11 +183,86 @@
               AIRCRAFT_ADAPTER: 'mock'
               WS_ALLOWED_ORIGINS: '{{ public_origin }}'
               WS_MAX_MESSAGE_BYTES: '65536'
+              IMAGE_ID: '{{ image_id }}'
+              IMAGE_REF: '{{ image_ref }}'
+
+        - name: Inspect promoted production container
+          community.docker.docker_container_info:
+            name: '{{ container_name }}'
+          register: promoted_container_info
+
+        - name: Assert promoted Docker image identity
+          assert:
+            that:
+              - promoted_container_info.container.Image == image_id
+
+        - name: Verify promoted runtime identity
+          uri:
+            url: 'http://localhost:{{ host_port }}/health'
+            return_content: true
+            timeout: 5
+          register: promoted_health
+          until:
+            - promoted_health is succeeded
+            - promoted_health.json.status == "ok"
+            - promoted_health.json.build.commit == deploy_sha
+            - promoted_health.json.build.imageId == image_id
+            - promoted_health.json.build.imageRef == image_ref
+          retries: 12
+          delay: 5
 
         - name: Remove canary container
           community.docker.docker_container:
             name: '{{ container_name }}_canary'
             state: absent
+      rescue:
+        - name: Remove canary after failed promotion
+          community.docker.docker_container:
+            name: '{{ container_name }}_canary'
+            state: absent
+
+        - name: Remove failed production container
+          community.docker.docker_container:
+            name: '{{ container_name }}'
+            state: absent
+
+        - name: Restore previous production image
+          community.docker.docker_container:
+            name: '{{ container_name }}'
+            image: '{{ previous_image_id }}'
+            state: started
+            restart_policy: unless-stopped
+            ports:
+              - '127.0.0.1:{{ host_port }}:{{ container_port }}'
+            memory: '512m'
+            cpus: 1.0
+            log_driver: 'json-file'
+            log_options:
+              max-size: '10m'
+              max-file: '3'
+            env:
+              NODE_ENV: production
+              PORT: '{{ container_port }}'
+              AIRCRAFT_ADAPTER: 'mock'
+              WS_ALLOWED_ORIGINS: '{{ public_origin }}'
+              WS_MAX_MESSAGE_BYTES: '65536'
+              IMAGE_ID: '{{ previous_image_id }}'
+              IMAGE_REF: '{{ previous_image_ref }}'
+          when: previous_container_info.exists
+
+        - name: Verify rollback health
+          uri:
+            url: 'http://localhost:{{ host_port }}/health'
+            timeout: 5
+          register: rollback_health
+          until: rollback_health is succeeded and rollback_health.json.status == "ok"
+          retries: 12
+          delay: 5
+          when: previous_container_info.exists
+
+        - name: Fail deployment after rollback
+          fail:
+            msg: 'Production promotion failed; the previous image was restored when available.'
 
     - name: Ensure Caddy config includes fmc.reidar.tech
       become: true
@@ -152,6 +286,26 @@
         name: '{{ container_name }}'
       register: container_info
 
+    - name: Assert production Docker image identity
+      assert:
+        that:
+          - container_info.container.Image == image_id
+
+    - name: Verify production runtime identity
+      uri:
+        url: 'http://localhost:{{ host_port }}/health'
+        return_content: true
+        timeout: 5
+      register: production_health
+      until:
+        - production_health is succeeded
+        - production_health.json.status == "ok"
+        - production_health.json.build.commit == deploy_sha
+        - production_health.json.build.imageId == image_id
+        - production_health.json.build.imageRef == image_ref
+      retries: 12
+      delay: 5
+
     - name: Report deployment status
       debug:
-        msg: 'VirtualCDU deployed: {{ container_info.container.State.Status }} -> https://fmc.reidar.tech'
+        msg: 'VirtualCDU deployed: {{ container_info.container.State.Status }} {{ deploy_sha }} {{ image_id }} -> https://fmc.reidar.tech'

docs/DEPLOYMENT.md

@@ -9,19 +9,23 @@ The following environment variables are required:
 - `NODE_ENV`: Set to `production`.
 - `PORT`: Server port (default 8080).
 - `WS_ALLOWED_ORIGINS`: Comma-separated list of allowed origins for WebSockets.
-- `APP_VERSION`: Current version string.
-- `COMMIT_SHA`: Full git commit hash.
+- `APP_VERSION`: Server package version, baked into the image by Ansible.
+- `COMMIT_SHA`: Full CI-tested git commit, baked into the image by Ansible.
+- `IMAGE_ID`: Exact local Docker image ID, injected when the container starts.
+- `IMAGE_REF`: Immutable local `virtual-cdu:sha-<full-commit>` image reference.
 
 ## 2. Deployment Process
 
-We use a "Pull-based" or "Push-to-Deploy" model (e.g., Coolify, Portainer, or GitHub Actions).
+GitHub Actions deploys the exact successful `main` CI commit with Ansible. Ansible checks out that commit, builds an immutable local SHA-tagged image, validates a canary on loopback port 8083, and only then replaces production on loopback port 8082.
 
 ### Safety Steps
 
 1. **Pre-flight**: CI must pass all tests and typechecks.
-2. **Build**: Build image with `COMMIT_SHA` as a tag.
-3. **Smoke Test**: Deploy to staging first and run E2E tests.
-4. **Production**: Deploy with a health-aware rolling update.
+2. **Build**: Build `virtual-cdu:sha-<full-commit>` with the package version and commit embedded.
+3. **Smoke Test**: Start the same image as a canary and require `/health` to attest the expected commit, image ID, and image reference.
+4. **Production**: Replace production only after the canary passes, then verify the production `/health` identity again.
+
+SHA-tagged local images are not pruned by the current playbook. Add a bounded cleanup policy separately, retaining at least the active and previous rollback images; this deployment change makes no cleanup claim.
 
 ## 3. Rolling Updates & Health Checks
 
@@ -34,9 +38,10 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \
 
 The deployment orchestrator should:
 
-- Start the new container.
-- Wait for it to become healthy.
+- Start the new image as `virtual-cdu_canary` without touching production.
+- Wait for it to become healthy and attest its exact build identity.
 - Stop the old container only after the new one is healthy.
+- Start production from the already-validated image and verify its identity.
 
 ## 4. Rollback Procedure
 
@@ -46,7 +51,15 @@ If a deployment fails or a regression is found:
 2. **Verification**: Confirm health via `/health` endpoint.
 3. **Logs**: Check structured logs for `SIM_ERROR` or `WS_VALIDATION_ERROR`.
 
+Promotion failures automatically attempt to restore the exact previous Docker image and verify basic health. This is fast recovery, not zero downtime: the container replacement still creates a brief interruption window.
+
 ## 5. Monitoring
 
 - **Logs**: Production logs are in JSON format.
-- **Metrics**: Visit `/health` to see active client counts and error rates.
+- **Metrics**: Visit `/health` to see active client counts, error rates, and exact runtime build identity.
+
+## 6. Reverse Proxy Trust Boundary
+
+Caddy is the single trusted proxy hop. It connects from the VPS host to the Docker container through the loopback-published port. Express and WebSocket connection limiting use the same one-hop trust function, so arbitrary left-most `X-Forwarded-For` values are not trusted.
+
+Cloudflare is upstream of Caddy. The repository-managed Caddy block does not trust or preserve upstream proxy chains, so rate limiting may group traffic by Cloudflare edge address rather than end-user address. This is intentionally fail-closed against spoofed forwarded headers; changing it requires an origin-access policy plus an explicit trusted-Cloudflare configuration.

inventory/hosts.yml

@@ -3,3 +3,4 @@ vps:
     198.23.137.16:
       ansible_user: deploy
       ansible_ssh_private_key_file: ~/.ssh/id_rsa_racknerd
+      ansible_ssh_extra_args: -o IdentitiesOnly=yes