cfs-energy · cfssteve · Sep 4, 2026 · Sep 3, 2026 · Sep 3, 2026 · Sep 3, 2026
diff --git a/.github/actions/setup_dev_environment/action.yml b/.github/actions/setup_dev_environment/action.yml
@@ -33,10 +33,12 @@ runs:
         node-version: '${{ steps.versions.outputs.node_version }}'
     - name: Install AWS CLI
       shell: bash
+      # Unpack outside the checkout: the installer expands to an aws/ directory of about
+      # 25,000 files, which every tool that walks the working tree would read as project files.
       run: |
-        curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-        unzip -q awscliv2.zip
-        sudo ./aws/install --update
+        curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "${RUNNER_TEMP}/awscliv2.zip"
+        unzip -q "${RUNNER_TEMP}/awscliv2.zip" -d "${RUNNER_TEMP}"
+        sudo "${RUNNER_TEMP}/aws/install" --update
     - name: Install system dependencies
       shell: bash
       run: |

diff --git a/.github/trufflehog-exclude-paths.txt b/.github/trufflehog-exclude-paths.txt
@@ -4,6 +4,10 @@
 ^node_modules/
 /node_modules/
 ^build/
+# The AWS CLI installer archive and the directory it expands to, in case either is
+# ever downloaded inside a checkout again.
+^awscliv2\.zip$
+^aws/
 ^dist/
 __pycache__/
 \.ruff_cache/

diff --git a/.github/workflows/build_push.yaml b/.github/workflows/build_push.yaml
@@ -8,17 +8,30 @@ on:
   workflow_dispatch:
     inputs:
       ecr_repository:
-        description: 'ECR Repository URI (e.g., public.ecr.aws/your-registry or 123456789012.dkr.ecr.us-east-1.amazonaws.com/your-repo)'
+        description: 'Public ECR registry to publish to, as public.ecr.aws/<alias>'
         required: true
         default: 'public.ecr.aws/s5o2b4m0'
         type: string
+      image_name:
+        description: 'Repository name within that registry. Leave empty for the released image. A dispatch from a branch other than main must set it, for example idea-administrator-ci-test.'
+        required: false
+        default: ''
+        type: string
 jobs:
   build_push:
     runs-on: ubuntu-large
     permissions:
       id-token: write
       contents: read
     steps:
+      - name: Validate the image name
+        env:
+          IMAGE_NAME: ${{ github.event.inputs.image_name }}
+        run: |
+          if [ -z "$IMAGE_NAME" ] && [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ] && [ "$GITHUB_REF_NAME" != "main" ]; then
+            echo "::error::A dispatch from $GITHUB_REF_NAME must set image_name, or it would overwrite the released image from an unmerged ref. Use a throwaway repository name such as idea-administrator-ci-test."
+            exit 1
+          fi
       - uses: actions/checkout@v4
       - name: Setup Development Environment
         uses: ./.github/actions/setup_dev_environment
@@ -38,7 +51,9 @@ jobs:
           aws-region: us-east-1
           role-to-assume: ${{ secrets.ECR_ROLE }}
       - name: Build and Push Docker Image
+        env:
+          ECR_REGISTRY: ${{ github.event.inputs.ecr_repository || 'public.ecr.aws/s5o2b4m0' }}
+          IMAGE_NAME: ${{ github.event.inputs.image_name || 'idea-administrator' }}
         run: |
           VERSION=$(cat IDEA_VERSION.txt)
-          ECR_REPO="${{ github.event.inputs.ecr_repository || 'public.ecr.aws/s5o2b4m0' }}"
-          invoke docker.build-push-multi "$ECR_REPO" "$VERSION" --gha-cache
+          invoke docker.build-push-multi "$ECR_REGISTRY" "$VERSION" --image-name "$IMAGE_NAME" --gha-cache
diff --git a/.github/workflows/lint_build.yaml b/.github/workflows/lint_build.yaml
@@ -3,12 +3,30 @@ on:
   pull_request:
     paths-ignore:
       - 'docs/**'
+  push:
+    branches:
+      - main
+      - 'release-*'
   workflow_dispatch:
 jobs:
   lint_and_build:
     runs-on: ubuntu-large
     steps:
       - uses: actions/checkout@v4
+      - name: Check version consistency
+        run: |
+          SH_REV=$(grep -m1 '^IDEA_REVISION=' idea-admin.sh | sed -E 's/.*:-"([^"]+)"\}.*/\1/')
+          PS1_REV=$(grep -m1 '[$]IDEARevision = if' idea-admin-windows.ps1 | sed -E 's/.*else \{"([^"]+)"\}.*/\1/')
+          VERSION=$(tr -d '[:space:]' < IDEA_VERSION.txt)
+          EXPECTED="v${VERSION}"
+          echo "idea-admin.sh revision:          ${SH_REV}"
+          echo "idea-admin-windows.ps1 revision: ${PS1_REV}"
+          echo "IDEA_VERSION.txt:                ${VERSION} (expected revision ${EXPECTED})"
+          if [ "${SH_REV}" != "${EXPECTED}" ] || [ "${PS1_REV}" != "${EXPECTED}" ]; then
+            echo "::error::Version mismatch: IDEA_VERSION.txt=${VERSION} (expected revision ${EXPECTED}), idea-admin.sh default=${SH_REV}, idea-admin-windows.ps1 default=${PS1_REV}"
+            exit 1
+          fi
+          echo "Version consistency check passed (${EXPECTED})"
       - name: Pre-commit Linting
         uses: ./.github/actions/pre-commit
       - name: Setup Development Environment

diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml
@@ -3,6 +3,10 @@ on:
   pull_request:
     paths-ignore:
       - 'docs/**'
+  push:
+    branches:
+      - main
+      - 'release-*'
   workflow_dispatch:
 jobs:
   test:
@@ -14,6 +18,9 @@ jobs:
       - name: SDK Tests
         run: |
           invoke tests.sdk
+      - name: Administrator Tests
+        run: |
+          invoke tests.administrator
       - name: Cluster Manager Tests
         run: |
           invoke tests.cluster-manager

diff --git a/.gitignore b/.gitignore
@@ -208,3 +208,6 @@ source/idea/idea-cluster-manager/tests/*.db
 
 # Claude Code worktrees
 .claude/
+
+# DynamoDB Local metadata written by the test suites
+**/dynamodb-local-metadata.json
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,61 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Calendar Versioning](https://calver.org/).
 
+## [26.09.0] - 2026-08-03
+
+**Upgrade Instructions:**
+* Configuration and code release on top of 26.08.0. Redeploy all modules; there are no base OS, AMI map or schema changes
+* New optional settings, all default off or empty: `cluster.network.preferred_subnet_id`, `virtual-desktop-controller.dcv_session.first_boot_dnf_update`, `cluster-manager.maintenance.{enabled,message,ends_at}`
+* The cluster-manager role gains `pricing:GetProducts`, so redeploy cluster-manager for desktop hours to price
+* `upgrade-cluster` gains `--disable-eol-stacks-in-use`, and `idea-admin.sh` gains `IDEA_ADMIN_NO_TTY` for unattended runs
+* The DCV broker DynamoDB tables move to on demand billing when the virtual-desktop-controller starts and when a broker boots; set `virtual-desktop-controller.dcv_broker.dynamodb_table.on_demand` to false to keep provisioned capacity
+```bash
+./idea-admin.sh upgrade-cluster --aws-region $IDEA_AWS_REGION --cluster-name $IDEA_CLUSTER_NAME
+```
+([Upgrade Documentation](https://docs.idea-hpc.com/first-time-users/cluster-operations/update-idea-cluster/upgrade-cluster))
+
+### **✨ New Features**
+* **My Costs**: Home > My Costs shows each user their own trailing 30 day AI, job and desktop costs, with every figure marked as an estimate
+* **User Costs**: Cluster Management > User Costs lists every user with a measured cost in the window and drills into any one of them
+* **AI Usage Page**: Cluster Management > AI Usage lists every Bedrock project over the trailing 30 days, and breaks the selected project down per model and per user
+* **Custom AMIs**: Scale-Out Computing > Custom AMIs lists the compute and desktop images the cluster launches from with the last build per base OS, and builds new ones from the portal
+* **Desktop Image Build**: `ideactl build-desktop-image` builds a DCV host image per base OS, so desktops provision in minutes rather than about 15
+* **Refresh Base Stack AMIs**: Administrators can refresh the base software stack AMIs from the portal, all base stacks or a selection, with per-stack results
+* **Maintenance Banner**: `cluster-manager.maintenance.enabled`, `.message` and `.ends_at` put a warning banner on every portal page and make the scheduler refuse job submissions with the same message, with no redeploy
+* **AZ Preference**: Jobs and desktops without an explicit subnet prefer `cluster.network.preferred_subnet_id` and keep the other subnets as capacity fallback
+* **Desktop Updates**: Opt-in first-boot `dnf update` for RHEL, Rocky Linux and Amazon Linux 2023 desktops
+* **Non-Interactive Admin**: `IDEA_ADMIN_NO_TTY=true ./idea-admin.sh <cmd> --force` runs fully unattended
+* **End-of-Life Stacks**: `upgrade-cluster --disable-eol-stacks-in-use` disables the blocking end-of-life software stacks and continues instead of aborting the upgrade
+* **Bedrock**: A failed reconcile records an error state on the project instead of passing unnoticed
+
+### **🔧 Improvements**
+* **AI Usage**: The Projects page column reports the trailing 30 days rather than the calendar month, so a project used late in the previous month no longer reads as unused on the first; aggregation runs every 15 minutes and project budgets stay on the calendar month through AWS Budgets
+* **Upgrades**: `upgrade-cluster` restores a missing `values.yml` from the cluster bucket and saves it back after a successful upgrade
+* **Instance Types**: The defaults move to the most current instance type of the same architecture that every configured region offers. Module hosts go from `m6i.large` to `m7i.large`, the compute node image builder and the `compute` queue profile from `c5.large` to `c7i.large`, the arm64 image builder from `m6g.large` to `m8g.large`, the desktop image builder from `m6i.large` to `m7i.large`, and the demo application form from the `m5` sizes to `m7i`. `m7i.large` and `c7i.large` are each offered in all 28 of the 29 regions in `region_ami_config.yml` that can be checked, where `m8i.large` and `c6i.large` are each missing from several; the twenty-ninth, me-south-1, is an opt-in region that could not be queried. New clusters install on these, and new jobs, desktops and image builds pick them up straight away. Module host types are stored settings and the settings sync is add-only, so `upgrade-cluster` moves a host still on the `m6i.large` default to `m7i.large` after checking that the region offers it, which the host runs when its instance is next replaced; any other stored type is one you chose and is kept, and an unreadable offering list leaves every host alone
+* **Analytics**: New clusters place the OpenSearch data nodes on `m7g.large.search`, a Graviton type with the same 2 vCPU and 8 GiB as the `m5.large.search` it replaces, listed in the commercial, GovCloud and China partitions. `upgrade-cluster` moves a cluster still on `m5.large.search` when the region offers the new type for the domain engine version, which OpenSearch Service applies as a blue/green deployment with no downtime, and keeps any other type the operator chose. On the current price list it saves about 0.007 USD per instance hour in us-east-1 (0.135 against 0.142) and about 0.007 USD in each GovCloud region (0.17065 against 0.178), roughly 0.014 USD per hour for the two node default, and it lists below the old type in cn-north-1 and cn-northwest-1 as well
+* **Config**: Empty list values are stored as empty lists instead of DynamoDB NULL
+* **DynamoDB Cost**: The DCV broker tables are moved to on demand billing in place of the five read and five write units each was provisioned with, which the broker never came close to using; the switch runs at virtual-desktop-controller start and on every broker boot, because a rolling update can hand the boot event to a controller task still draining on the previous release, and the autoscaling policies that held that floor are no longer applied
+* **Admin Image**: Ships `awscrt` so `aws login` credential profiles work, and pins `greenlet` in the lock file
+* **Tests**: Added the missing coverage for architecture-mismatch validation, for EC2 lookup failure tolerance and queue profile save rejection
+* **CI**: The build fails when the admin wrapper revision does not match `IDEA_VERSION.txt`
+* **Release Publishing**: The workflow runs again, with the AWS CLI installer unpacked outside the checkout so the secret scan no longer walks the 25,000 files it left in the workspace; the published repository is named by a new `image_name` input that a dispatch from a branch other than `main` has to set
+* **Docs**: New administrator guide for enabling Amazon Bedrock, plus link and package list fixes
+
+### **🐛 Bug Fixes**
+* **AI Usage**: Usage aggregation no longer deletes stored rows when the model invocation log query returns nothing, so a missing or recreated log group cannot wipe every project's recorded usage in the lookback window
+* **Upgrades**: `upgrade-cluster` without `--base-os` keeps the base OS the cluster already runs instead of defaulting to Amazon Linux 2023, which with `--force` redeployed every module onto a different OS without a prompt; the upgrade refuses to start when the current value cannot be read
+* **Compute Images**: `upgrade-cluster` keeps the compute image the cluster runs when it was built from the Custom AMIs page and is newer than the release image, instead of resetting `scheduler.compute_node_ami` to the stock AMI on every upgrade; an older built image is still replaced and the run says so
+* **Credentials**: SSO profiles no longer send the admin CLI into infinite recursion, and an initialization failure reports the underlying error
+* **Software Stacks**: The controller reconciles the search index against DynamoDB at startup, so a deletion made outside the portal cannot leave a stack listed; `update-base-stacks` Rocky Linux patterns now match RESF image names
+* **Provisioning**: A failed compute stack costs one retry rather than two, a stack that cannot be deleted still counts each cycle so the retry cap holds the job with a visible reason, and controller sweeps resume where they stopped instead of rescanning the same page
+* **Desktops**: The Ubuntu first-boot upgrade runs noninteractively, so a debconf prompt cannot hang provisioning
+* **Desktop Events**: One failing event no longer blocks the controller queue: a message whose handler keeps failing is dropped after `virtual-desktop-controller.events.max_receive_count` receives, 3 by default, with its id, event type and session logged. A scheduled stop no longer fails for a host that never registered with SSM
+* **Web Portal**: The API invoker honors its timeout option instead of a fixed 11 minutes
+* **API**: Application tokens carrying the right module scope are accepted by the virtual desktop admin and DCV broker APIs; session material and username-coupled namespaces stay admin-user only
+* **File Browser**: The header checkbox and Ctrl+A select every entry in the directory listing rather than the current page, the header count reads "N of M selected", and Escape clears the selection
+* **Costs outside the commercial partition**: Instance hours are priced from the public AWS price list files when the Pricing API is unreachable, so desktop and job costs are no longer blank or zero in GovCloud and China regions. The cluster needs outbound HTTPS to `pricing.us-east-1.amazonaws.com`, the price map is built in the background and refreshed daily, and costs read "Not available" until the first load completes
+* **Job Costs**: A job the scheduler could not price records that on its estimate, so My Costs and User Costs read "Not available" instead of $0.00, while a job that genuinely cost nothing still reads $0.00
+
 ## [26.08.0] - 2026-08-28
 
 **Upgrade Instructions:**

diff --git a/IDEA_VERSION.txt b/IDEA_VERSION.txt
@@ -1 +1 @@
-26.08.0
+26.09.0
diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md
@@ -12,6 +12,7 @@
   * [SSH](first-time-users/access-your-idea-cluster/ssh.md)
   * [APIs](first-time-users/access-your-idea-cluster/apis.md)
 * [File Browser](first-time-users/file-browser.md)
+* [My Costs](first-time-users/my-costs.md)
 * [Let's get started](first-time-users/lets-get-started.md)
 * [Cluster operations](first-time-users/cluster-operations/README.md)
   * [Update IDEA cluster](first-time-users/cluster-operations/update-idea-cluster/README.md)
@@ -57,10 +58,13 @@
     * [Queue Profiles](modules/hpc-workloads/admin-documentation/queue-profiles.md)
     * [Create Web Based Job Submission Workflows](modules/hpc-workloads/admin-documentation/create-web-based-job-submission-workflows.md)
     * [Configure Floating License resources](modules/hpc-workloads/admin-documentation/configure-floating-license-resources.md)
+    * [Custom AMIs](modules/hpc-workloads/admin-documentation/custom-amis.md)
 * [Cluster Manager](modules/cluster-manager/README.md)
   * [Users Management](modules/cluster-manager/users-management.md)
   * [Groups Management](modules/cluster-manager/groups-management.md)
   * [Projects Management](modules/cluster-manager/projects-management.md)
+  * [Maintenance Banner](modules/cluster-manager/maintenance-banner.md)
+  * [Enable Amazon Bedrock](modules/cluster-manager/enable-amazon-bedrock.md)
   * [AI Usage Tracking](modules/cluster-manager/ai-usage-tracking.md)
   * [Data Sharing Between Users](modules/cluster-manager/data-sharing-between-users.md)
   * [APIs](modules/cluster-manager/apis.md)

diff --git a/docs/best-practices/budget/review-your-aws-spend.md b/docs/best-practices/budget/review-your-aws-spend.md
@@ -26,7 +26,7 @@ Billing console." A member account has no Cost allocation tags page, so somebody
 management account has to activate the `idea:` tag keys.
 
 IDEA cannot do this for you, and nothing fails loudly when it has not been done. Until the tag keys
-are active, `idea:` tags do not appear in Cost Explorer and a budget filtered on them reads as no
+are active, `idea:` tags do not appear in AWS Cost Explorer and a budget filtered on them reads as no
 spend, which looks the same as a project that has not spent anything.
 
 Tag keys can take up to 24 hours to appear on the cost allocation tags page, and up to another 24

diff --git a/docs/developer-portal/developer-onboarding.md b/docs/developer-portal/developer-onboarding.md
@@ -221,3 +221,51 @@ Export IDEA_DEV_MODE=true on your terminal, before executing idea-admin.sh on fr
 </strong><strong>To verify, if Developer Mode is enabled, run below command. This should print (Developer Mode) at the end of the banner.
 </strong>| ./idea-admin.sh about'####:'########::'########::::'###::::. ##:: ##.... ##: ##.....::::'## ##:::: ##:: ##:::: ##: ######:::'##:::. ##:: ##:: ##:::: ##: ##...:::: #########:'####: ########:: ########: ##:::: ##:Integrated Digital Engineering on AWSVersion 3.0.0-beta.1(Developer Mode)
 </code></pre>
+
+## Publishing the administrator image
+
+The administrator container image is published by the Build and Push workflow in
+`.github/workflows/build_push.yaml`.
+
+### Normal path
+
+Merging to `main` runs the workflow. It lints, runs the unit tests, builds every
+module, assumes the OIDC role held in the `ECR_ROLE` repository secret, and pushes
+one multi-architecture manifest to `public.ecr.aws/s5o2b4m0/idea-administrator`
+under three tags: the contents of `IDEA_VERSION.txt`, the same value prefixed with
+`v`, and `latest`.
+
+### Rerun path
+
+If that run fails after the merge, dispatch the same workflow again rather than
+publishing by hand:
+
+```bash
+gh workflow run build_push.yaml --ref main
+```
+
+Two inputs change the target. `ecr_repository` selects the registry and
+`image_name` selects the repository within it. From a ref other than `main` the
+workflow stops immediately unless `image_name` is set, so a branch dispatch cannot
+overwrite the released image:
+
+```bash
+gh workflow run build_push.yaml --ref release-26.09.0 \
+  -f image_name=idea-administrator-ci-test
+```
+
+The named repository has to exist already, because ECR Public does not create one
+on push. Delete a throwaway repository once the check is finished.
+
+### Emergency path
+
+`invoke docker.build-push-multi <registry> <version>` produces the same three tags
+from a workstation, but it skips the lint, test and build gates the workflow
+applies. Use it only when the workflow itself cannot run.
+
+### The publishing role
+
+The role named by `ECR_ROLE` trusts any ref of this repository, so the `image_name`
+guard above is the only control that stops a branch dispatch from replacing the
+released image. Narrowing the role trust condition to `main` would remove the need
+for that guard.