From be70b202dda345b9aaa2ac424a68de694f453198 Mon Sep 17 00:00:00 2001 From: Steve Miller Date: Thu, 3 Sep 2026 12:33:54 +0000 Subject: [PATCH 1/3] feat: september fixes on late august release 26.08.0 --- .../actions/setup_dev_environment/action.yml | 8 +- .github/trufflehog-exclude-paths.txt | 4 + .github/workflows/build_push.yaml | 21 +- .github/workflows/lint_build.yaml | 18 + .github/workflows/unit_tests.yaml | 7 + .gitignore | 3 + CHANGELOG.md | 51 + IDEA_VERSION.txt | 2 +- docs/SUMMARY.md | 4 + .../budget/review-your-aws-spend.md | 2 +- docs/developer-portal/developer-onboarding.md | 48 + .../update-idea-cluster/upgrade-cluster.md | 81 +- docs/first-time-users/my-costs.md | 63 + docs/modules/cluster-manager/README.md | 2 +- .../cluster-manager/ai-usage-tracking.md | 34 +- .../cluster-manager/enable-amazon-bedrock.md | 38 + .../cluster-manager/maintenance-banner.md | 48 + .../admin-documentation/README.md | 1 + .../admin-documentation/custom-amis.md | 39 + .../supported-ec2-parameters.md | 9 + .../admin-documentation/sessions.md | 16 + .../virtual-desktop-images-software-stacks.md | 21 + .../user-documentation/choose-an-ai-model.md | 2 +- .../create-a-virtual-desktop-linux-windows.md | 2 + idea-admin-windows.ps1 | 2 +- idea-admin.sh | 31 +- requirements/idea-administrator.in | 2 + requirements/idea-administrator.txt | 2 + scripts/dev/ami_update_stacks.py | 4 +- .../templates/cluster-manager/settings.yml | 15 +- .../config/templates/cluster/settings.yml | 10 + .../templates/global-settings/settings.yml | 4 + .../config/templates/scheduler/settings.yml | 4 + .../virtual-desktop-controller/settings.yml | 7 + .../resources/policies/cluster-manager.yml | 11 + .../resources/policies/scheduler.yml | 1 + .../ideaadministrator/app/config_generator.py | 7 +- .../app/deployment_helper.py | 30 +- .../src/ideaadministrator/app_main.py | 744 ++++++++-- .../src/ideaadministrator_meta/__init__.py | 2 +- .../tests/test_ami_update_entries.py | 237 ++- .../tests/test_config_generator_empty_list.py | 35 + .../tests/test_eol_base_os_preflight.py | 123 +- .../tests/test_upgrade_cluster_guards.py | 168 +++ .../tests/test_values_file_s3_fallback.py | 198 +++ .../_templates/linux/openpbs.jinja2 | 2 +- .../_templates/linux/system_packages.jinja2 | 18 +- .../_templates/linux/system_upgrade.jinja2 | 16 +- .../dcv_host_ami_builder.sh.jinja2 | 94 ++ ...dcv_host_ami_builder_post_reboot.sh.jinja2 | 101 ++ .../dcv-host-ami-builder/setup.sh.jinja2 | 67 + .../_templates/install_openldap.jinja2 | 4 +- .../ideaclustermanager/app/api/api_invoker.py | 8 + .../app/api/cluster_settings_api.py | 13 +- .../ideaclustermanager/app/api/costs_api.py | 54 + .../app/api/my_costs_api.py | 32 + .../app/api/projects_api.py | 29 +- .../ideaclustermanager/app/costs/__init__.py | 1 + .../app/costs/my_costs_service.py | 926 ++++++++++++ .../app/projects/bedrock_provisioner.py | 39 +- .../app/projects/bedrock_usage_service.py | 248 +++- .../app/projects/db/projects_dao.py | 15 + .../app/projects/projects_service.py | 108 +- .../src/ideaclustermanager/app/web_portal.py | 15 + .../src/ideaclustermanager_meta/__init__.py | 2 +- .../test_bedrock_provisioner.py | 119 ++ .../test_bedrock_usage.py | 295 +++- .../test_cluster_settings_scoping.py | 26 +- .../test_cluster_settings_validation.py | 21 + .../ideaclustermanagertests/test_costs_api.py | 190 +++ .../ideaclustermanagertests/test_my_costs.py | 1297 +++++++++++++++++ source/idea/idea-cluster-manager/webapp/.env | 2 +- .../idea-cluster-manager/webapp/package.json | 2 +- .../idea-cluster-manager/webapp/src/App.tsx | 68 + .../webapp/src/client/clients.ts | 15 + .../webapp/src/client/data-model.ts | 243 +++ .../src/client/idea-api-invoker.test.ts | 22 +- .../webapp/src/client/idea-api-invoker.ts | 3 +- .../webapp/src/client/my-costs-client.ts | 40 + .../webapp/src/client/projects-client.ts | 9 + .../src/client/scheduler-admin-client.test.ts | 26 + .../src/client/scheduler-admin-client.ts | 20 +- .../virtual-desktop-admin-client.test.ts | 47 + .../client/virtual-desktop-admin-client.ts | 47 +- .../components/app-layout/app-layout.test.tsx | 89 ++ .../src/components/app-layout/app-layout.tsx | 41 +- .../webapp/src/components/cost-sections.tsx | 303 ++++ .../webapp/src/navigation/side-nav-items.tsx | 22 + .../webapp/src/pages/auth/auth-layout.tsx | 11 +- .../src/pages/cluster-admin/ai-usage.test.tsx | 146 ++ .../src/pages/cluster-admin/ai-usage.tsx | 294 ++++ .../pages/cluster-admin/cluster-settings.tsx | 127 +- .../src/pages/cluster-admin/projects.test.tsx | 49 +- .../src/pages/cluster-admin/projects.tsx | 23 +- .../pages/cluster-admin/user-costs.test.tsx | 271 ++++ .../src/pages/cluster-admin/user-costs.tsx | 289 ++++ .../src/pages/home/file-browser-table.tsx | 57 +- .../src/pages/home/file-browser.test.tsx | 92 +- .../webapp/src/pages/home/my-costs.test.tsx | 336 +++++ .../webapp/src/pages/home/my-costs.tsx | 88 ++ .../src/pages/hpc/hpc-custom-amis.test.tsx | 283 ++++ .../webapp/src/pages/hpc/hpc-custom-amis.tsx | 826 ++++++++++- .../virtual-desktop-software-stacks.tsx | 97 +- .../service/cluster-settings-service.test.ts | 47 +- .../src/service/cluster-settings-service.ts | 100 ++ .../src/ideadatamodel/__init__.py | 1 + .../src/ideadatamodel/aws/__init__.py | 1 + .../ideadatamodel/aws/image_build_model.py | 56 + .../src/ideadatamodel/my_costs/__init__.py | 1 + .../ideadatamodel/my_costs/my_costs_api.py | 183 +++ .../ideadatamodel/projects/projects_api.py | 21 + .../ideadatamodel/projects/projects_model.py | 20 +- .../ideadatamodel/scheduler/scheduler_api.py | 51 +- .../scheduler/scheduler_model.py | 4 + .../virtual_desktop/virtual_desktop_api.py | 129 ++ .../virtual_desktop/virtual_desktop_model.py | 8 + .../src/ideadatamodel_meta/__init__.py | 2 +- .../resources/opensearch/template_jobs.json | 3 + .../app/api/scheduler_admin_api.py | 49 +- .../ideascheduler/app/api/scheduler_api.py | 29 + .../src/ideascheduler/app/app_protocols.py | 4 +- .../ideascheduler/app/aws/pricing_helper.py | 35 +- .../src/ideascheduler/app/images/__init__.py | 8 + .../app/images/compute_images.py | 274 ++++ .../app/images/compute_node_ami_builder.py | 680 +++++++++ .../app/provisioning/job_monitor/job_cache.py | 37 +- .../job_provisioner/job_provisioner.py | 18 +- .../job_provisioner/job_provisioning_util.py | 13 +- .../node_monitor/node_house_keeper.py | 4 + .../app/scheduler/job_param_builder.py | 86 +- .../app/scheduler/openpbs/openpbs_model.py | 4 +- .../src/ideascheduler/app/scheduler_app.py | 15 + .../src/ideascheduler/cli/ami_builder.py | 583 +------- .../src/ideascheduler_meta/__init__.py | 2 +- source/idea/idea-scheduler/tests/conftest.py | 1 + .../tests/test_app_imports_no_cli.py | 38 + .../tests/test_compute_images.py | 498 +++++++ .../tests/test_compute_node_ami_builder.py | 64 + .../tests/test_hpc_queue_profiles_service.py | 36 +- .../idea-scheduler/tests/test_job_duration.py | 161 ++ .../tests/test_job_param_builder.py | 315 ++++ .../tests/test_job_submission_maintenance.py | 147 ++ .../tests/test_provisioning_retry_cap.py | 237 ++- .../tests/test_scheduler_admin_api_scoping.py | 89 ++ .../src/ideasdk/aws/aws_client_provider.py | 17 +- .../idea/idea-sdk/src/ideasdk/aws/aws_util.py | 75 +- .../src/ideasdk/aws/ec2_price_list.py | 226 +++ .../idea-sdk/src/ideasdk/aws/image_builds.py | 622 ++++++++ .../idea-sdk/src/ideasdk/aws/stock_amis.py | 230 +++ .../src/ideasdk/config/soca_config.py | 14 +- .../src/ideasdk/context/soca_cli_context.py | 4 + .../src/ideasdk/protocols/__init__.py | 4 +- .../idea-sdk/src/ideasdk_meta/__init__.py | 2 +- .../tests/test_are_credentials_expired.py | 100 ++ .../idea-sdk/tests/test_config_empty_list.py | 71 + .../tests/test_cost_explorer_tagged_spend.py | 44 + .../test_dcv_host_ami_builder_bootstrap.py | 85 ++ .../tests/test_ec2_instance_type_pricing.py | 210 +++ .../idea-sdk/tests/test_ec2_price_list.py | 300 ++++ .../idea/idea-sdk/tests/test_image_builds.py | 477 ++++++ .../tests/test_system_packages_bootstrap.py | 87 ++ .../tests/test_system_upgrade_bootstrap.py | 65 + .../opensearch/session_entry_template.yml | 3 + .../app/api/virtual_desktop_admin_api.py | 100 +- .../app/api/virtual_desktop_dcv_api.py | 6 +- ...ea_session_scheduled_stop_event_handler.py | 24 +- .../events/service/events_handler_thread.py | 37 +- .../app/sessions/constants.py | 1 + .../sessions/virtual_desktop_session_db.py | 89 ++ .../virtual_desktop_session_history_db.py | 156 ++ .../sessions/virtual_desktop_session_utils.py | 61 +- .../app/software_stacks/constants.py | 1 + .../software_stacks/dcv_host_image_builder.py | 631 ++++++++ .../app/software_stacks/desktop_images.py | 424 ++++++ .../virtual_desktop_software_stack_db.py | 4 + .../virtual_desktop_software_stack_utils.py | 223 ++- .../app/virtual_desktop_controller_app.py | 71 + .../app/virtual_desktop_controller_utils.py | 72 +- .../cli/build_desktop_image.py | 210 +++ .../cli/cli_main.py | 2 + .../cli/software_stacks.py | 269 +--- .../__init__.py | 2 +- .../tests/conftest.py | 20 + .../tests/test_app_imports_no_cli.py | 44 + .../tests/test_build_desktop_image.py | 225 +++ .../tests/test_desktop_images.py | 667 +++++++++ .../tests/test_event_queue_poison_message.py | 347 +++++ .../tests/test_instance_profile_repair.py | 24 +- .../tests/test_provisioning_timeout.py | 24 +- .../tests/test_refresh_base_stack_amis.py | 318 ++++ .../tests/test_session_history.py | 164 +++ .../tests/test_session_launch_failure.py | 177 +++ .../test_session_stop_time_round_trip.py | 281 ++++ .../tests/test_session_stopped_on.py | 140 ++ .../test_software_stack_startup_reindex.py | 287 ++++ .../tests/test_software_stacks_ami_lookup.py | 18 + .../tests/test_stopped_session_cleanup.py | 82 +- .../tests/test_sweep_cursor_sharing.py | 93 ++ .../tests/test_vdc_api_scoping.py | 227 +++ tasks/docker.py | 20 +- tasks/web_portal.py | 1 + 201 files changed, 20869 insertions(+), 1252 deletions(-) create mode 100644 docs/first-time-users/my-costs.md create mode 100644 docs/modules/cluster-manager/enable-amazon-bedrock.md create mode 100644 docs/modules/cluster-manager/maintenance-banner.md create mode 100644 docs/modules/hpc-workloads/admin-documentation/custom-amis.md create mode 100644 source/idea/idea-administrator/tests/test_config_generator_empty_list.py create mode 100644 source/idea/idea-administrator/tests/test_upgrade_cluster_guards.py create mode 100644 source/idea/idea-administrator/tests/test_values_file_s3_fallback.py create mode 100644 source/idea/idea-bootstrap/dcv-host-ami-builder/dcv_host_ami_builder.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/dcv-host-ami-builder/dcv_host_ami_builder_post_reboot.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/dcv-host-ami-builder/setup.sh.jinja2 create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/costs_api.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/my_costs_api.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/costs/__init__.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/costs/my_costs_service.py create mode 100644 source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_costs_api.py create mode 100644 source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_my_costs.py create mode 100644 source/idea/idea-cluster-manager/webapp/src/client/my-costs-client.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/client/scheduler-admin-client.test.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-admin-client.test.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/app-layout/app-layout.test.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/cost-sections.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/cluster-admin/ai-usage.test.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/cluster-admin/ai-usage.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/cluster-admin/user-costs.test.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/cluster-admin/user-costs.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/home/my-costs.test.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/home/my-costs.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/hpc/hpc-custom-amis.test.tsx create mode 100644 source/idea/idea-data-model/src/ideadatamodel/aws/image_build_model.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/my_costs/__init__.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/my_costs/my_costs_api.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/images/__init__.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/images/compute_images.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/images/compute_node_ami_builder.py create mode 100644 source/idea/idea-scheduler/tests/test_app_imports_no_cli.py create mode 100644 source/idea/idea-scheduler/tests/test_compute_images.py create mode 100644 source/idea/idea-scheduler/tests/test_compute_node_ami_builder.py create mode 100644 source/idea/idea-scheduler/tests/test_job_submission_maintenance.py create mode 100644 source/idea/idea-scheduler/tests/test_scheduler_admin_api_scoping.py create mode 100644 source/idea/idea-sdk/src/ideasdk/aws/ec2_price_list.py create mode 100644 source/idea/idea-sdk/src/ideasdk/aws/image_builds.py create mode 100644 source/idea/idea-sdk/src/ideasdk/aws/stock_amis.py create mode 100644 source/idea/idea-sdk/tests/test_are_credentials_expired.py create mode 100644 source/idea/idea-sdk/tests/test_config_empty_list.py create mode 100644 source/idea/idea-sdk/tests/test_dcv_host_ami_builder_bootstrap.py create mode 100644 source/idea/idea-sdk/tests/test_ec2_instance_type_pricing.py create mode 100644 source/idea/idea-sdk/tests/test_ec2_price_list.py create mode 100644 source/idea/idea-sdk/tests/test_image_builds.py create mode 100644 source/idea/idea-sdk/tests/test_system_packages_bootstrap.py create mode 100644 source/idea/idea-sdk/tests/test_system_upgrade_bootstrap.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/sessions/virtual_desktop_session_history_db.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/software_stacks/dcv_host_image_builder.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/software_stacks/desktop_images.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/cli/build_desktop_image.py create mode 100644 source/idea/idea-virtual-desktop-controller/tests/test_app_imports_no_cli.py create mode 100644 source/idea/idea-virtual-desktop-controller/tests/test_build_desktop_image.py create mode 100644 source/idea/idea-virtual-desktop-controller/tests/test_desktop_images.py create mode 100644 source/idea/idea-virtual-desktop-controller/tests/test_event_queue_poison_message.py create mode 100644 source/idea/idea-virtual-desktop-controller/tests/test_refresh_base_stack_amis.py create mode 100644 source/idea/idea-virtual-desktop-controller/tests/test_session_history.py create mode 100644 source/idea/idea-virtual-desktop-controller/tests/test_session_stop_time_round_trip.py create mode 100644 source/idea/idea-virtual-desktop-controller/tests/test_session_stopped_on.py create mode 100644 source/idea/idea-virtual-desktop-controller/tests/test_software_stack_startup_reindex.py create mode 100644 source/idea/idea-virtual-desktop-controller/tests/test_sweep_cursor_sharing.py create mode 100644 source/idea/idea-virtual-desktop-controller/tests/test_vdc_api_scoping.py diff --git a/.github/actions/setup_dev_environment/action.yml b/.github/actions/setup_dev_environment/action.yml index 2c2155ad..e98c59e6 100644 --- a/.github/actions/setup_dev_environment/action.yml +++ b/.github/actions/setup_dev_environment/action.yml @@ -33,10 +33,12 @@ runs: node-version: '${{ steps.versions.outputs.node_version }}' - name: Install AWS CLI shell: bash + # Unpack outside the checkout: the installer expands to an aws/ directory of about + # 25,000 files, which every tool that walks the working tree would read as project files. run: | - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip -q awscliv2.zip - sudo ./aws/install --update + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "${RUNNER_TEMP}/awscliv2.zip" + unzip -q "${RUNNER_TEMP}/awscliv2.zip" -d "${RUNNER_TEMP}" + sudo "${RUNNER_TEMP}/aws/install" --update - name: Install system dependencies shell: bash run: | diff --git a/.github/trufflehog-exclude-paths.txt b/.github/trufflehog-exclude-paths.txt index 03010588..6a350399 100644 --- a/.github/trufflehog-exclude-paths.txt +++ b/.github/trufflehog-exclude-paths.txt @@ -4,6 +4,10 @@ ^node_modules/ /node_modules/ ^build/ +# The AWS CLI installer archive and the directory it expands to, in case either is +# ever downloaded inside a checkout again. +^awscliv2\.zip$ +^aws/ ^dist/ __pycache__/ \.ruff_cache/ diff --git a/.github/workflows/build_push.yaml b/.github/workflows/build_push.yaml index 6c48ccf9..3de6e38a 100644 --- a/.github/workflows/build_push.yaml +++ b/.github/workflows/build_push.yaml @@ -8,10 +8,15 @@ on: workflow_dispatch: inputs: ecr_repository: - description: 'ECR Repository URI (e.g., public.ecr.aws/your-registry or 123456789012.dkr.ecr.us-east-1.amazonaws.com/your-repo)' + description: 'Public ECR registry to publish to, as public.ecr.aws/' required: true default: 'public.ecr.aws/s5o2b4m0' type: string + image_name: + description: 'Repository name within that registry. Leave empty for the released image. A dispatch from a branch other than main must set it, for example idea-administrator-ci-test.' + required: false + default: '' + type: string jobs: build_push: runs-on: ubuntu-large @@ -19,6 +24,14 @@ jobs: id-token: write contents: read steps: + - name: Validate the image name + env: + IMAGE_NAME: ${{ github.event.inputs.image_name }} + run: | + if [ -z "$IMAGE_NAME" ] && [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ] && [ "$GITHUB_REF_NAME" != "main" ]; then + echo "::error::A dispatch from $GITHUB_REF_NAME must set image_name, or it would overwrite the released image from an unmerged ref. Use a throwaway repository name such as idea-administrator-ci-test." + exit 1 + fi - uses: actions/checkout@v4 - name: Setup Development Environment uses: ./.github/actions/setup_dev_environment @@ -38,7 +51,9 @@ jobs: aws-region: us-east-1 role-to-assume: ${{ secrets.ECR_ROLE }} - name: Build and Push Docker Image + env: + ECR_REGISTRY: ${{ github.event.inputs.ecr_repository || 'public.ecr.aws/s5o2b4m0' }} + IMAGE_NAME: ${{ github.event.inputs.image_name || 'idea-administrator' }} run: | VERSION=$(cat IDEA_VERSION.txt) - ECR_REPO="${{ github.event.inputs.ecr_repository || 'public.ecr.aws/s5o2b4m0' }}" - invoke docker.build-push-multi "$ECR_REPO" "$VERSION" --gha-cache + invoke docker.build-push-multi "$ECR_REGISTRY" "$VERSION" --image-name "$IMAGE_NAME" --gha-cache diff --git a/.github/workflows/lint_build.yaml b/.github/workflows/lint_build.yaml index 47e40841..4f336a9f 100644 --- a/.github/workflows/lint_build.yaml +++ b/.github/workflows/lint_build.yaml @@ -3,12 +3,30 @@ on: pull_request: paths-ignore: - 'docs/**' + push: + branches: + - main + - 'release-*' workflow_dispatch: jobs: lint_and_build: runs-on: ubuntu-large steps: - uses: actions/checkout@v4 + - name: Check version consistency + run: | + SH_REV=$(grep -m1 '^IDEA_REVISION=' idea-admin.sh | sed -E 's/.*:-"([^"]+)"\}.*/\1/') + PS1_REV=$(grep -m1 '\$IDEARevision = if' idea-admin-windows.ps1 | sed -E 's/.*else \{"([^"]+)"\}.*/\1/') + VERSION=$(tr -d '[:space:]' < IDEA_VERSION.txt) + EXPECTED="v${VERSION}" + echo "idea-admin.sh revision: ${SH_REV}" + echo "idea-admin-windows.ps1 revision: ${PS1_REV}" + echo "IDEA_VERSION.txt: ${VERSION} (expected revision ${EXPECTED})" + if [ "${SH_REV}" != "${EXPECTED}" ] || [ "${PS1_REV}" != "${EXPECTED}" ]; then + echo "::error::Version mismatch: IDEA_VERSION.txt=${VERSION} (expected revision ${EXPECTED}), idea-admin.sh default=${SH_REV}, idea-admin-windows.ps1 default=${PS1_REV}" + exit 1 + fi + echo "Version consistency check passed (${EXPECTED})" - name: Pre-commit Linting uses: ./.github/actions/pre-commit - name: Setup Development Environment diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 07e812c6..d14b1fa1 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -3,6 +3,10 @@ on: pull_request: paths-ignore: - 'docs/**' + push: + branches: + - main + - 'release-*' workflow_dispatch: jobs: test: @@ -14,6 +18,9 @@ jobs: - name: SDK Tests run: | invoke tests.sdk + - name: Administrator Tests + run: | + invoke tests.administrator - name: Cluster Manager Tests run: | invoke tests.cluster-manager diff --git a/.gitignore b/.gitignore index fe790c2c..ffb61e2f 100644 --- a/.gitignore +++ b/.gitignore @@ -208,3 +208,6 @@ source/idea/idea-cluster-manager/tests/*.db # Claude Code worktrees .claude/ + +# DynamoDB Local metadata written by the test suites +**/dynamodb-local-metadata.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 8af41954..0f6f051d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,57 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Calendar Versioning](https://calver.org/). +## [26.09.0] - 2026-08-03 + +**Upgrade Instructions:** +* Configuration and code release on top of 26.08.0. Redeploy all modules; there are no base OS, AMI map or schema changes +* New optional settings, all default off or empty: `cluster.network.preferred_subnet_id`, `virtual-desktop-controller.dcv_session.first_boot_dnf_update`, `cluster-manager.maintenance.{enabled,message,ends_at}` +* The cluster-manager role gains `pricing:GetProducts`, so redeploy cluster-manager for desktop hours to price +* `upgrade-cluster` gains `--disable-eol-stacks-in-use`, and `idea-admin.sh` gains `IDEA_ADMIN_NO_TTY` for unattended runs +```bash +./idea-admin.sh upgrade-cluster --aws-region $IDEA_AWS_REGION --cluster-name $IDEA_CLUSTER_NAME +``` +([Upgrade Documentation](https://docs.idea-hpc.com/first-time-users/cluster-operations/update-idea-cluster/upgrade-cluster)) + +### **✨ New Features** +* **My Costs**: Home > My Costs shows each user their own trailing 30 day AI, job and desktop costs, with every figure marked as an estimate +* **User Costs**: Cluster Management > User Costs lists every user with a measured cost in the window and drills into any one of them +* **AI Usage Page**: Cluster Management > AI Usage lists every Bedrock project over the trailing 30 days, and breaks the selected project down per model and per user +* **Custom AMIs**: Scale-Out Computing > Custom AMIs lists the compute and desktop images the cluster launches from with the last build per base OS, and builds new ones from the portal +* **Desktop Image Build**: `ideactl build-desktop-image` builds a DCV host image per base OS, so desktops provision in minutes rather than about 15 +* **Refresh Base Stack AMIs**: Administrators can refresh the base software stack AMIs from the portal, all base stacks or a selection, with per-stack results +* **Maintenance Banner**: `cluster-manager.maintenance.enabled`, `.message` and `.ends_at` put a warning banner on every portal page and make the scheduler refuse job submissions with the same message, with no redeploy +* **AZ Preference**: Jobs and desktops without an explicit subnet prefer `cluster.network.preferred_subnet_id` and keep the other subnets as capacity fallback +* **Desktop Updates**: Opt-in first-boot `dnf update` for RHEL, Rocky Linux and Amazon Linux 2023 desktops +* **Non-Interactive Admin**: `IDEA_ADMIN_NO_TTY=true ./idea-admin.sh --force` runs fully unattended +* **End-of-Life Stacks**: `upgrade-cluster --disable-eol-stacks-in-use` disables the blocking end-of-life software stacks and continues instead of aborting the upgrade +* **Bedrock**: A failed reconcile records an error state on the project instead of passing unnoticed + +### **🔧 Improvements** +* **AI Usage**: The Projects page column reports the trailing 30 days rather than the calendar month, so a project used late in the previous month no longer reads as unused on the first; aggregation runs every 15 minutes and project budgets stay on the calendar month through AWS Budgets +* **Upgrades**: `upgrade-cluster` restores a missing `values.yml` from the cluster bucket and saves it back after a successful upgrade +* **Config**: Empty list values are stored as empty lists instead of DynamoDB NULL +* **Admin Image**: Ships `awscrt` so `aws login` credential profiles work, and pins `greenlet` in the lock file +* **Tests**: Added the missing coverage for architecture-mismatch validation, for EC2 lookup failure tolerance and queue profile save rejection +* **CI**: The build fails when the admin wrapper revision does not match `IDEA_VERSION.txt` +* **Release Publishing**: The workflow runs again, with the AWS CLI installer unpacked outside the checkout so the secret scan no longer walks the 25,000 files it left in the workspace; the published repository is named by a new `image_name` input that a dispatch from a branch other than `main` has to set +* **Docs**: New administrator guide for enabling Amazon Bedrock, plus link and package list fixes + +### **🐛 Bug Fixes** +* **AI Usage**: Usage aggregation no longer deletes stored rows when the model invocation log query returns nothing, so a missing or recreated log group cannot wipe every project's recorded usage in the lookback window +* **Upgrades**: `upgrade-cluster` without `--base-os` keeps the base OS the cluster already runs instead of defaulting to Amazon Linux 2023, which with `--force` redeployed every module onto a different OS without a prompt; the upgrade refuses to start when the current value cannot be read +* **Compute Images**: `upgrade-cluster` keeps the compute image the cluster runs when it was built from the Custom AMIs page and is newer than the release image, instead of resetting `scheduler.compute_node_ami` to the stock AMI on every upgrade; an older built image is still replaced and the run says so +* **Credentials**: SSO profiles no longer send the admin CLI into infinite recursion, and an initialization failure reports the underlying error +* **Software Stacks**: The controller reconciles the search index against DynamoDB at startup, so a deletion made outside the portal cannot leave a stack listed; `update-base-stacks` Rocky Linux patterns now match RESF image names +* **Provisioning**: A failed compute stack costs one retry rather than two, a stack that cannot be deleted still counts each cycle so the retry cap holds the job with a visible reason, and controller sweeps resume where they stopped instead of rescanning the same page +* **Desktops**: The Ubuntu first-boot upgrade runs noninteractively, so a debconf prompt cannot hang provisioning +* **Desktop Events**: One failing event no longer blocks the controller queue: a message whose handler keeps failing is dropped after `virtual-desktop-controller.events.max_receive_count` receives, 3 by default, with its id, event type and session logged. A scheduled stop no longer fails for a host that never registered with SSM +* **Web Portal**: The API invoker honors its timeout option instead of a fixed 11 minutes +* **API**: Application tokens carrying the right module scope are accepted by the virtual desktop admin and DCV broker APIs; session material and username-coupled namespaces stay admin-user only +* **File Browser**: The header checkbox and Ctrl+A select every entry in the directory listing rather than the current page, the header count reads "N of M selected", and Escape clears the selection +* **Costs outside the commercial partition**: Instance hours are priced from the public AWS price list files when the Pricing API is unreachable, so desktop and job costs are no longer blank or zero in GovCloud and China regions. The cluster needs outbound HTTPS to `pricing.us-east-1.amazonaws.com`, the price map is built in the background and refreshed daily, and costs read "Not available" until the first load completes +* **Job Costs**: A job the scheduler could not price records that on its estimate, so My Costs and User Costs read "Not available" instead of $0.00, while a job that genuinely cost nothing still reads $0.00 + ## [26.08.0] - 2026-08-28 **Upgrade Instructions:** diff --git a/IDEA_VERSION.txt b/IDEA_VERSION.txt index bf58b94d..a6cca8a2 100644 --- a/IDEA_VERSION.txt +++ b/IDEA_VERSION.txt @@ -1 +1 @@ -26.08.0 +26.09.0 diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 472fa780..78dd52df 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -12,6 +12,7 @@ * [SSH](first-time-users/access-your-idea-cluster/ssh.md) * [APIs](first-time-users/access-your-idea-cluster/apis.md) * [File Browser](first-time-users/file-browser.md) +* [My Costs](first-time-users/my-costs.md) * [Let's get started](first-time-users/lets-get-started.md) * [Cluster operations](first-time-users/cluster-operations/README.md) * [Update IDEA cluster](first-time-users/cluster-operations/update-idea-cluster/README.md) @@ -57,10 +58,13 @@ * [Queue Profiles](modules/hpc-workloads/admin-documentation/queue-profiles.md) * [Create Web Based Job Submission Workflows](modules/hpc-workloads/admin-documentation/create-web-based-job-submission-workflows.md) * [Configure Floating License resources](modules/hpc-workloads/admin-documentation/configure-floating-license-resources.md) + * [Custom AMIs](modules/hpc-workloads/admin-documentation/custom-amis.md) * [Cluster Manager](modules/cluster-manager/README.md) * [Users Management](modules/cluster-manager/users-management.md) * [Groups Management](modules/cluster-manager/groups-management.md) * [Projects Management](modules/cluster-manager/projects-management.md) + * [Maintenance Banner](modules/cluster-manager/maintenance-banner.md) + * [Enable Amazon Bedrock](modules/cluster-manager/enable-amazon-bedrock.md) * [AI Usage Tracking](modules/cluster-manager/ai-usage-tracking.md) * [Data Sharing Between Users](modules/cluster-manager/data-sharing-between-users.md) * [APIs](modules/cluster-manager/apis.md) diff --git a/docs/best-practices/budget/review-your-aws-spend.md b/docs/best-practices/budget/review-your-aws-spend.md index 9f7aaeac..1414bbd8 100644 --- a/docs/best-practices/budget/review-your-aws-spend.md +++ b/docs/best-practices/budget/review-your-aws-spend.md @@ -26,7 +26,7 @@ Billing console." A member account has no Cost allocation tags page, so somebody management account has to activate the `idea:` tag keys. IDEA cannot do this for you, and nothing fails loudly when it has not been done. Until the tag keys -are active, `idea:` tags do not appear in Cost Explorer and a budget filtered on them reads as no +are active, `idea:` tags do not appear in AWS Cost Explorer and a budget filtered on them reads as no spend, which looks the same as a project that has not spent anything. Tag keys can take up to 24 hours to appear on the cost allocation tags page, and up to another 24 diff --git a/docs/developer-portal/developer-onboarding.md b/docs/developer-portal/developer-onboarding.md index 30325891..4bbd4a1c 100644 --- a/docs/developer-portal/developer-onboarding.md +++ b/docs/developer-portal/developer-onboarding.md @@ -221,3 +221,51 @@ Export IDEA_DEV_MODE=true on your terminal, before executing idea-admin.sh on fr To verify, if Developer Mode is enabled, run below command. This should print (Developer Mode) at the end of the banner. | ./idea-admin.sh about'####:'########::'########::::'###::::. ##:: ##.... ##: ##.....::::'## ##:::: ##:: ##:::: ##: ######:::'##:::. ##:: ##:: ##:::: ##: ##...:::: #########:'####: ########:: ########: ##:::: ##:Integrated Digital Engineering on AWSVersion 3.0.0-beta.1(Developer Mode) + +## Publishing the administrator image + +The administrator container image is published by the Build and Push workflow in +`.github/workflows/build_push.yaml`. + +### Normal path + +Merging to `main` runs the workflow. It lints, runs the unit tests, builds every +module, assumes the OIDC role held in the `ECR_ROLE` repository secret, and pushes +one multi-architecture manifest to `public.ecr.aws/s5o2b4m0/idea-administrator` +under three tags: the contents of `IDEA_VERSION.txt`, the same value prefixed with +`v`, and `latest`. + +### Rerun path + +If that run fails after the merge, dispatch the same workflow again rather than +publishing by hand: + +```bash +gh workflow run build_push.yaml --ref main +``` + +Two inputs change the target. `ecr_repository` selects the registry and +`image_name` selects the repository within it. From a ref other than `main` the +workflow stops immediately unless `image_name` is set, so a branch dispatch cannot +overwrite the released image: + +```bash +gh workflow run build_push.yaml --ref release-26.09.0 \ + -f image_name=idea-administrator-ci-test +``` + +The named repository has to exist already, because ECR Public does not create one +on push. Delete a throwaway repository once the check is finished. + +### Emergency path + +`invoke docker.build-push-multi ` produces the same three tags +from a workstation, but it skips the lint, test and build gates the workflow +applies. Use it only when the workflow itself cannot run. + +### The publishing role + +The role named by `ECR_ROLE` trusts any ref of this repository, so the `image_name` +guard above is the only control that stops a branch dispatch from replacing the +released image. Narrowing the role trust condition to `main` would remove the need +for that guard. diff --git a/docs/first-time-users/cluster-operations/update-idea-cluster/upgrade-cluster.md b/docs/first-time-users/cluster-operations/update-idea-cluster/upgrade-cluster.md index e9d0d125..f09b1bc4 100644 --- a/docs/first-time-users/cluster-operations/update-idea-cluster/upgrade-cluster.md +++ b/docs/first-time-users/cluster-operations/update-idea-cluster/upgrade-cluster.md @@ -11,6 +11,46 @@ The `upgrade-cluster` command combines multiple steps that were previously separ 3. Backing up and regenerating global settings 4. Deploying all modules with the `--upgrade` flag +### values.yml Restore and Save + +The first phase reads `~/.idea/clusters///values.yml` to set the new Base +OS. If that file is missing locally, the command downloads `values/values.yml` from the cluster S3 +bucket, writes it to that path and continues, so an upgrade can run from a workstation that never +held the original file. If the bucket has no copy either, the command stops and names both the local +path and the S3 URI it checked; upload an existing copy with `idea-admin.sh config save-values` +before retrying. When both copies exist and differ, the command reports which keys differ and uses +the local copy. + +After every module deploys successfully, the command uploads the local `values.yml` back to +`values/values.yml` in the cluster bucket so the next upgrade can restore it. A failed upload logs a +warning naming the `idea-admin.sh config save-values` command to run and does not fail the upgrade, +because static STS credentials often expire before the last phase of a long run finishes. + +### Compute Node Image + +The upgrade moves compute nodes onto the release's AMI for the cluster's Base OS, unless +`scheduler.compute_node_ami` names an image built from the Custom AMIs page that is newer than the +release image, which is kept and reported. An older built image is replaced, and can be rebuilt +from Custom AMIs after the upgrade. + +### Before You Start + +Turn the maintenance banner on before you close the scheduler, and off after you have verified the +upgraded cluster. While it is on, every portal page carries your message and new job submissions are +refused with it instead of a generic failure. Neither change needs a redeploy. + +```bash +./idea-admin.sh config \ + set "Key=cluster-manager.maintenance.enabled,Type=bool,Value=true" \ + "Key=cluster-manager.maintenance.message,Type=string,Value=HPC scheduler is closed for a cluster upgrade." \ + --cluster-name \ + --aws-region +``` + +The same three settings are editable from Cluster Management, then Settings, then the Maintenance +tab. See [Maintenance Banner](../../../modules/cluster-manager/maintenance-banner.md) for the +optional end time and for what the banner does not cover. + ### Usage The basic syntax for the upgrade command is: @@ -29,12 +69,14 @@ If no modules are specified, all modules will be upgraded automatically. #### Optional Parameters * `MODULES`: List of modules to upgrade (e.g., `cluster`, `metrics`, `scheduler`, etc.). If not specified, all modules will be upgraded. -* `--base-os`: New base OS to upgrade to. If not specified, defaults to `amazonlinux2023`. Supported options are: - * `amazonlinux2023` (default) +* `--base-os`: Base OS to upgrade to. If not specified, the cluster keeps the Base OS it already runs, read from its cluster settings and printed before the upgrade starts. Pass this option only to change the Base OS. Supported options are: + * `amazonlinux2023` * `rhel8` * `rhel9` + * `rhel10` * `rocky8` * `rocky9` + * `rocky10` * `--aws-profile`: AWS profile to use for the operation * `--termination-protection`: Set CloudFormation stack termination protection (default: true) * `--force-build-bootstrap`: Re-build bootstrap package even if directory exists @@ -44,10 +86,34 @@ If no modules are specified, all modules will be upgraded automatically. * `--skip-global-settings-update`: Skip the global settings update if you've already done it * `--module-set`: Name of the module set to use (default: default) * `--deployment-id`: UUID to identify the deployment +* `--disable-eol-stacks-in-use`: Disable, rather than delete, end-of-life virtual desktop software stacks that a live session still uses + +The end-of-life check runs before the upgrade is confirmed and changes nothing: it lists the +software stacks it will delete or disable, prefixed with `will delete` or `will disable`. Those +changes are applied only after you confirm the upgrade, or immediately when you pass `--force`. + +By default the upgrade stops before making any change when a virtual desktop session still runs on +a software stack whose base OS has reached end-of-life, and lists the sessions that block it. +Passing `--disable-eol-stacks-in-use` sets `enabled` to false on each of those stacks instead and +continues. The stack record is kept, so running desktops are unaffected, but no new session can be +launched from a disabled stack. End-of-life stacks that no live session uses are still deleted, and +end-of-life references in cluster settings and HPC queue profiles remain a hard stop that this flag +does not change. + +A stack is disabled in DynamoDB, while the portal lists software stacks from the search index, so it +keeps reading as enabled until the index catches up. The virtual-desktop-controller redeploy later in +the same upgrade reconciles the index at startup. To reindex sooner, run +`ideactl reindex-software-stacks --reset` on the virtual-desktop-controller host. ### Examples -#### Full Upgrade to Default Base OS (Amazon Linux 2023) +#### Full Upgrade Keeping the Current Base OS + +Without `--base-os` the upgrade keeps the Base OS the cluster already runs. It reads that value from +the cluster settings, prints it, and refuses to start if the settings hold no Base OS or more than +one, because guessing would redeploy every module onto an OS nobody asked for. That matters most +with `--force`, which answers every confirmation prompt for you. Passing `--base-os` changes the +Base OS and prints what it is changing from. The simplest way to upgrade all infrastructure components: @@ -60,7 +126,7 @@ The simplest way to upgrade all infrastructure components: #### Full Upgrade with Explicit Base OS -Explicitly specify the base OS (same as default): +Move the cluster to a different Base OS: ```bash ./idea-admin.sh upgrade-cluster --base-os amazonlinux2023 \ @@ -104,6 +170,13 @@ Use parallel deployment where possible: --aws-profile default ``` +### ECR Credentials Reset + +Before launching the container, `idea-admin.sh` resets its public ECR credentials, which needs +`dig`. A stock Amazon Linux 2023 host does not ship `dig`; when it is missing the reset is skipped +with a warning and the command continues. Install `bind-utils` to restore the reset, or set +`IDEA_ECR_CREDS_RESET=false` to skip it without the warning. + ### Troubleshooting If the upgrade fails during the pre-upgrade configuration stage: diff --git a/docs/first-time-users/my-costs.md b/docs/first-time-users/my-costs.md new file mode 100644 index 00000000..fd5e1384 --- /dev/null +++ b/docs/first-time-users/my-costs.md @@ -0,0 +1,63 @@ +--- +description: My Costs shows each user what the cluster recorded against them over the last 30 days. +--- + +# My Costs + +Click "**My Costs**" under **Home** on the left sidebar. It always shows your own numbers: the request carries no username and the server reads yours from your session token, so there is no way to ask it about another user, administrators included. An administrator who needs someone else's numbers uses the admin pages. + +Everything on the page covers a trailing 30 day window ending today. + +The sections appear in the order AI usage, Desktops, Jobs. If you have not used a model in the window the AI section is not shown at all. It does appear, with a message, when IDEA could not read your usage, because a failed read is not the same as no usage. + +## AI usage + +One row per project you belong to and used, with the requests and tokens IDEA attributed to you, and your share of that project's Bedrock spend. Expand a row to see the same figures per model. + +The cost is an apportionment, not a measurement. IDEA reads the project's Bedrock spend for the window, then splits it by your share of the project's tokens. If two people in a project spent tokens on different models at different prices, the split will not match what each of them actually cost. + +When the project's spend cannot be read, the row shows **Not available** rather than a zero. + +## Jobs + +The jobs you completed in the window, taken from the cost estimate the scheduler recorded for each one at submission time. The totals and the per project and per queue breakdowns cover every job in the window; the table underneath lists your 20 most recent. + +A job's estimate is what the scheduler predicted the job would cost, not what the instances it ran on were billed at. Spot pricing, savings plans and reserved capacity are not reflected. + +## Desktops + +Your virtual desktop sessions that were up at some point in the window, with the hours IDEA recorded and those hours priced at the public on-demand rate for the instance type. + +A desktop you have since deleted still appears, shown as **Terminated**, and still counts toward your hours and cost for the time it ran. IDEA keeps a small record of a terminated desktop for 400 days, which is what this page reads once the desktop itself is gone. + +IDEA does not record billable desktop uptime, so the hours are derived: a running session is counted to now, and a session that is no longer running is counted to its last update. A session that was stopped and then edited reads long, and a session stopped and restarted inside the window reads as one stretch. + +When there is no on-demand price for an instance type, the row shows its hours and **Price not available** instead of a cost. + +The same rule applies to the section total. If none of your sessions could be priced the Cost reads **Not available** rather than $0.00. If only some could, the total carries a `*` and a line underneath saying how many sessions are missing from it. Job costs behave the same way when the scheduler recorded no estimate for a job. + +{% hint style="info" %} +If every desktop reads **Price not available**, the cluster-manager role is probably missing `pricing:GetProducts`. That permission ships with the cluster-manager policy, so an existing cluster needs the module redeployed to pick it up. Tell your administrator. +{% endhint %} + +## What this page is not + +{% hint style="warning" %} +These are IDEA's own measurements, not your AWS bill, which is why every figure is labeled **Estimated**. Nothing on this page comes from AWS Cost Explorer and nothing is reconciled against an invoice. Use it to see the shape of what you are consuming, not to settle a chargeback. +{% endhint %} + +Costs that IDEA never sees do not appear here at all: storage, data transfer, the cluster's own infrastructure, and anything you launched outside IDEA. A section shows as unavailable when the underlying read failed or the module is not deployed. An empty section means IDEA recorded nothing for you in the window. + +## For administrators + +Administrators get a second, all-users view at **Cluster Management > User Costs**. It lists every user with a measured cost in the window, one row each, with AI tokens and cost, desktop hours and cost, job count and cost, and a total. Selecting a user shows the same three sections that user sees on their own page. + +The two pages read the same measurements and apply the same rules, including **Not available** in place of a misleading zero. Where a whole source could not be read, the listing says so above the table. + +### Pricing outside the commercial partition + +The AWS Price List Query API has a single commercial endpoint and none in the GovCloud or China partitions, so on those partitions IDEA prices instance hours from the public AWS price list files instead. It reads the EC2 offer file for the cluster region over HTTPS from `pricing.us-east-1.amazonaws.com`, so the cluster needs outbound HTTPS to that host; no credentials are sent and the file is public. + +The offer file is a few hundred megabytes, so it loads on a background thread and refreshes once a day, and desktop and job costs read **Not available** until the first load finishes. A load that fails is logged and not retried for six hours. The same fallback is used in the commercial partition when the Pricing API call itself fails. + +Access is enforced on the server, not by hiding the link. The admin API requires elevated access, and the self-scoped one accepts no username at all. diff --git a/docs/modules/cluster-manager/README.md b/docs/modules/cluster-manager/README.md index ab9be369..324d2e4d 100644 --- a/docs/modules/cluster-manager/README.md +++ b/docs/modules/cluster-manager/README.md @@ -18,4 +18,4 @@ The Cluster Manager module is a central component of the IDEA platform. It serve ## Related Topics -Refer to the [Budget section](../best-practices/budget/ "mention") to learn more about configuring budgets for projects and users. +Refer to the [Budget section](../../best-practices/budget/ "mention") to learn more about configuring budgets for projects and users. diff --git a/docs/modules/cluster-manager/ai-usage-tracking.md b/docs/modules/cluster-manager/ai-usage-tracking.md index 6d90ce1e..47804433 100644 --- a/docs/modules/cluster-manager/ai-usage-tracking.md +++ b/docs/modules/cluster-manager/ai-usage-tracking.md @@ -4,9 +4,30 @@ description: Read the Amazon Bedrock invocation logging that AI usage reporting # AI Usage Tracking -The **AI Usage** column on the **Projects** page reports month to date tokens and requests per project, broken down per model, alongside the project's month to date Bedrock spend. Tokens are aggregated from Amazon Bedrock **model invocation logging**, not from billing data, so they are visible within the hour rather than a day later. If invocation logging is not delivering to the cluster log group, nothing is collected and every project reads as unused. +The **AI Usage** column on the **Projects** page reports the last 30 days of tokens and requests per project, alongside the project's Bedrock spend over the same 30 days. The breakdown per model and per user is on the **AI Usage** page under **Cluster Management**. -The spend figure comes from Cost Explorer, filtered to the project's `idea:Project` cost allocation tag and summed over the Bedrock services, so it trails the recorded tokens by about a day. It is available in the commercial partition only; where Cost Explorer cannot answer the column reads "cost unavailable" rather than zero. Per user attribution is not in the column: read it through the usage API and the downstream tooling built on it. +The window is a trailing one ending today, not the calendar month, so a project last used a few weeks ago still reads as used on the first of a month. Tokens are aggregated from Amazon Bedrock **model invocation logging**, not from billing data, and the aggregation runs every 15 minutes by default. If invocation logging is not delivering to the cluster log group, nothing is collected and every project reads as unused. + +Project budgets are unaffected by this window. They are enforced per calendar month by AWS Budgets against the project's own actual spend, which is where model charges land through the cost allocation tag. + +The spend figure comes from AWS Cost Explorer, filtered to the project's `idea:Project` cost allocation tag and summed over the Bedrock services for the same 30 days, so it trails the recorded tokens by about a day. It is available in the commercial partition only; where Cost Explorer cannot answer the column reads "cost unavailable" rather than zero. + +## The AI Usage page + +**Cluster Management** > **AI Usage** lists every project with a Bedrock configuration over the same trailing 30 days: tokens, requests, cost and the model the project spent the most tokens on. A project that has not been used is still listed, reading as no usage, so an idle project is distinguishable from a missing one. + +Selecting a project opens a breakdown in the split panel: + +| Table | Columns | +| --- | --- | +| Per model | Input tokens, output tokens, total tokens, requests, cost | +| Per user | Tokens, requests, cost, the user's top model | + +The input and output token split is recorded per day, per user and per model, so both figures are counted rather than inferred. + +The cost in both tables is **estimated** and labeled as such in the portal. AWS Cost Explorer prices a cost allocation tag, not a model or a caller, so the project's 30 day spend is shared out in proportion to tokens. Only the project level figure is a priced total. Where Cost Explorer has no answer for the project, the breakdown carries no cost at all rather than an estimate of zero. + +The page is administrator only: `Projects.ListBedrockUsage` has no non-elevated route, so a project member never reads another user's attribution. ## Model invocation logging @@ -27,7 +48,7 @@ Setting `manage_configuration` to false leaves the account and region configurat --aws-region ``` -The setting is read on the next aggregation cycle, which runs hourly by default. +The setting is read on the next aggregation cycle, which runs every 15 minutes by default (`bedrock.usage.interval_minutes`). To configure logging yourself, in the Bedrock console under **Settings** > **Model invocation logging**, enable logging to CloudWatch Logs with the log group named in `cluster-manager.bedrock.invocation_log_group_name` and the role in `cluster-manager.bedrock.invocation_log_role_arn`. Usage is attributed only for records delivered to that log group. @@ -36,8 +57,8 @@ To configure logging yourself, in the Bedrock console under **Settings** > **Mod | Cell | Meaning | | --- | --- | | `--` | Bedrock is not enabled for the project. | -| Tokens and requests | Usage recorded for the current month. | -| `No usage recorded` | Logging is being managed and no invocation was recorded for the project this month. | +| Tokens and requests | Usage recorded over the last 30 days. | +| `No usage recorded` | Logging is being managed and no invocation was recorded for the project in the last 30 days. | | `Not collected` | `manage_configuration` is false, so IDEA is not setting the account configuration. Unless logging was configured outside IDEA, no usage is collected for any project. | | `Usage unavailable` | The usage read failed. Check the cluster-manager logs. | @@ -72,5 +93,6 @@ Enforcement is only as current as the budget it reads. The AWS Budgets figure is * Prompts and completions are not delivered unless `bedrock.invocation_logging.include_request_response_data` is set to true. Leaving it false keeps the log group to metadata and token counts. * An invocation is attributed to the user who owns the instance that made the call. Calls from hosts that are not IDEA sessions or jobs are counted against no project, and calls from a project role with no owning instance land in an unattributed bucket. -* `bedrock.usage.lookback_days` is the trailing window recomputed on every run. Keep it at or below `bedrock.invocation_logging.log_retention_in_days`: nothing enforces the relationship, and a window longer than retention simply reads days the log group no longer holds. Usage from before logging was enabled is not backfilled. +* `bedrock.usage.lookback_days` is the trailing window of invocation logs recomputed on every run, and is unrelated to the 30 days the column reports: the column is served from the stored per day rows, not by re-reading the logs. Keep `lookback_days` at or below `bedrock.invocation_logging.log_retention_in_days`: nothing enforces the relationship, and a window longer than retention simply reads days the log group no longer holds. Usage from before logging was enabled is not backfilled. +* `bedrock.usage.retention_days` is the time to live on the stored usage rows, 400 days by default. It must stay comfortably above the 30 days the column reports; the cluster-manager holds it to a floor of 45 days whatever it is set to. * Usage is not spend. The priced equivalent reaches Cost Explorer and the project budget about a day later, and is not backfilled for activity recorded before cost allocation tags were activated. Recorded tokens are never valued as money and feed no limit. diff --git a/docs/modules/cluster-manager/enable-amazon-bedrock.md b/docs/modules/cluster-manager/enable-amazon-bedrock.md new file mode 100644 index 00000000..97844a71 --- /dev/null +++ b/docs/modules/cluster-manager/enable-amazon-bedrock.md @@ -0,0 +1,38 @@ +--- +description: Turn on Amazon Bedrock model access for the cluster, projects, and HPC jobs +--- + +# Enable Amazon Bedrock + +Amazon Bedrock access in IDEA is off by default and gated at three levels: the AWS account, the cluster, and each project. All three have to be on before a user can call a model. + +## 1. Request model access in the console + +Before IDEA can provision anything, the account itself needs access to the foundation models you plan to use. In the Bedrock console, under **Model access**, request access to each model. This is a one-time, per-account, per-region grant. IDEA does not make this request for you, and a model missing from this list fails every invocation regardless of the configuration below. + +## 2. Turn Bedrock on for the cluster + +```bash +./idea-admin.sh config \ + set "Key=cluster-manager.bedrock.enabled,Type=bool,Value=true" \ + --cluster-name \ + --aws-region +``` + +Then set the model catalog, `cluster-manager.bedrock.model_ids`. Use geographic inference profile ids, prefixed `us.`, `eu.` or `apac.`, rather than base model ids or `global.` ids. A global id is rejected at config time because it can route a call outside the account's geography; use the profile id for your own region's geography instead. + +{% hint style="warning" %} +Toggling `cluster-manager.bedrock.enabled` requires a redeploy of both **cluster-manager** and **virtual-desktop-controller**. The flag gates the IAM permissions boundary and inference profile provisioning cluster-manager owns, and the invocation logging destination virtual-desktop-controller's hosts write against. +{% endhint %} + +## 3. Turn it on per project + +Cluster-level enablement provisions nothing by itself. Each project opts in separately, and opt-in defaults to off. See [Projects Management](projects-management.md) for enabling Bedrock on a project, picking its models, and what gets provisioned. + +## 4. Turn it on for HPC jobs + +Desktops pick up project-level Bedrock access on their own. Jobs do not: set `scheduler.bedrock.enabled` to `true` as well and redeploy the scheduler module. Without it, jobs keep running under the compute node role and get no model access, even on a Bedrock-enabled project. + +## Usage and spend + +The Projects page reports tokens and requests per model, aggregated from Bedrock invocation logging. Spend is a separate figure pulled from AWS Cost Explorer, which has no endpoint in GovCloud, so spend reads "cost unavailable" there; token counts are unaffected. See [AI Usage Tracking](ai-usage-tracking.md) for both. diff --git a/docs/modules/cluster-manager/maintenance-banner.md b/docs/modules/cluster-manager/maintenance-banner.md new file mode 100644 index 00000000..5b164f11 --- /dev/null +++ b/docs/modules/cluster-manager/maintenance-banner.md @@ -0,0 +1,48 @@ +--- +description: Tell users what is happening during a maintenance window, from the portal and at job submission +--- + +# Maintenance Banner + +A maintenance window puts a warning banner on every page of the web portal, including the sign-in page, and makes the scheduler refuse new job submissions with the same message. It is off by default. + +It is meant for the days before an upgrade, when the scheduler has been stopped but the portal is still up. Without it, a user submitting a job gets a connection error and no explanation. + +## Settings + +| Key | Type | Default | Notes | +| --- | --- | --- | --- | +| `cluster-manager.maintenance.enabled` | bool | `false` | Shows the banner and refuses job submissions | +| `cluster-manager.maintenance.message` | string | empty | Plain text. No markup, no links | +| `cluster-manager.maintenance.ends_at` | string | empty | Optional ISO 8601 timestamp | + +All three are read live from the config store. Changing them needs no redeploy and no module restart. Open portal pages pick a change up within a minute, and the scheduler within about half a minute. + +`ends_at` is optional. When it is set, the banner appends "until" and the time, rendered in each user's own timezone. A value with no offset is read as UTC, so `2026-09-15T18:00:00Z` and `2026-09-15T18:00:00` mean the same thing. A value that cannot be parsed is dropped and the rest of the banner still shows. + +If the window is turned on with no message, the banner and the job rejection both read "This cluster is undergoing maintenance." + +## Setting it from the portal + +Cluster Management, then Settings, then the **Maintenance** tab. Set the message and the end time, turn the toggle on, and save. All three values are written together, so the banner never appears carrying the previous window's text. + +You need to be a cluster administrator. Every signed-in user can read the three keys, which is how the banner reaches them; the rest of the cluster-manager settings stay admin-only. + +## Setting it from the CLI + +```bash +./idea-admin.sh config \ + set "Key=cluster-manager.maintenance.enabled,Type=bool,Value=true" \ + "Key=cluster-manager.maintenance.message,Type=string,Value=HPC scheduler is closed for a cluster upgrade. Running desktops are unaffected." \ + "Key=cluster-manager.maintenance.ends_at,Type=string,Value=2026-09-15T18:00:00Z" \ + --cluster-name \ + --aws-region +``` + +To close the window, set `enabled` back to `false`. + +## What it does not cover + +The banner is served by the cluster-manager module. During the few minutes that module is itself being replaced, the portal is down and serves nothing at all, so nobody sees a banner. Announce that outage separately. + +A user running `qsub` directly on the login node is also not covered. Submissions from the portal are refused with the maintenance message; a direct `qsub` against a stopped scheduler fails the way PBS fails. diff --git a/docs/modules/hpc-workloads/admin-documentation/README.md b/docs/modules/hpc-workloads/admin-documentation/README.md index 2416d9d8..31968881 100644 --- a/docs/modules/hpc-workloads/admin-documentation/README.md +++ b/docs/modules/hpc-workloads/admin-documentation/README.md @@ -11,6 +11,7 @@ HPC workload administrators have extensive capabilities to manage job submission * [Queue Profiles](queue-profiles.md) - Configure job execution environments and default parameters * [Web-Based Job Submission](create-web-based-job-submission-workflows.md) - Create custom forms for simplified job submission * [License Resources](configure-floating-license-resources.md) - Manage floating license resources for applications +* [Custom AMIs](custom-amis.md) - See which images the cluster launches from and build new ones from the portal * [OpenPBS Hooks](openpbs-hooks.md) - Understand and configure OpenPBS hooks used by IDEA ## Key Administrative Capabilities diff --git a/docs/modules/hpc-workloads/admin-documentation/custom-amis.md b/docs/modules/hpc-workloads/admin-documentation/custom-amis.md new file mode 100644 index 00000000..8df94d75 --- /dev/null +++ b/docs/modules/hpc-workloads/admin-documentation/custom-amis.md @@ -0,0 +1,39 @@ +# Custom AMIs + +Scale-Out Computing > Custom AMIs lists the images your cluster launches from and lets an administrator build new ones from the portal. Use it to see which operating systems have a built image and which still launch from a stock vendor image. + +## What the page shows + +Two tables, one row per base OS. + +**Compute images** are what jobs run on. The row shows the image the cluster uses for that OS today: the scheduler default (`scheduler.compute_node_ami`) first, then any queue profile whose `instance_ami` names an image of that OS. Operating systems nothing references show the newest `idea-compute-node--*` image in the account, if there is one. + +**Desktop images** are the `ss-base---base` software stacks eVDI desktops launch from. Windows is not listed; there is no build for it. Each base stack carries two images: `ami_id`, what desktops launch from, and `base_ami_id`, the stock image the next build starts from (shown as the base subline). The Refresh Base Stack AMIs action always moves `base_ami_id` to the newest stock image and moves `ami_id` only while it is still a stock image; a stack launching from a built image keeps it until you rebuild, and shows `Built (base outdated)` when the base has moved past what the image was built from. **Use built image** on a row points the stack back at its last completed build without rebuilding. + +Each row carries: + +* **State**: `Built` when the image was built by IDEA (`idea-compute-node-*` or `idea-dcv-host-*`), `Stock` when it is a vendor image, `Missing` when the referenced AMI no longer exists in the account, `None` when nothing references the OS, `Building` while a build is running. +* **Build date**: parsed from the image name. +* **Referenced by**: the scheduler default and queue profiles for compute; the base stack and the number of custom stacks sharing the image for desktops. +* **Last build**: how the most recent build for that OS ended, with the builder instance id and the error when it failed. Builds started with `ideactl` show up here too. + +## Build + +Build launches a builder instance from a stock image, runs the instance-agnostic half of the node bootstrap (packages, system updates, drivers, DCV for desktops), snapshots it and terminates the builder. It takes about 20 minutes and costs one instance hour plus the snapshot. The base image defaults to the newest stock image the vendor publishes for that OS; a build never starts from a previous build. You can override the base AMI and the instance type in the dialog. + +* **Compute**: the EFA and FSx for Lustre drivers are included by default; uncheck them only for an image that will never touch them. The new image is not made the default automatically. When it is ready, **Set as default** on the row updates `scheduler.compute_node_ami`. Queue profiles pinned to an explicit `instance_ami` are edited in Queue Profiles. +* **Desktop**: the base stack is repointed at the new image and reindexed when the build finishes (on by default). Running desktops are unaffected; new desktops from that stack use the new image and reach READY in a few minutes instead of 15 or more. **Build all desktop images** on the desktop table starts one build per base stack in parallel with no inputs; rows already building are skipped and each stack is repointed only after its own build succeeds. + +A build that fails leaves its builder instance stopped, not terminated, so the bootstrap logs stay available under `/apps///ami_builder/`. Terminate it once you have what you need. Only one build per OS and architecture runs at a time; a row still building three hours after it started is marked failed, which usually means the module restarted mid-build. + +## Command line equivalents + +```bash +# on the scheduler host, as root +ideactl ami-builder build --base-os rocky9 --base-ami --enable-driver efa --enable-driver fsx_lustre --force + +# on the virtual desktop controller host, as root +ideactl build-desktop-image --base-os rocky9 --base-ami --update-stack --force +``` + +Both write the same build record the page reads, so a scripted build and a clicked one look identical afterwards. diff --git a/docs/modules/hpc-workloads/user-documentation/supported-ec2-parameters.md b/docs/modules/hpc-workloads/user-documentation/supported-ec2-parameters.md index a3ad8ac6..1b2933e5 100644 --- a/docs/modules/hpc-workloads/user-documentation/supported-ec2-parameters.md +++ b/docs/modules/hpc-workloads/user-documentation/supported-ec2-parameters.md @@ -148,6 +148,15 @@ Some instance types are On-Demand only, the HPC families among them. A spot requ If you specify more than 1 subnet and have `placement_group` set to True, IDEA will automatically provision capacity and placement group on the first subnet from the list {% endhint %} +Administrators can set `cluster.network.preferred_subnet_id` in the cluster configuration. Jobs that +do not pass `subnet_id` themselves then try the preferred subnet first, and the remaining private +subnets stay available, so a capacity shortfall in the preferred zone still provisions elsewhere. +Point it at the subnet holding a single availability zone shared filesystem to keep job I/O out of a +cross zone path. + +The same setting places virtual desktops, so one value covers both. Leaving it empty preserves the +random selection described above, and a job that passes `subnet_id` explicitly is never affected. + ## Storage {% hint style="info" %} diff --git a/docs/modules/virtual-desktop-interfaces/admin-documentation/sessions.md b/docs/modules/virtual-desktop-interfaces/admin-documentation/sessions.md index f35b82d6..0c112bde 100644 --- a/docs/modules/virtual-desktop-interfaces/admin-documentation/sessions.md +++ b/docs/modules/virtual-desktop-interfaces/admin-documentation/sessions.md @@ -49,3 +49,19 @@ This administrative override allows flexibility for special use cases while main {% hint style="info" %} Sessions created by administrators on behalf of users will be owned by the selected user and will count against that user's session limit. {% endhint %} + +## Desktop Placement + +By default a desktop is launched into the first available subnet from the desktop subnet list, or from the cluster private subnets when no desktop list is configured. Setting `cluster.network.preferred_subnet_id` to one of those subnets moves it to the front of that order, so every session launched without an explicit subnet tries it first. Point it at the subnet holding a shared filesystem that lives in a single availability zone, such as an FSx for NetApp ONTAP single-AZ file system, to keep desktop I/O out of a cross zone path. + +If the preferred zone has no capacity for the requested instance type, the launch falls through to the next subnet within the same request, so the preference never costs a user a desktop. That fallback depends on `virtual-desktop-controller.dcv_session.network.subnet_autoretry`, which is true by default. If you turn it off, the preferred subnet becomes a hard pin and every desktop fails while its availability zone is out of capacity; the controller logs a warning at startup when it finds that combination. + +The same setting places scheduler jobs, so one value covers both. Leaving it empty keeps the existing behavior, and a session launched with an explicit subnet ID under Advanced Options is never affected. + +## Desktop Event Queue + +Every session action reaches the controller as a message on the `--events.fifo` SQS queue. A message whose handler fails is left on the queue so it is retried, but a message that fails every time would be redelivered forever and hold up the events behind it. + +`virtual-desktop-controller.events.max_receive_count`, default `3`, bounds that. Below the bound the message is redelivered as before, so a transient failure still recovers on its own. On the bounding receive the controller logs the message ID, the event type and the session ID at error level, then deletes the message. Raise the setting to give slow-to-recover failures more attempts; it takes effect on the next controller restart and needs no redeploy. + +The queue also has a dead letter queue, `--events-dlq.fifo`, with a redrive policy of 16 receives, which catches messages the controller never processes at all, such as a restart between handling and deleting. The controller-side bound fires first, so anything reaching the dead letter queue was not dropped by the controller. Search the virtual desktop controller log for `Deleting the message instead of blocking the queue` to find the events that were dropped. diff --git a/docs/modules/virtual-desktop-interfaces/admin-documentation/virtual-desktop-images-software-stacks.md b/docs/modules/virtual-desktop-interfaces/admin-documentation/virtual-desktop-images-software-stacks.md index 0beddaab..a2aa1e09 100644 --- a/docs/modules/virtual-desktop-interfaces/admin-documentation/virtual-desktop-images-software-stacks.md +++ b/docs/modules/virtual-desktop-interfaces/admin-documentation/virtual-desktop-images-software-stacks.md @@ -120,3 +120,24 @@ This setting will override the global instance type restrictions and only show t ### Use your new Virtual Desktop Software Stack Once created, the Software Stack will be visible to all users who belong to the associated project(s). Refer to [create-a-virtual-desktop-linux-windows.md](../user-documentation/create-a-virtual-desktop-linux-windows.md "mention") to learn how to launch your desktop with the new image + +## Two images on a base stack + +A base software stack (`ss-base---base`) records `ami_id`, the image desktops launch from, and `base_ami_id`, the stock image the next build starts from. Refresh Base Stack AMIs (the portal action and `ideactl update-base-stacks`) always advances `base_ami_id` and changes `ami_id` only while it is still a stock image; once a build has repointed the stack, the built image stays until a rebuild or an explicit change. Custom AMIs shows both and offers Use built image to return to the last completed build. + +## Build a desktop image + +A desktop launched from a stock vendor AMI spends 13 to 18 minutes installing packages, DCV and drivers on its first boot. `ideactl build-desktop-image` moves that work into a reusable image: it launches a temporary instance from a stock base AMI, runs the instance-independent half of the desktop bootstrap (system packages, system updates, DCV server, session manager agent, and GPU drivers on a GPU instance type), and snapshots it as `idea-dcv-host--v`. Desktops launched from the built image run only per-session configuration and typically reach READY in a few minutes. + +Run it as root on the virtual desktop controller host: + +```bash +ideactl build-desktop-image \ + --base-os amazonlinux2023 \ + --base-ami \ + --update-stack --force +``` + +The build takes 20 to 30 minutes. With `--update-stack`, the matching `ss-base---base` software stack is pointed at the new image and the search index is rebuilt, so the next desktop from that stack uses it immediately; without the flag, update the stack's Instance AMI yourself. + +The instance profile, security groups, subnet and key pair default to the cluster's DCV host settings and can be overridden with the corresponding options. Built images carry no session or user state. Rebuild after changing DCV versions or GPU driver settings, or when the stock base AMI moves. diff --git a/docs/modules/virtual-desktop-interfaces/user-documentation/choose-an-ai-model.md b/docs/modules/virtual-desktop-interfaces/user-documentation/choose-an-ai-model.md index eeba2221..4676526d 100644 --- a/docs/modules/virtual-desktop-interfaces/user-documentation/choose-an-ai-model.md +++ b/docs/modules/virtual-desktop-interfaces/user-documentation/choose-an-ai-model.md @@ -118,4 +118,4 @@ Approval is a commitment by the account owner rather than a UI toggle. In commer ## Cost attribution -Usage is tagged with the project and shows up in that project's AWS budget. Cost allocation data arrives with the usual Cost Explorer delay of about a day and is not backfilled, so the first day of a new model reads low. +Usage is tagged with the project and shows up in that project's AWS budget. Cost allocation data arrives with the usual AWS Cost Explorer delay of about a day and is not backfilled, so the first day of a new model reads low. diff --git a/docs/modules/virtual-desktop-interfaces/user-documentation/create-a-virtual-desktop-linux-windows.md b/docs/modules/virtual-desktop-interfaces/user-documentation/create-a-virtual-desktop-linux-windows.md index 26c34d67..9cd8a6f4 100644 --- a/docs/modules/virtual-desktop-interfaces/user-documentation/create-a-virtual-desktop-linux-windows.md +++ b/docs/modules/virtual-desktop-interfaces/user-documentation/create-a-virtual-desktop-linux-windows.md @@ -39,6 +39,8 @@ The Create Session form has been improved to dynamically show only the options y Click "**Submit**" button to launch your virtual desktop creation. You will instantly see a new card with your desktop information. Your virtual desktop will be ready within 10-15 minutes. Startup time is based on the image selected, the operating system as well as the instance type. +Red Hat Enterprise Linux, Rocky Linux and Amazon Linux 2023 desktops skip the first-boot package update by default, while Ubuntu desktops always run one. An administrator can turn it on for the other three with `virtual-desktop-controller.dcv_session.first_boot_dnf_update`, which runs `dnf -y update` before the desktop is marked ready. It is off by default because it adds to the time a desktop takes to become available on its first launch. +

Desktop being started

Wait a couple of minutes until your desktop is ready. diff --git a/idea-admin-windows.ps1 b/idea-admin-windows.ps1 index 3523cf42..3b98141f 100755 --- a/idea-admin-windows.ps1 +++ b/idea-admin-windows.ps1 @@ -38,7 +38,7 @@ function Verify-Command($type,$message,$command) { $IDEADevMode = if ($Env:IDEA_DEV_MODE) {$Env:IDEA_DEV_MODE} else {""} $VirtualEnv = if ($Env:VIRTUAL_ENV) {$Env:VIRTUAL_ENV} else {""} $ScriptDir = $PSScriptRoot -$IDEARevision = if ($Env:IDEA_REVISION) {$Env:IDEA_REVISION} else {"v26.08.0"} +$IDEARevision = if ($Env:IDEA_REVISION) {$Env:IDEA_REVISION} else {"v26.09.0"} $IDEADockerRepo = "public.ecr.aws/s5o2b4m0" $DocumentationError = "https://docs.idea-hpc.com" $AWSProfile = if ($Env:AWS_PROFILE) {$Env:AWS_PROFILE} else {"default"} diff --git a/idea-admin.sh b/idea-admin.sh index 108d4677..7c3af898 100755 --- a/idea-admin.sh +++ b/idea-admin.sh @@ -26,9 +26,12 @@ # using Instance Profile credentials from EC2 Instance Metadata. # * IDEA_ADMIN_ENABLE_CDK_NAG_SCAN - Set to "false", if you want to disable cdk-nag scan. Default: true # * IDEA_DEV_MODE - Set to "true" if you are working with IDEA sources +# * IDEA_ADMIN_NO_TTY - Set to "true" to drop docker's -t flag even when stdin is a +# terminal. Non-interactive stdin (ssm, cron, CI) is detected +# automatically. Combine with --force for an unattended run. SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -IDEA_REVISION=${IDEA_REVISION:-"v26.08.0"} +IDEA_REVISION=${IDEA_REVISION:-"v26.09.0"} IDEA_DOCKER_REPO_DEFAULT="public.ecr.aws/s5o2b4m0/idea-administrator" IDEA_DOCKER_REPO=${IDEA_DOCKER_REPO:-"${IDEA_DOCKER_REPO_DEFAULT}"} IDEA_ECR_CREDS_RESET=${IDEA_ECR_CREDS_RESET:-"true"} @@ -118,12 +121,18 @@ if [[ "${IDEA_ECR_CREDS_RESET}" == "true" && "${IDEA_DOCKER_REPO}" == *"/"* ]]; # Check if user is connected to internet an can ping ECR repo DIG_BIN=$(command -v dig) IDEA_DOCKER_REPO_HOSTNAME=$(echo "${IDEA_DOCKER_REPO}" | cut -d '/' -f 1) - ${DIG_BIN} +tries=1 +time=3 "${IDEA_DOCKER_REPO_HOSTNAME}" >> /dev/null 2>&1 - verify_command "Unable to query ECR host ${IDEA_DOCKER_REPO_HOSTNAME} . Are you connected to internet?" + if [[ -z "${DIG_BIN}" ]]; then + # dig ships in bind-utils, which a stock Amazon Linux 2023 host does not have. The + # reset is a convenience, so skip it rather than fail the command that was asked for. + echo -e "${YELLOW}[INFO] dig not found: skipping ECR credentials reset. Install bind-utils, or set IDEA_ECR_CREDS_RESET=false to skip this step without the warning.${NC}" + else + ${DIG_BIN} +tries=1 +time=3 "${IDEA_DOCKER_REPO_HOSTNAME}" >> /dev/null 2>&1 + verify_command "Unable to query ECR host ${IDEA_DOCKER_REPO_HOSTNAME} . Are you connected to internet?" - ${DOCKER_BIN} logout public.ecr.aws >> /dev/null 2>&1 - verify_command "Failed to refresh ECR credentials. docker logout public.ecr.aws failed" - echo -e "${GREEN}✓ ECR credentials reset${NC}" + ${DOCKER_BIN} logout public.ecr.aws >> /dev/null 2>&1 + verify_command "Failed to refresh ECR credentials. docker logout public.ecr.aws failed" + echo -e "${GREEN}✓ ECR credentials reset${NC}" + fi else echo -e "${YELLOW}[INFO] Skipping ECR credentials reset (IDEA_ECR_CREDS_RESET=false)${NC}" fi @@ -141,9 +150,17 @@ fi IDEA_IMAGE_CREATED=$(${DOCKER_BIN} image inspect --format "{{.Created}}" "${IDEA_DOCKER_REPO}:${IDEA_REVISION}" 2>/dev/null) echo -e "${YELLOW}[INFO] Administrator image: ${IDEA_DOCKER_REPO}:${IDEA_REVISION} (created ${IDEA_IMAGE_CREATED:-unknown})${NC}" echo -e "${YELLOW}[INFO] Launching IDEA administrator...${NC}" +# Keep -it when stdin is an interactive terminal; otherwise drop -t so docker does not +# try to attach a TTY to non-interactive stdin. +if [[ -t 0 && "${IDEA_ADMIN_NO_TTY}" != "true" ]]; then + DOCKER_TTY_FLAGS="-it" +else + DOCKER_TTY_FLAGS="-i" +fi # Launch installer -${DOCKER_BIN} run --rm -it -v "${HOME}/.idea/clusters:/root/.idea/clusters" \ +${DOCKER_BIN} run --rm ${DOCKER_TTY_FLAGS} -v "${HOME}/.idea/clusters:/root/.idea/clusters" \ -e AWS_SESSION_TOKEN -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_PROFILE \ + -e AWS_REGION -e AWS_DEFAULT_REGION \ -e AWS_SDK_LOAD_CONFIG=1 \ -e IDEA_ADMIN_AWS_CREDENTIAL_PROVIDER="${IDEA_ADMIN_AWS_CREDENTIAL_PROVIDER}" \ -e IDEA_ADMIN_ENABLE_CDK_NAG_SCAN="${IDEA_ADMIN_ENABLE_CDK_NAG_SCAN}" \ diff --git a/requirements/idea-administrator.in b/requirements/idea-administrator.in index 8a04b625..cae8a7b8 100644 --- a/requirements/idea-administrator.in +++ b/requirements/idea-administrator.in @@ -7,3 +7,5 @@ aws-cdk-lib # via ~43 add_nag_suppression call sites; migrating is its own ticket. cdk-nag<3 prettytable +# needed by botocore's login credential provider (aws login profiles), the optional [crt] extra +awscrt diff --git a/requirements/idea-administrator.txt b/requirements/idea-administrator.txt index c792ec45..436c8c83 100644 --- a/requirements/idea-administrator.txt +++ b/requirements/idea-administrator.txt @@ -8,6 +8,7 @@ aws-cdk-asset-awscli-v1==2.2.292 aws-cdk-asset-node-proxy-agent-v6==2.1.2 aws-cdk-cloud-assembly-schema==54.19.0 aws-cdk-lib==2.265.0 +awscrt==0.36.2 blinker==1.9.0 boto3==1.43.73 botocore==1.43.73 @@ -25,6 +26,7 @@ cryptography==50.0.0 dataset==2.0.0 events==0.5 fastcounter==1.1.0 +greenlet==3.5.5 html5tagger==2.0.0 httptools==0.8.0 idna==3.19 diff --git a/scripts/dev/ami_update_stacks.py b/scripts/dev/ami_update_stacks.py index 12a914d7..b87ba191 100644 --- a/scripts/dev/ami_update_stacks.py +++ b/scripts/dev/ami_update_stacks.py @@ -168,8 +168,8 @@ def get_session(profile: str) -> Optional[boto3.Session]: def get_owners(ami_type: str, region: str) -> List[str]: - if region.startswith('us-gov-'): - return GOV_AMI_OWNERS.get(ami_type, AMI_OWNERS.get(ami_type, ['amazon'])) + if region.startswith('us-gov-') and ami_type in GOV_AMI_OWNERS: + return GOV_AMI_OWNERS[ami_type] return AMI_OWNERS.get(ami_type, ['amazon']) diff --git a/source/idea/idea-administrator/resources/config/templates/cluster-manager/settings.yml b/source/idea/idea-administrator/resources/config/templates/cluster-manager/settings.yml index 133f4f58..00a529cb 100644 --- a/source/idea/idea-administrator/resources/config/templates/cluster-manager/settings.yml +++ b/source/idea/idea-administrator/resources/config/templates/cluster-manager/settings.yml @@ -52,6 +52,17 @@ web_portal: title: 'Dashboard' url: ~ +# maintenance window notice, read live from the config store, so turning it on or off needs +# no redeploy. every portal page shows the banner while enabled is true, and the scheduler +# rejects new job submissions with the same message. +maintenance: + enabled: false + # plain text shown to users. no markup, no links. + message: '' + # optional ISO 8601 timestamp, e.g. '2026-09-15T18:00:00Z'. when set, the banner adds + # "until ". an offset-less value is read as UTC. left empty, no time is shown. + ends_at: '' + oauth2_client: # cluster manager OAuth 2.0 client is used for managing authentication for web-portal (when single sign-on is disabled) # set the below to an appropriate value based on your requirements and security posture. @@ -120,11 +131,11 @@ bedrock: include_request_response_data: false # true also delivers prompts and completions to the log group usage: enabled: true - interval_minutes: 60 # how often invocation records are aggregated + interval_minutes: 15 # how often invocation records are aggregated lookback_days: 2 # trailing days recomputed on every run (must be <= log_retention_in_days) max_query_results: 10000 query_timeout_seconds: 300 - retention_days: 400 # time to live on the usage rows + retention_days: 400 # time to live on the usage rows (must exceed the 30 day reporting window) max_users_per_project: 50 # per-user rows returned with a project # off by default. spend is the project's actual bedrock cost from cost explorer, about a # day behind and never estimated; an exhausted budget blocks new desktops/jobs but does diff --git a/source/idea/idea-administrator/resources/config/templates/cluster/settings.yml b/source/idea/idea-administrator/resources/config/templates/cluster/settings.yml index 40f60bd8..72aa84bd 100644 --- a/source/idea/idea-administrator/resources/config/templates/cluster/settings.yml +++ b/source/idea/idea-administrator/resources/config/templates/cluster/settings.yml @@ -37,6 +37,16 @@ network: {{ utils.to_yaml(client_ip) | indent(4) }} ssh_key_pair: "{{ssh_key_pair_name}}" + # subnet in the availability zone of a single-AZ shared filesystem, for example an FSx + # for NetApp ONTAP single-AZ file system. jobs and desktops submitted without an explicit + # subnet try it first and fall back to the other private subnets. leave empty to keep the + # default placement. + # desktops fall back only while + # virtual-desktop-controller.dcv_session.network.subnet_autoretry is true, the default; + # with autoretry off this is a hard pin. a job that needs a single subnet takes the + # preferred one on its first attempt and draws from the others on each retry. + preferred_subnet_id: "" + {% if use_existing_vpc %} use_existing_vpc: true vpc_id: "{{vpc_id}}" diff --git a/source/idea/idea-administrator/resources/config/templates/global-settings/settings.yml b/source/idea/idea-administrator/resources/config/templates/global-settings/settings.yml index cd6640c8..874b3ecb 100644 --- a/source/idea/idea-administrator/resources/config/templates/global-settings/settings.yml +++ b/source/idea/idea-administrator/resources/config/templates/global-settings/settings.yml @@ -221,6 +221,10 @@ package_config: - krb5-user - sssd-krb5 + # kept non-empty for the upgrade window. an upgrade overwrites global-settings + # before the modules themselves redeploy, and a module still running the previous + # SDK reads an empty list as NULL and fails bootstrap on ' '.join(None). safe to + # empty once no cluster runs a module older than this release. sssd_7: # Additional packages required for amazonlinux2023 - python-sssdconfig diff --git a/source/idea/idea-administrator/resources/config/templates/scheduler/settings.yml b/source/idea/idea-administrator/resources/config/templates/scheduler/settings.yml index cab3c174..7d111a2e 100644 --- a/source/idea/idea-administrator/resources/config/templates/scheduler/settings.yml +++ b/source/idea/idea-administrator/resources/config/templates/scheduler/settings.yml @@ -100,6 +100,10 @@ job_provisioning: # maximum provisioning retries for a job; once exhausted the job is held (failure reason recorded) # instead of re-queued - the owner can qdel and resubmit, or qrls to retry. + # one failed compute stack costs one retry, however many times it is seen while + # cloudformation deletes it. a stack that cannot be deleted costs a retry every cycle it + # blocks the job, so the cap still holds the job. + # 0 or any negative value disables the cap: the job is retried indefinitely, never held. max_provisioning_retries: 3 # the interval at which node housekeeping session is executed. diff --git a/source/idea/idea-administrator/resources/config/templates/virtual-desktop-controller/settings.yml b/source/idea/idea-administrator/resources/config/templates/virtual-desktop-controller/settings.yml index 157cd4ea..1a7f1982 100644 --- a/source/idea/idea-administrator/resources/config/templates/virtual-desktop-controller/settings.yml +++ b/source/idea/idea-administrator/resources/config/templates/virtual-desktop-controller/settings.yml @@ -54,6 +54,11 @@ controller: priority: 13 path_patterns: [ '/{{ module_id }}/*' ] +events: + # how many times an event whose handler keeps failing is redelivered before the + # controller deletes it, so one bad event cannot block the queue indefinitely + max_receive_count: 3 + dcv_broker: autoscaling: public: false @@ -147,6 +152,8 @@ opensearch: dcv_session: idle_timeout: 240 # in minutes. Default is 240-minutes / 4-hours + # run dnf -y update on the first boot of rhel/rocky/al2023 desktops; ubuntu always updates + first_boot_dnf_update: false idle_timeout_warning: 300 # in seconds idle_autostop_delay: 60 # Time (in minutes) a disconnected (but running) DCV session is eligible for autostop. Default is 60-minutes / 1-hour idle_autostop_delay_max: 240 # Upper bound (in minutes) for the per-session idle_autostop_delay a user may set. Set to 0 to disable per-session overrides diff --git a/source/idea/idea-administrator/resources/policies/cluster-manager.yml b/source/idea/idea-administrator/resources/policies/cluster-manager.yml index e5349440..b75b775d 100644 --- a/source/idea/idea-administrator/resources/policies/cluster-manager.yml +++ b/source/idea/idea-administrator/resources/policies/cluster-manager.yml @@ -9,6 +9,7 @@ Statement: - ec2:DescribeNetworkInterfaces - ec2:DescribeInstances - ec2:DescribeInstanceTypes + - pricing:GetProducts - budgets:ViewBudget Resource: '*' Effect: Allow @@ -330,6 +331,16 @@ Statement: Effect: Allow Sid: BedrockUsageQueryResults + # the cost pages read desktops that no longer exist from the record the virtual + # desktop controller writes when it terminates one. read only: the controller owns it. + - Action: + - dynamodb:Query + - dynamodb:Scan + - dynamodb:DescribeTable + Resource: + - '{{ context.arns.get_ddb_table_arn(context.config.get_module_id("virtual-desktop-controller") + ".controller.session-history") }}' + Effect: Allow + - Action: - dynamodb:GetItem - dynamodb:Query diff --git a/source/idea/idea-administrator/resources/policies/scheduler.yml b/source/idea/idea-administrator/resources/policies/scheduler.yml index 8b74a717..bc5c40ee 100644 --- a/source/idea/idea-administrator/resources/policies/scheduler.yml +++ b/source/idea/idea-administrator/resources/policies/scheduler.yml @@ -195,6 +195,7 @@ Statement: - '{{ context.arns.get_ddb_table_arn(context.module_id + ".queue-profiles") }}' - '{{ context.arns.get_ddb_table_arn(context.module_id + ".applications") }}' - '{{ context.arns.get_ddb_table_arn(context.module_id + ".license-resources") }}' + - '{{ context.arns.get_ddb_table_arn(context.module_id + ".image-builds") }}' Effect: Allow - Condition: diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/config_generator.py b/source/idea/idea-administrator/src/ideaadministrator/app/config_generator.py index 88a33447..abe4a8b7 100644 --- a/source/idea/idea-administrator/src/ideaadministrator/app/config_generator.py +++ b/source/idea/idea-administrator/src/ideaadministrator/app/config_generator.py @@ -10,6 +10,7 @@ # and limitations under the License. from ideasdk.utils import Utils, Jinja2Utils +from ideasdk.config.soca_config import is_null_value from ideadatamodel import exceptions, constants from ideaadministrator.app_props import AdministratorProps @@ -701,7 +702,11 @@ def traverse_config( f'Config key name: {key} under: {prefix} cannot contain a dot(.), colon(:) or comma(,)' ) - value = Utils.get_any_value(key, config) + # Utils.get_any_value() flattens every empty value to None, which would store + # `key: []` in the cluster settings table as NULL and read it back as None. + value = config[key] + if is_null_value(value): + value = None if Utils.is_not_empty(prefix): path_prefix = f'{prefix}.{key}' diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/deployment_helper.py b/source/idea/idea-administrator/src/ideaadministrator/app/deployment_helper.py index 0541cabf..1f085e48 100644 --- a/source/idea/idea-administrator/src/ideaadministrator/app/deployment_helper.py +++ b/source/idea/idea-administrator/src/ideaadministrator/app/deployment_helper.py @@ -14,7 +14,7 @@ from ideaadministrator.app.cdk.cdk_invoker import CdkInvoker from ideasdk.utils import Utils, ModuleMetadataHelper -from typing import List +from typing import Dict, List from collections import OrderedDict import threading import botocore.exceptions @@ -159,6 +159,18 @@ def print_no_op_message(self): f'[{module_s}] are already deployed. use the --upgrade flag to re-deploy these modules.' ) + def deploy_module_capturing( + self, module_id: str, failures: Dict[str, BaseException] + ): + """ + deploy_module in a thread, recording the exception for invoke() to fail on. CdkInvoker + signals a failed cdk run with SystemExit, which would otherwise pass silently. + """ + try: + self.deploy_module(module_id=module_id) + except BaseException as e: + failures[module_id] = e + def invoke(self): if self.optimize_deployment and len(self.module_ids) > 1: optimized_deployment_order = self.get_optimized_deployment_order() @@ -168,12 +180,13 @@ def invoke(self): print(f'optimized deployment order: {optimized_deployment_order}') for modules in optimized_deployment_order: + failures: Dict[str, BaseException] = {} threads = [] for module_id in modules: thread = threading.Thread( name=f'Thread: {module_id}', - target=self.deploy_module, - kwargs={'module_id': module_id}, + target=self.deploy_module_capturing, + kwargs={'module_id': module_id, 'failures': failures}, ) threads.append(thread) thread.start() @@ -183,6 +196,17 @@ def invoke(self): for thread in threads: thread.join() + # the status check below cannot see this: a module already deployed by the + # previous release still reads 'deployed' after a failed re-deploy. + if failures: + raise exceptions.general_exception( + 'deployment failed. could not deploy module(s): ' + + ', '.join( + f'{module_id} ({error})' + for module_id, error in failures.items() + ) + ) + # check for deployment status of previous modules # if any of them are not deployed, skip deployment of the next module set try: diff --git a/source/idea/idea-administrator/src/ideaadministrator/app_main.py b/source/idea/idea-administrator/src/ideaadministrator/app_main.py index 10cdbb2c..c7c0868a 100644 --- a/source/idea/idea-administrator/src/ideaadministrator/app_main.py +++ b/source/idea/idea-administrator/src/ideaadministrator/app_main.py @@ -21,6 +21,7 @@ ) from ideadatamodel.constants import CLICK_SETTINGS from ideasdk.utils import Utils, ModuleMetadataHelper +from ideasdk.aws.image_builds import COMPUTE_IMAGE_PREFIX, image_state from ideasdk.user_input.framework import SocaUserInputParamRegistry, SocaUserInputArgs from ideasdk.config.cluster_config_db import ClusterConfigDB from ideasdk.config.cluster_config import ClusterConfig @@ -52,8 +53,9 @@ from ideaadministrator.app.shared_storage_helper import SharedStorageHelper from prettytable import PrettyTable -from typing import Dict, List, Tuple +from typing import Dict, List, Optional, Set, Tuple import os +import re import sys import click import requests @@ -308,6 +310,118 @@ def config_generate( return values +def _values_file_s3_location( + cluster_name: str, aws_region: str, aws_profile: str, context: SocaCliContext +) -> Tuple[str, str]: + """(bucket name, s3 key) of the cluster's saved values.yml""" + cluster_config_db = ClusterConfigDB( + cluster_name=cluster_name, aws_region=aws_region, aws_profile=aws_profile + ) + return ( + get_bucket_name(cluster_name, aws_region, cluster_config_db, context), + ValuesDiff(cluster_name, aws_region).get_values_file_s3_key(), + ) + + +def download_values_file( + cluster_name: str, + aws_region: str, + aws_profile: str, + context: SocaCliContext, + values_file: str = None, +) -> str: + """ + download values.yml from the cluster's s3 bucket to values_file, defaulting to the local + cluster region directory. exits naming both locations when the bucket has no copy. + """ + if Utils.is_empty(values_file): + values_file = ValuesDiff(cluster_name, aws_region).get_values_file_path() + + bucket_name, s3_key = _values_file_s3_location( + cluster_name, aws_region, aws_profile, context + ) + + try: + response = context.aws().s3().get_object(Bucket=bucket_name, Key=s3_key) + except Exception as e: + context.error( + f'Values file not found at {values_file} and could not be downloaded from ' + f's3://{bucket_name}/{s3_key}: {e}. Restore values.yml to {values_file} from ' + f'a backup, then upload it with: idea-admin.sh config save-values' + ) + raise SystemExit(1) + + with open(values_file, 'w') as f: + f.write(Utils.to_yaml(Utils.from_yaml(response['Body']))) + + context.info(f'downloaded s3://{bucket_name}/{s3_key} to {values_file}') + return values_file + + +def _warn_values_file_drift( + context: SocaCliContext, + cluster_name: str, + aws_region: str, + aws_profile: str, + values_file: str, +) -> None: + """ + The local values.yml is the one the upgrade uses, so say so when the cluster's bucket holds a + different one. Best effort: a bucket that cannot be read is not a reason to stop an upgrade. + """ + try: + bucket_name, s3_key = _values_file_s3_location( + cluster_name, aws_region, aws_profile, context + ) + response = context.aws().s3().get_object(Bucket=bucket_name, Key=s3_key) + saved = Utils.get_as_dict(Utils.from_yaml(response['Body']), {}) + with open(values_file, 'r') as f: + local = Utils.get_as_dict(Utils.from_yaml(f.read()), {}) + except Exception: + return + + differing = sorted( + key for key in set(saved) | set(local) if saved.get(key) != local.get(key) + ) + if not differing: + return + + shown = ', '.join(differing[:10]) + if len(differing) > 10: + shown += f', and {len(differing) - 10} more' + context.warning( + f'the local {values_file} and s3://{bucket_name}/{s3_key} differ in ' + f'{len(differing)} key(s): {shown}. The upgrade uses the local copy.' + ) + + +def upload_values_file( + cluster_name: str, + aws_region: str, + aws_profile: str, + context: SocaCliContext, + values_file: str = None, +) -> str: + """ + upload values.yml to the cluster's s3 bucket, defaulting to the local cluster region + directory. returns the s3 uri written. + """ + values_diff = ValuesDiff(cluster_name, aws_region) + if Utils.is_empty(values_file): + values_file = values_diff.get_values_file_path() + + cluster_config_db = ClusterConfigDB( + cluster_name=cluster_name, aws_region=aws_region, aws_profile=aws_profile + ) + bucket_name = get_bucket_name(cluster_name, aws_region, cluster_config_db, context) + s3_key = values_diff.get_values_file_s3_key() + + context.aws().s3().upload_file(Bucket=bucket_name, Filename=values_file, Key=s3_key) + + context.info(f'saved {values_file} to s3://{bucket_name}/{s3_key}') + return f's3://{bucket_name}/{s3_key}' + + @config.command('save-values', context_settings=CLICK_SETTINGS) @click.option('--cluster-name', required=True, help='Cluster Name') @click.option('--aws-profile', help='AWS Profile') @@ -322,22 +436,11 @@ def save_values(cluster_name: str, aws_profile: str, aws_region: str, values_fil options=SocaContextOptions(enable_aws_client_provider=True) ) - cluster_config_db = ClusterConfigDB( - cluster_name=cluster_name, aws_region=aws_region, aws_profile=aws_profile - ) - - bucket_name = get_bucket_name(cluster_name, aws_region, cluster_config_db, context) - - values_diff = ValuesDiff(cluster_name, aws_region) - if Utils.is_empty(values_file): - values_file = values_diff.get_values_file_path() + values_file = ValuesDiff(cluster_name, aws_region).get_values_file_path() print_using_default_warning('Values file', values_file, context) - context.info(f'Saving in bucket: {bucket_name} at location: values/value.yml') - context.aws().s3().upload_file( - Bucket=bucket_name, Filename=values_file, Key='values/values.yml' - ) + upload_values_file(cluster_name, aws_region, aws_profile, context, values_file) @config.command('update', context_settings=CLICK_SETTINGS) @@ -711,33 +814,18 @@ def download_values( options=SocaContextOptions(enable_aws_client_provider=True) ) - cluster_config_db = ClusterConfigDB( - cluster_name=cluster_name, aws_region=aws_region, aws_profile=aws_profile - ) - - values_diff = ValuesDiff(cluster_name, aws_region) + values_file = None if Utils.is_empty(values_dir): - values_file = values_diff.get_values_file_path() print_using_default_warning( - 'Values file directory', values_diff.get_cluster_region_dir(), context + 'Values file directory', + ValuesDiff(cluster_name, aws_region).get_cluster_region_dir(), + context, ) else: os.makedirs(values_dir, exist_ok=True) values_file = os.path.join(values_dir, 'values.yml') - bucket_name = get_bucket_name(cluster_name, aws_region, cluster_config_db, context) - - response = ( - context.aws() - .s3() - .get_object(Bucket=bucket_name, Key=values_diff.get_values_file_s3_key()) - ) - - values = Utils.from_yaml(response['Body']) - - print(f'saving values to: {values_file}') - with open(values_file, 'w') as f: - f.write(Utils.to_yaml(values)) + download_values_file(cluster_name, aws_region, aws_profile, context, values_file) @config.command('diff', context_settings=CLICK_SETTINGS) @@ -2354,12 +2442,22 @@ def _find_eol_base_os_references(db: ClusterConfigDB) -> List[str]: return findings -def _delete_eol_software_stacks(context, db: ClusterConfigDB): +def _describe_stack(stack: Dict) -> str: + return ( + f'{Utils.get_value_as_string("stack_id", stack)} ' + f'({Utils.get_value_as_string("name", stack, "")}, ' + f'{Utils.get_value_as_string("architecture", stack, "")})' + ) + + +def _plan_eol_software_stacks(db: ClusterConfigDB) -> List[Dict]: """ - Delete eVDI software stacks on an end-of-life base_os: they can no longer launch, and the - admin cannot edit a stack's base_os. A stack a live session still references aborts the - upgrade instead, with nothing deleted, because that desktop must be deleted by hand first. + Read-only: what the upgrade would do to eVDI software stacks on an end-of-life base_os. + They can no longer launch and their base_os cannot be edited, so they are deleted; a stack + a live session still references is disabled instead, leaving running desktops untouched. + One entry per eVDI module. """ + plans = [] for module in db.get_cluster_modules(): module_id = Utils.get_value_as_string('module_id', module) module_name = Utils.get_value_as_string('name', module) @@ -2381,7 +2479,7 @@ def _delete_eol_software_stacks(context, db: ClusterConfigDB): eol_stack_ids = { Utils.get_value_as_string('stack_id', stack) for stack in eol_stacks } - sessions = [] + sessions: List[Dict] = [] for session in _scan_module_table( db, f'{db.cluster_name}.{module_id}.controller.user-sessions' ): @@ -2389,56 +2487,138 @@ def _delete_eol_software_stacks(context, db: ClusterConfigDB): continue software_stack = Utils.get_value_as_dict('software_stack', session, {}) stack_id = Utils.get_value_as_string('stack_id', software_stack) - stack_base_os = Utils.get_value_as_string('base_os', software_stack) - session_base_os = Utils.get_value_as_string('base_os', session) - if ( - stack_id in eol_stack_ids - or stack_base_os in constants.EOL_BASEOS - or session_base_os in constants.EOL_BASEOS - ): - owner = Utils.get_value_as_string('owner', session, '') - idea_session_id = Utils.get_value_as_string( - 'idea_session_id', session, '' - ) + base_os = Utils.get_value_as_string( + 'base_os', software_stack + ) or Utils.get_value_as_string('base_os', session) + if stack_id in eol_stack_ids or base_os in constants.EOL_BASEOS: sessions.append( - f'session {idea_session_id} owned by {owner} on software stack ' - f'{stack_id or stack_base_os or session_base_os}' + { + 'stack_id': stack_id, + 'base_os': base_os, + 'session_id': Utils.get_value_as_string( + 'idea_session_id', session, '' + ), + 'owner': Utils.get_value_as_string( + 'owner', session, '' + ), + 'name': Utils.get_value_as_string('name', session, ''), + } ) - if Utils.is_not_empty(sessions): - context.error( - f'{len(sessions)} virtual desktop session(s) still use a Base OS that has ' - f'reached end-of-life. Nothing has been deleted.' - ) - for session in sessions: - context.error(f' - {session}') - context.error('Delete these virtual desktops, then re-run upgrade-cluster.') - raise SystemExit(1) + # a session that carries no stack_id can only be matched to a stack by base_os; keying on + # the stack reference alone deletes a stack a running desktop is still on. + stack_ids_in_use = { + session['stack_id'] for session in sessions if session['stack_id'] + } + base_os_in_use = { + session['base_os'] for session in sessions if not session['stack_id'] + } - table = db.aws.dynamodb_table().Table(stacks_table) + to_delete, to_disable = [], [] for stack in eol_stacks: - stack_id = Utils.get_value_as_string('stack_id', stack) + in_use = ( + Utils.get_value_as_string('stack_id', stack) in stack_ids_in_use + or Utils.get_value_as_string('base_os', stack) in base_os_in_use + ) + (to_disable if in_use else to_delete).append(stack) + + plans.append( + { + 'stacks_table': stacks_table, + 'sessions': sessions, + 'to_delete': to_delete, + 'to_disable': to_disable, + } + ) + return plans + + +def _stack_sessions(plan: Dict, stack: Dict) -> List[Dict]: + stack_id = Utils.get_value_as_string('stack_id', stack) + stack_base_os = Utils.get_value_as_string('base_os', stack) + return [ + session + for session in plan['sessions'] + if session['stack_id'] == stack_id + or (not session['stack_id'] and session['base_os'] == stack_base_os) + ] + + +def _apply_eol_software_stacks( + context, cluster_name: str, aws_region: str, aws_profile: str, plans: List[Dict] +) -> None: + """ + Delete or disable the software stacks _plan_eol_software_stacks found. Called only once the + upgrade is confirmed: this is the first change the upgrade makes to the cluster. + """ + if not plans: + return + db = ClusterConfigDB( + cluster_name=cluster_name, aws_region=aws_region, aws_profile=aws_profile + ) + disabled = 0 + for plan in plans: + table = db.aws.dynamodb_table().Table(plan['stacks_table']) + for stack in plan['to_disable']: + # enabled is a DynamoDB reserved word, hence the expression attribute name + table.update_item( + Key={ + 'base_os': Utils.get_value_as_string('base_os', stack), + 'stack_id': Utils.get_value_as_string('stack_id', stack), + }, + UpdateExpression='SET #enabled = :enabled', + ExpressionAttributeNames={'#enabled': 'enabled'}, + ExpressionAttributeValues={':enabled': False}, + ) + disabled += 1 + still_in_use = ', '.join( + f'{session["owner"]} ({session["name"]})' + for session in _stack_sessions(plan, stack) + ) + context.info( + f'disabled end-of-life eVDI software stack {_describe_stack(stack)}, ' + f'still in use by {still_in_use}' + ) + for stack in plan['to_delete']: table.delete_item( Key={ 'base_os': Utils.get_value_as_string('base_os', stack), - 'stack_id': stack_id, + 'stack_id': Utils.get_value_as_string('stack_id', stack), } ) - name = Utils.get_value_as_string('name', stack, '') - architecture = Utils.get_value_as_string('architecture', stack, '') context.info( - f'deleted end-of-life eVDI software stack {stack_id} ({name}, {architecture})' + f'deleted end-of-life eVDI software stack {_describe_stack(stack)}' ) - context.info( - f'deleted {len(eol_stacks)} end-of-life eVDI software stack(s) from {stacks_table}' + if plan['to_delete']: + context.info( + f'deleted {len(plan["to_delete"])} end-of-life eVDI software stack(s) from ' + f'{plan["stacks_table"]}' + ) + + if disabled: + # the DynamoDB row is the source of truth, but the portal lists stacks from the search + # index, so a disabled stack keeps showing as enabled until the index catches up. + context.warning( + f'{disabled} software stack(s) are disabled in DynamoDB but still read as enabled ' + f'in the eVDI search index until it is reindexed. The virtual-desktop-controller ' + f'redeploy later in this upgrade reconciles the index against DynamoDB at startup. ' + f'To reindex sooner, run "ideactl reindex-software-stacks --reset" on the ' + f'virtual-desktop-controller host.' ) -def _check_eol_base_os(context, cluster_name: str, aws_region: str, aws_profile: str): +def _check_eol_base_os( + context, + cluster_name: str, + aws_region: str, + aws_profile: str, + disable_stacks_in_use: bool = False, +) -> List[Dict]: """ - Clear every end-of-life base_os reference before the upgrade. Cluster settings and queue - profiles are migrated by the admin, eVDI software stacks are deleted here once no session - references them. + Report every end-of-life base_os reference before the upgrade, changing nothing. Cluster + settings and queue profiles are migrated by the admin and stay a hard stop, as does a live + session on an end-of-life software stack unless disable_stacks_in_use is set. Returns the + software stack plan for _apply_eol_software_stacks to run once the upgrade is confirmed. """ db = ClusterConfigDB( cluster_name=cluster_name, aws_region=aws_region, aws_profile=aws_profile @@ -2462,33 +2642,234 @@ def _check_eol_base_os(context, cluster_name: str, aws_region: str, aws_profile: ) raise SystemExit(1) - _delete_eol_software_stacks(context, db) + plans = _plan_eol_software_stacks(db) + sessions = [session for plan in plans for session in plan['sessions']] + if sessions and not disable_stacks_in_use: + context.error( + f'{len(sessions)} virtual desktop session(s) still use a Base OS that has ' + f'reached end-of-life. Nothing has been changed.' + ) + for session in sessions: + context.error( + f' - session {session["session_id"]} owned by {session["owner"]} on software ' + f'stack {session["stack_id"] or session["base_os"]}' + ) + context.error('Delete these virtual desktops, then re-run upgrade-cluster.') + context.error( + 'Alternatively, re-run upgrade-cluster with --disable-eol-stacks-in-use to ' + 'disable these software stacks and continue.' + ) + raise SystemExit(1) + + for plan in plans: + for stack in plan['to_disable']: + context.info( + f'will disable end-of-life eVDI software stack {_describe_stack(stack)}' + ) + for stack in plan['to_delete']: + context.info( + f'will delete end-of-life eVDI software stack {_describe_stack(stack)}' + ) + return plans + + +# module name -> the (base_os key, ami key) pairs that module templates, relative to its module +# id. Every module that templates base_os / instance_ami must appear here, or the module keeps its +# pre-upgrade OS while the rest of the cluster moves. +AMI_UPDATE_KEYS = { + constants.MODULE_BASTION_HOST: [('base_os', 'instance_ami')], + constants.MODULE_CLUSTER_MANAGER: [ + ('ec2.autoscaling.base_os', 'ec2.autoscaling.instance_ami') + ], + constants.MODULE_DIRECTORYSERVICE: [('base_os', 'instance_ami')], + constants.MODULE_SCHEDULER: [ + ('base_os', 'instance_ami'), + ('compute_node_os', 'compute_node_ami'), + ], + constants.MODULE_VIRTUAL_DESKTOP_CONTROLLER: [ + ('controller.autoscaling.base_os', 'controller.autoscaling.instance_ami'), + ('dcv_broker.autoscaling.base_os', 'dcv_broker.autoscaling.instance_ami'), + ( + 'dcv_connection_gateway.autoscaling.base_os', + 'dcv_connection_gateway.autoscaling.instance_ami', + ), + ], +} + + +def build_ami_update_entries( + ami_id: str, base_os: str, modules: List[Dict], keep_keys: Optional[Set[str]] = None +) -> List[str]: + """ + The `config set` entries applied by upgrade_cluster Phase 3, for the modules this cluster + actually deploys. Module ids come from the cluster config: config keys are prefixed with + the module id, so a hardcoded id writes nothing on a cluster that renamed its modules. + + A key named in keep_keys keeps its current value, and so does the other half of its pair: + writing one without the other leaves the module's base OS and AMI disagreeing. + """ + keep_keys = keep_keys or set() + entries = [] + for module in modules: + module_id = Utils.get_value_as_string('module_id', module) + key_pairs = AMI_UPDATE_KEYS.get(Utils.get_value_as_string('name', module)) + if Utils.is_empty(module_id) or not key_pairs: + continue + for base_os_key, ami_key in key_pairs: + keys = (f'{module_id}.{base_os_key}', f'{module_id}.{ami_key}') + if keep_keys.intersection(keys): + continue + entries.append(f'Key={keys[0]},Type=string,Value={base_os}') + entries.append(f'Key={keys[1]},Type=string,Value={ami_id}') + return entries + + +def keep_built_compute_image(current: Optional[Dict], stock: Optional[Dict]) -> bool: + """ + Whether an upgrade must leave the scheduler's compute image alone. True only for an image IDEA + built that is newer than the release's stock image: an operator who adopted a build from the + Custom AMIs page should not lose it, but a build older than the release is stale. Creation dates + are the ISO 8601 UTC strings EC2 returns, so they order as strings, and equal is not newer. + """ + if not current or not stock: + return False + name = Utils.get_value_as_string('Name', current) + if image_state(name, COMPUTE_IMAGE_PREFIX) != 'built': + return False + created = Utils.get_value_as_string('CreationDate', current, '') + release_created = Utils.get_value_as_string('CreationDate', stock, '') + if Utils.is_empty(created) or Utils.is_empty(release_created): + return False + return created > release_created -def build_ami_update_entries(ami_id: str, base_os: str) -> List[str]: +def resolve_compute_ami_keep_keys( + context, db: ClusterConfigDB, modules: List[Dict], ami_id: str, ec2 +) -> Set[str]: """ - Build the `config set` entries applied by upgrade_cluster Phase 3. - Every module that templates base_os / instance_ami must appear here, or the - module keeps its pre-upgrade OS while the rest of the cluster moves. + The scheduler compute keys Phase 3 must not write. Compute moves onto the release image like + every other module, unless the cluster already runs a newer image built from the Custom AMIs + page, which an unconditional rewrite would silently discard. Anything it cannot determine + leaves the keys writable, so an unreadable image never blocks an upgrade. """ - return [ - f'Key=bastion-host.base_os,Type=string,Value={base_os}', - f'Key=bastion-host.instance_ami,Type=string,Value={ami_id}', - f'Key=cluster-manager.ec2.autoscaling.base_os,Type=string,Value={base_os}', - f'Key=cluster-manager.ec2.autoscaling.instance_ami,Type=string,Value={ami_id}', - f'Key=directoryservice.base_os,Type=string,Value={base_os}', - f'Key=directoryservice.instance_ami,Type=string,Value={ami_id}', - f'Key=scheduler.base_os,Type=string,Value={base_os}', - f'Key=scheduler.instance_ami,Type=string,Value={ami_id}', - f'Key=scheduler.compute_node_os,Type=string,Value={base_os}', - f'Key=scheduler.compute_node_ami,Type=string,Value={ami_id}', - f'Key=vdc.controller.autoscaling.base_os,Type=string,Value={base_os}', - f'Key=vdc.controller.autoscaling.instance_ami,Type=string,Value={ami_id}', - f'Key=vdc.dcv_broker.autoscaling.base_os,Type=string,Value={base_os}', - f'Key=vdc.dcv_broker.autoscaling.instance_ami,Type=string,Value={ami_id}', - f'Key=vdc.dcv_connection_gateway.autoscaling.base_os,Type=string,Value={base_os}', - f'Key=vdc.dcv_connection_gateway.autoscaling.instance_ami,Type=string,Value={ami_id}', - ] + keep: Set[str] = set() + for module in modules: + if Utils.get_value_as_string('name', module) != constants.MODULE_SCHEDULER: + continue + module_id = Utils.get_value_as_string('module_id', module) + if Utils.is_empty(module_id): + continue + entry = db.get_config_entry(f'{module_id}.compute_node_ami') + current_ami = Utils.get_value_as_string('value', entry) if entry else None + if Utils.is_empty(current_ami) or current_ami == ami_id: + continue + + try: + described = ec2.describe_images(ImageIds=[current_ami, ami_id]) + except Exception as e: + context.warning( + f'could not describe compute image {current_ami} or release image {ami_id}: {e}. ' + f'Compute moves to {ami_id}.' + ) + continue + + images = { + image['ImageId']: image + for image in Utils.get_value_as_list('Images', described, []) + } + current = images.get(current_ami) + stock = images.get(ami_id) + if keep_built_compute_image(current, stock): + keep.update( + {f'{module_id}.compute_node_os', f'{module_id}.compute_node_ami'} + ) + context.info( + f'keeping built compute image {current_ami} ' + f'({Utils.get_value_as_string("Name", current)}, created ' + f'{Utils.get_value_as_string("CreationDate", current)}), newer than the release ' + f'image {ami_id} (created {Utils.get_value_as_string("CreationDate", stock)})' + ) + elif ( + current + and image_state( + Utils.get_value_as_string('Name', current), COMPUTE_IMAGE_PREFIX + ) + == 'built' + ): + context.info( + f'built compute image {current_ami} is older than the release image {ami_id} and ' + f'was replaced; rebuild it from the Custom AMIs page to run a built image again' + ) + return keep + + +def boto3_session(aws_profile: str): + """a boto3 session for the profile the command was given, or the ambient credentials""" + import boto3 + + if Utils.is_not_empty(aws_profile): + return boto3.Session(profile_name=aws_profile) + return boto3.Session() + + +def get_cluster_base_os(db: ClusterConfigDB) -> List[str]: + """ + The distinct Base OS values the cluster settings already carry, read from the same keys the + upgrade writes. More than one means the cluster is part-migrated and the upgrade cannot pick + for the admin. + """ + values = { + Utils.get_value_as_string('value', entry) + for entry in db.get_config_entries() + if Utils.get_value_as_string('key', entry, '').endswith('.base_os') + } + return sorted(value for value in values if Utils.is_not_empty(value)) + + +def resolve_upgrade_base_os( + context, cluster_name: str, aws_region: str, aws_profile: str, base_os: str +) -> str: + """ + The Base OS this upgrade applies. Without an explicit --base-os the cluster keeps the Base OS + its settings already carry, so an unattended --force run cannot silently move every module + onto a different OS. Refuses rather than guessing when that value cannot be read. + """ + try: + current = get_cluster_base_os( + ClusterConfigDB( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + ) + ) + except Exception as e: + if Utils.is_empty(base_os): + context.error( + f'Could not read the cluster settings to determine the current Base OS: {e}. ' + f'Re-run with an explicit --base-os.' + ) + raise SystemExit(1) + current = [] + + if Utils.is_empty(base_os): + if len(current) != 1: + found = ', '.join(current) if current else 'no base_os setting found' + context.error( + f'Could not determine the Base OS this cluster runs from its settings ({found}). ' + f'Re-run with an explicit --base-os to say which Base OS every module should use.' + ) + raise SystemExit(1) + context.info( + f'No --base-os given: keeping the Base OS this cluster runs: {current[0]}' + ) + return current[0] + + if current and base_os not in current: + context.warning( + f'--base-os {base_os} changes this cluster from {", ".join(current)}: every module ' + f'is redeployed onto {base_os}.' + ) + return base_os def get_module_instance_ids( @@ -2625,8 +3006,8 @@ def _warn_termination_protection_cleared(context, cleared: List[Tuple[str, str]] @click.option('--deployment-id', help='A UUID to identify the deployment.') @click.option( '--base-os', - default='amazonlinux2023', - help='New base OS to upgrade to (e.g., amazonlinux2023, rhel8, rhel9, rhel10, rocky8, rocky9, rocky10). Default: amazonlinux2023', + help='Base OS to upgrade to (e.g., amazonlinux2023, rhel8, rhel9, rhel10, rocky8, rocky9, ' + 'rocky10). Defaults to the Base OS the cluster already runs; pass this only to change it.', ) @click.option( '--force-build-bootstrap', @@ -2651,6 +3032,13 @@ def _warn_termination_protection_cleared(context, cleared: List[Tuple[str, str]] is_flag=True, help='Skip updating global settings. Use if you have already updated global settings.', ) +@click.option( + '--disable-eol-stacks-in-use', + is_flag=True, + help='Disable end-of-life eVDI software stacks that a live virtual desktop session still ' + 'uses, instead of aborting the upgrade. Running desktops are left untouched, but no new ' + 'session can launch from a disabled stack.', +) @click.argument('MODULES', required=False, nargs=-1) def upgrade_cluster( cluster_name: str, @@ -2665,6 +3053,7 @@ def upgrade_cluster( module_set: str, force: bool, skip_global_settings_update: bool, + disable_eol_stacks_in_use: bool, modules=None, ): """ @@ -2682,7 +3071,22 @@ def upgrade_cluster( ./idea-admin.sh upgrade-cluster cluster metrics scheduler --base-os amazonlinux2023 --aws-region us-east-2 --cluster-name idea-test1 """ - context = SocaCliContext() + context = SocaCliContext( + options=SocaContextOptions( + enable_aws_client_provider=True, + aws_region=aws_region, + aws_profile=aws_profile, + ) + ) + + # An unattended --force run must not move every module onto a different Base OS by default. + base_os = resolve_upgrade_base_os( + context=context, + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + base_os=base_os, + ) # If no modules are specified, use 'all' if not modules: @@ -2745,12 +3149,14 @@ def upgrade_cluster( ) raise SystemExit(1) - # Fail before any change is made if the cluster still points at an end-of-life Base OS. - _check_eol_base_os( + # Report end-of-life Base OS references before the upgrade is confirmed. This changes nothing: + # the software stack plan it returns is applied further down, once the admin has said yes. + eol_stack_plans = _check_eol_base_os( context=context, cluster_name=cluster_name, aws_region=aws_region, aws_profile=aws_profile, + disable_stacks_in_use=disable_eol_stacks_in_use, ) # Display upgrade context information @@ -2779,6 +3185,14 @@ def upgrade_cluster( context.info('Upgrade aborted by user') raise SystemExit(0) + _apply_eol_software_stacks( + context=context, + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + plans=eol_stack_plans, + ) + # bound before the try so the failure path can always report what the sweep cleared cleared_protection: List[Tuple[str, str]] = [] @@ -2805,31 +3219,13 @@ def upgrade_cluster( raise SystemExit(1) # Update base_os in the cluster values file first - props = AdministratorProps() - cluster_dir = props.cluster_dir(cluster_name) - cluster_region_dir = props.cluster_region_dir(cluster_dir, aws_region) - values_file_path = os.path.join(cluster_region_dir, 'values.yml') - - if os.path.exists(values_file_path): - with open(values_file_path, 'r') as f: - values = f.read() - - # Replace base_os value using standard regex module - import re - - updated_values = re.sub( - r'^base_os:.*$', f'base_os: {base_os}', values, flags=re.MULTILINE - ) - - with open(values_file_path, 'w') as f: - f.write(updated_values) - - context.success( - f'✅ Successfully updated base_os to {base_os} in values.yml' - ) - else: - context.error(f'Values file not found at {values_file_path}') - raise SystemExit(1) + _update_values_base_os( + context=context, + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + base_os=base_os, + ) # PHASE 2: Perform pre-upgrade configuration if not skipped config_generator = None @@ -2925,7 +3321,21 @@ def upgrade_cluster( ) context.info('🔄 Updating AMI IDs and base_os values in DynamoDB...') - ami_updates = build_ami_update_entries(ami_id=ami_id, base_os=base_os) + cluster_modules = db.get_cluster_modules() + ami_updates = build_ami_update_entries( + ami_id=ami_id, + base_os=base_os, + modules=cluster_modules, + keep_keys=resolve_compute_ami_keep_keys( + context=context, + db=db, + modules=cluster_modules, + ami_id=ami_id, + ec2=boto3_session(aws_profile).client( + 'ec2', region_name=aws_region + ), + ), + ) for entry in ami_updates: tokens = entry.split(',', 2) @@ -2984,13 +3394,7 @@ def upgrade_cluster( # cfn's delete of the original fail silently, leaving a running host no stack references. ec2 = None try: - import boto3 - - session = ( - boto3.Session(profile_name=aws_profile) - if Utils.is_not_empty(aws_profile) - else boto3.Session() - ) + session = boto3_session(aws_profile) ec2 = session.client('ec2', region_name=aws_region) cleared_protection = clear_termination_protection( get_module_instance_ids( @@ -3031,6 +3435,13 @@ def upgrade_cluster( f'could not restore termination protection: {e}. Re-enable it by hand.' ) + _save_values_file_to_bucket( + context=context, + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + ) + fancy_title(context, 'Cluster Upgrade Completed', '🎉') context.success('✅ All upgrade phases completed successfully') context.info( @@ -3047,6 +3458,71 @@ def upgrade_cluster( raise +def _update_values_base_os( + context: SocaCliContext, + cluster_name: str, + aws_region: str, + aws_profile: str, + base_os: str, +) -> str: + """ + set base_os in the cluster's values.yml, restoring the file from the cluster's s3 bucket + first when the local copy is missing. returns the values file path. + """ + values_file_path = ValuesDiff(cluster_name, aws_region).get_values_file_path() + + if os.path.exists(values_file_path): + _warn_values_file_drift( + context, cluster_name, aws_region, aws_profile, values_file_path + ) + else: + context.warning( + f'values.yml not found at {values_file_path}, ' + f'restoring it from the cluster s3 bucket ...' + ) + download_values_file(cluster_name, aws_region, aws_profile, context) + + with open(values_file_path, 'r') as f: + values = f.read() + + # a values.yml old enough to predate the key would otherwise be reported as updated. + updated_values, replaced = re.subn( + r'^base_os:.*$', f'base_os: {base_os}', values, flags=re.MULTILINE + ) + if not replaced: + context.error( + f'{values_file_path} has no base_os key, so the upgrade cannot set it to {base_os}. ' + f'Add "base_os: {base_os}" to the file and re-run upgrade-cluster.' + ) + raise SystemExit(1) + + with open(values_file_path, 'w') as f: + f.write(updated_values) + + context.success(f'✅ Successfully updated base_os to {base_os} in values.yml') + return values_file_path + + +def _save_values_file_to_bucket( + context: SocaCliContext, cluster_name: str, aws_region: str, aws_profile: str +) -> None: + """ + save values.yml back to the cluster's s3 bucket after a successful upgrade, so the next + upgrade can restore it. a failed upload warns rather than failing the upgrade. + """ + try: + upload_values_file(cluster_name, aws_region, aws_profile, context) + except Exception as e: + profile = ( + f' --aws-profile {aws_profile}' if Utils.is_not_empty(aws_profile) else '' + ) + context.warning( + f'could not save values.yml to the cluster s3 bucket: {e}. The upgrade itself ' + f'succeeded; save the file with: idea-admin.sh config save-values ' + f'--cluster-name {cluster_name} --aws-region {aws_region}{profile}' + ) + + # Helper function to sync full configuration without overwrite def _sync_full_config_without_overwrite( cluster_name: str, diff --git a/source/idea/idea-administrator/src/ideaadministrator_meta/__init__.py b/source/idea/idea-administrator/src/ideaadministrator_meta/__init__.py index e49aa3e3..581887cb 100644 --- a/source/idea/idea-administrator/src/ideaadministrator_meta/__init__.py +++ b/source/idea/idea-administrator/src/ideaadministrator_meta/__init__.py @@ -12,4 +12,4 @@ # pkg config for idea-admin. no dependencies. __name__ = 'idea-administrator' -__version__ = '26.08.0' +__version__ = '26.09.0' diff --git a/source/idea/idea-administrator/tests/test_ami_update_entries.py b/source/idea/idea-administrator/tests/test_ami_update_entries.py index c8292942..1dadfe5d 100644 --- a/source/idea/idea-administrator/tests/test_ami_update_entries.py +++ b/source/idea/idea-administrator/tests/test_ami_update_entries.py @@ -11,7 +11,12 @@ import pytest -from ideaadministrator.app_main import build_ami_update_entries +from ideadatamodel import constants +from ideaadministrator.app_main import ( + build_ami_update_entries, + keep_built_compute_image, + resolve_compute_ami_keep_keys, +) AMI_ID = 'ami-0123456789abcdef0' @@ -28,14 +33,19 @@ 'vdc.dcv_connection_gateway.autoscaling': 'virtual-desktop-controller/settings.yml', } - -@pytest.fixture(scope='module') -def entries(): - return build_ami_update_entries(ami_id=AMI_ID, base_os=BASE_OS) +# what ClusterConfigDB.get_cluster_modules() returns for a cluster installed with default ids +MODULES = [ + {'module_id': 'bastion-host', 'name': constants.MODULE_BASTION_HOST}, + {'module_id': 'cluster-manager', 'name': constants.MODULE_CLUSTER_MANAGER}, + {'module_id': 'directoryservice', 'name': constants.MODULE_DIRECTORYSERVICE}, + {'module_id': 'scheduler', 'name': constants.MODULE_SCHEDULER}, + {'module_id': 'vdc', 'name': constants.MODULE_VIRTUAL_DESKTOP_CONTROLLER}, + # config-only modules template no AMI and must contribute nothing + {'module_id': 'cluster', 'name': constants.MODULE_CLUSTER}, +] -@pytest.fixture(scope='module') -def entries_by_key(entries): +def parse(entries): parsed = {} for entry in entries: match = re.fullmatch(r'Key=(.+?),Type=(.+?),Value=(.*)', entry) @@ -45,6 +55,16 @@ def entries_by_key(entries): return parsed +@pytest.fixture(scope='module') +def entries(): + return build_ami_update_entries(ami_id=AMI_ID, base_os=BASE_OS, modules=MODULES) + + +@pytest.fixture(scope='module') +def entries_by_key(entries): + return parse(entries) + + def test_every_entry_is_parseable_by_the_caller(entries, entries_by_key): """ upgrade_cluster splits each entry on ',' - the format must survive that @@ -73,6 +93,53 @@ def test_scheduler_compute_node_keys_are_updated(entries_by_key): assert entries_by_key['scheduler.compute_node_os'] == ('string', BASE_OS) +def test_keys_use_the_module_ids_the_cluster_actually_has(): + """ + config keys are prefixed with the module id, which the installer lets you choose; + a hardcoded id writes settings no module reads + """ + entries_by_key = parse( + build_ami_update_entries( + ami_id=AMI_ID, + base_os=BASE_OS, + modules=[ + { + 'module_id': 'vdc2', + 'name': constants.MODULE_VIRTUAL_DESKTOP_CONTROLLER, + }, + {'module_id': 'sched2', 'name': constants.MODULE_SCHEDULER}, + ], + ) + ) + + assert entries_by_key['sched2.base_os'] == ('string', BASE_OS) + assert entries_by_key['vdc2.controller.autoscaling.instance_ami'] == ( + 'string', + AMI_ID, + ) + assert not any(key.startswith('vdc.') for key in entries_by_key) + + +def test_a_module_the_cluster_does_not_deploy_gets_no_entries(): + """ + writing bastion-host keys on a cluster with no bastion host is settings nothing reads + """ + entries_by_key = parse( + build_ami_update_entries( + ami_id=AMI_ID, + base_os=BASE_OS, + modules=[{'module_id': 'scheduler', 'name': constants.MODULE_SCHEDULER}], + ) + ) + + assert sorted(entries_by_key) == [ + 'scheduler.base_os', + 'scheduler.compute_node_ami', + 'scheduler.compute_node_os', + 'scheduler.instance_ami', + ] + + def test_write_list_covers_every_templated_module(entries_by_key): """ guards against a new module templating base_os without being added here @@ -97,3 +164,159 @@ def test_write_list_covers_every_templated_module(entries_by_key): for prefix in MODULE_TEMPLATES: assert f'{prefix}.base_os' in entries_by_key + + +# an upgrade must not undo a compute image the operator built and adopted from Custom AMIs + +SCHEDULER_ONLY = [{'module_id': 'scheduler', 'name': constants.MODULE_SCHEDULER}] +COMPUTE_KEYS = {'scheduler.compute_node_ami', 'scheduler.compute_node_os'} + +BUILT_AMI = 'ami-0b17fd2c9a4e5d360' +STOCK_IMAGE = { + 'ImageId': AMI_ID, + 'Name': 'al2023-ami-2026.0.20260801.0-kernel-6.1-x86_64', + 'CreationDate': '2026-08-01T10:00:00.000Z', +} +NEWER_BUILT_IMAGE = { + 'ImageId': BUILT_AMI, + 'Name': 'idea-compute-node-amazonlinux2023-v09012026-120000', + 'CreationDate': '2026-09-01T12:00:00.000Z', +} +OLDER_BUILT_IMAGE = dict( + NEWER_BUILT_IMAGE, + Name='idea-compute-node-amazonlinux2023-v07012026-120000', + CreationDate='2026-07-01T12:00:00.000Z', +) +ADOPTED_STOCK_IMAGE = { + 'ImageId': BUILT_AMI, + 'Name': 'RHEL-9.6.0_HVM-20260501-x86_64-0-Hourly2-GP3', + 'CreationDate': '2026-09-01T12:00:00.000Z', +} + + +class FakeConfigDB: + """the one cluster settings read resolve_compute_ami_keep_keys makes""" + + def __init__(self, compute_node_ami): + self.compute_node_ami = compute_node_ami + + def get_config_entry(self, key): + if key != 'scheduler.compute_node_ami': + return None + return {'key': key, 'value': self.compute_node_ami} + + +class FakeEC2: + def __init__(self, images, error=None): + self.images = images + self.error = error + self.calls = [] + + def describe_images(self, **kwargs): + image_ids = kwargs['ImageIds'] + self.calls.append(list(image_ids)) + if self.error is not None: + raise self.error + return { + 'Images': [image for image in self.images if image['ImageId'] in image_ids] + } + + +class RecordingContext: + def __init__(self): + self.messages = [] + + def info(self, message): + self.messages.append(message) + + def warning(self, message): + self.messages.append(message) + + +def resolve(current_ami, images, error=None): + context = RecordingContext() + ec2 = FakeEC2(images, error) + keep = resolve_compute_ami_keep_keys( + context=context, + db=FakeConfigDB(current_ami), + modules=SCHEDULER_ONLY, + ami_id=AMI_ID, + ec2=ec2, + ) + return keep, context.messages, ec2 + + +def test_a_built_compute_image_newer_than_the_release_is_kept(): + """ + the operator built and adopted this image after the release; the upgrade would discard it + """ + keep, messages, ec2 = resolve(BUILT_AMI, [NEWER_BUILT_IMAGE, STOCK_IMAGE]) + + assert keep == COMPUTE_KEYS + assert ec2.calls == [[BUILT_AMI, AMI_ID]], 'both images describe in one call' + assert any('keeping built compute image' in message for message in messages) + + entries_by_key = parse( + build_ami_update_entries( + ami_id=AMI_ID, base_os=BASE_OS, modules=SCHEDULER_ONLY, keep_keys=keep + ) + ) + assert sorted(entries_by_key) == ['scheduler.base_os', 'scheduler.instance_ami'] + + +def test_a_built_compute_image_older_than_the_release_is_replaced(): + keep, messages, _ = resolve(BUILT_AMI, [OLDER_BUILT_IMAGE, STOCK_IMAGE]) + + assert keep == set() + assert any('Custom AMIs page' in message for message in messages) + + entries_by_key = parse( + build_ami_update_entries( + ami_id=AMI_ID, base_os=BASE_OS, modules=SCHEDULER_ONLY, keep_keys=keep + ) + ) + assert entries_by_key['scheduler.compute_node_ami'] == ('string', AMI_ID) + + +def test_a_stock_compute_image_is_replaced(): + """ + only a build is worth keeping; a stock image the operator pinned by hand is not recognised + """ + keep, messages, _ = resolve(BUILT_AMI, [ADOPTED_STOCK_IMAGE, STOCK_IMAGE]) + + assert keep == set() + assert messages == [] + + +def test_an_image_that_cannot_be_described_is_replaced(): + """ + a deregistered image fails the describe; the upgrade proceeds and says why + """ + keep, messages, _ = resolve( + BUILT_AMI, [STOCK_IMAGE], error=RuntimeError('InvalidAMIID.NotFound') + ) + + assert keep == set() + assert any('InvalidAMIID.NotFound' in message for message in messages) + + +def test_the_current_compute_image_is_not_described_when_it_is_already_the_release(): + keep, _, ec2 = resolve(AMI_ID, [STOCK_IMAGE]) + + assert keep == set() + assert ec2.calls == [] + + +def test_keep_built_compute_image_compares_creation_dates(): + assert keep_built_compute_image(NEWER_BUILT_IMAGE, STOCK_IMAGE) + assert not keep_built_compute_image(OLDER_BUILT_IMAGE, STOCK_IMAGE) + assert not keep_built_compute_image(ADOPTED_STOCK_IMAGE, STOCK_IMAGE) + assert not keep_built_compute_image(None, STOCK_IMAGE) + assert not keep_built_compute_image(NEWER_BUILT_IMAGE, None) + + same_date = dict(NEWER_BUILT_IMAGE, CreationDate=STOCK_IMAGE['CreationDate']) + assert not keep_built_compute_image(same_date, STOCK_IMAGE), 'equal is not newer' + + undated = dict(NEWER_BUILT_IMAGE) + undated.pop('CreationDate') + assert not keep_built_compute_image(undated, STOCK_IMAGE) diff --git a/source/idea/idea-administrator/tests/test_config_generator_empty_list.py b/source/idea/idea-administrator/tests/test_config_generator_empty_list.py new file mode 100644 index 00000000..e73bf93b --- /dev/null +++ b/source/idea/idea-administrator/tests/test_config_generator_empty_list.py @@ -0,0 +1,35 @@ +""" +Test Cases for config entry flattening of empty lists + +traverse_config() flattens the rendered yaml into the key/value entries that get synced +into the cluster settings table. it used to route every value through +Utils.get_any_value(), which reports an empty list as absent, so `system_7: []` was +written as None and stored as NULL. +""" + +from ideaadministrator.app.config_generator import ConfigGenerator + + +def flatten(config: dict) -> dict: + entries = [] + ConfigGenerator({}).traverse_config(entries, '', config) + return {entry['key']: entry['value'] for entry in entries} + + +def test_empty_list_survives_flattening(): + values = flatten({'global-settings': {'package_config': {'system_7': []}}}) + assert values['global-settings.package_config.system_7'] == [] + + +def test_null_and_blank_still_flatten_to_none(): + values = flatten({'m': {'not_set': None, 'blank': ' '}}) + assert values['m.not_set'] is None + assert values['m.blank'] is None + + +def test_other_values_unchanged(): + values = flatten({'m': {'ids': ['a', 'b'], 'count': 0, 'flag': False, 'name': 'x'}}) + assert values['m.ids'] == ['a', 'b'] + assert values['m.count'] == 0 + assert values['m.flag'] is False + assert values['m.name'] == 'x' diff --git a/source/idea/idea-administrator/tests/test_eol_base_os_preflight.py b/source/idea/idea-administrator/tests/test_eol_base_os_preflight.py index ebf08ec2..39925c71 100644 --- a/source/idea/idea-administrator/tests/test_eol_base_os_preflight.py +++ b/source/idea/idea-administrator/tests/test_eol_base_os_preflight.py @@ -31,6 +31,7 @@ class FakeTable: def __init__(self, items=None): self.items = items self.deleted = [] + self.updated = [] def scan(self, **_kwargs): if self.items is None: @@ -48,6 +49,9 @@ def scan(self, **_kwargs): def delete_item(self, Key): self.deleted.append(Key) + def update_item(self, Key, ExpressionAttributeValues, **_kwargs): + self.updated.append((Key, ExpressionAttributeValues)) + class FakeAws: def __init__(self, tables): @@ -77,12 +81,25 @@ def get_config_entries(self): monkeypatch.setattr(app_main, 'ClusterConfigDB', FakeDb) -def check(context): - app_main._check_eol_base_os( +def check(context, disable_stacks_in_use=False): + """the read-only preflight that runs before the upgrade is confirmed""" + return app_main._check_eol_base_os( + context=context, + cluster_name=CLUSTER_NAME, + aws_region='us-east-2', + aws_profile=None, + disable_stacks_in_use=disable_stacks_in_use, + ) + + +def apply_plans(context, plans): + """the mutations, which only run once the admin has confirmed""" + app_main._apply_eol_software_stacks( context=context, cluster_name=CLUSTER_NAME, aws_region='us-east-2', aws_profile=None, + plans=plans, ) @@ -95,9 +112,10 @@ def stack(stack_id, base_os=constants.OS_AMAZONLINUX2): } -def session(idea_session_id, owner, stack_id, state='READY'): +def session(idea_session_id, owner, stack_id, state='READY', name=None): return { 'owner': owner, + 'name': name or f'desktop {idea_session_id}', 'idea_session_id': idea_session_id, 'state': state, 'base_os': constants.OS_AMAZONLINUX2, @@ -113,12 +131,31 @@ def test_no_eol_software_stacks_is_a_no_op(monkeypatch): stub_db(monkeypatch, {SOFTWARE_STACKS_TABLE: stacks}, [VDC_MODULE]) context = FakeContext() - check(context) + assert check(context) == [] assert stacks.deleted == [] assert context.messages == [] +def test_the_preflight_reports_but_changes_nothing(monkeypatch): + # the upgrade is not confirmed yet at this point, so nothing may be written + stacks = FakeTable([stack('ss-al2'), stack('ss-al2-arm')]) + stub_db(monkeypatch, {SOFTWARE_STACKS_TABLE: stacks}, [VDC_MODULE]) + context = FakeContext() + + plans = check(context) + + assert stacks.deleted == [] + assert stacks.updated == [] + assert any( + 'will delete' in message and 'ss-al2' in message for message in context.messages + ) + assert [s['stack_id'] for plan in plans for s in plan['to_delete']] == [ + 'ss-al2', + 'ss-al2-arm', + ] + + def test_eol_software_stacks_without_a_live_session_are_deleted(monkeypatch): stacks = FakeTable([stack('ss-al2'), stack('ss-al2-arm')]) sessions = FakeTable( @@ -131,7 +168,7 @@ def test_eol_software_stacks_without_a_live_session_are_deleted(monkeypatch): ) context = FakeContext() - check(context) + apply_plans(context, check(context)) assert stacks.deleted == [ {'base_os': constants.OS_AMAZONLINUX2, 'stack_id': 'ss-al2'}, @@ -164,6 +201,82 @@ def test_a_stack_a_live_session_uses_aborts_and_deletes_nothing(monkeypatch): ) +def test_a_stack_in_use_is_disabled_with_the_flag_and_the_upgrade_continues( + monkeypatch, +): + stacks = FakeTable([stack('ss-al2'), stack('ss-al2-unused')]) + sessions = FakeTable([session('sess-1', 'user1', 'ss-al2', name='my desktop')]) + stub_db( + monkeypatch, + {SOFTWARE_STACKS_TABLE: stacks, USER_SESSIONS_TABLE: sessions}, + [VDC_MODULE], + ) + context = FakeContext() + + apply_plans(context, check(context, disable_stacks_in_use=True)) + + assert stacks.updated == [ + ( + {'base_os': constants.OS_AMAZONLINUX2, 'stack_id': 'ss-al2'}, + {':enabled': False}, + ) + ] + # an end-of-life stack no live session uses is still deleted + assert stacks.deleted == [ + {'base_os': constants.OS_AMAZONLINUX2, 'stack_id': 'ss-al2-unused'} + ] + assert any( + 'disabled' in message + and 'ss-al2' in message + and 'user1' in message + and 'my desktop' in message + for message in context.messages + ) + + +def test_disabling_a_stack_says_the_search_index_still_reads_enabled(monkeypatch): + # the row is disabled in DynamoDB but the portal lists stacks from the search index + stacks = FakeTable([stack('ss-al2')]) + sessions = FakeTable([session('sess-1', 'user1', 'ss-al2')]) + stub_db( + monkeypatch, + {SOFTWARE_STACKS_TABLE: stacks, USER_SESSIONS_TABLE: sessions}, + [VDC_MODULE], + ) + context = FakeContext() + + apply_plans(context, check(context, disable_stacks_in_use=True)) + + assert any( + 'ideactl reindex-software-stacks' in message + and 'virtual-desktop-controller' in message + for message in context.messages + ) + + +def test_a_session_without_a_stack_id_still_protects_its_stack(monkeypatch): + # matching only on the session's stack_id deletes a stack a running desktop is still on + orphan = session('sess-1', 'user1', 'ss-al2') + orphan['software_stack'] = {'base_os': constants.OS_AMAZONLINUX2} + stacks = FakeTable([stack('ss-al2')]) + stub_db( + monkeypatch, + {SOFTWARE_STACKS_TABLE: stacks, USER_SESSIONS_TABLE: FakeTable([orphan])}, + [VDC_MODULE], + ) + context = FakeContext() + + apply_plans(context, check(context, disable_stacks_in_use=True)) + + assert stacks.deleted == [] + assert stacks.updated == [ + ( + {'base_os': constants.OS_AMAZONLINUX2, 'stack_id': 'ss-al2'}, + {':enabled': False}, + ) + ] + + def test_an_eol_queue_profile_is_a_hard_stop(monkeypatch): # queue profiles are edited, not deleted, so nothing is done for the admin here queue_profiles = FakeTable( diff --git a/source/idea/idea-administrator/tests/test_upgrade_cluster_guards.py b/source/idea/idea-administrator/tests/test_upgrade_cluster_guards.py new file mode 100644 index 00000000..f99445a1 --- /dev/null +++ b/source/idea/idea-administrator/tests/test_upgrade_cluster_guards.py @@ -0,0 +1,168 @@ +import pytest +from click.testing import CliRunner + +from ideaadministrator import app_main +from ideaadministrator.app import deployment_helper + +CLUSTER_NAME = 'c' +AWS_REGION = 'us-east-2' + + +class FakeContext: + def __init__(self): + self.messages = [] + + def info(self, message): + self.messages.append(message) + + warning = info + error = info + success = info + + +def stub_db(monkeypatch, config_entries=(), error=None): + class FakeDb: + def __init__(self, **_kwargs): + if error is not None: + raise error + + def get_config_entries(self): + return config_entries + + monkeypatch.setattr(app_main, 'ClusterConfigDB', FakeDb) + + +def settings(*base_os_values): + return [ + {'key': f'module{index}.base_os', 'value': value} + for index, value in enumerate(base_os_values) + ] + + +def resolve(context, base_os): + return app_main.resolve_upgrade_base_os( + context=context, + cluster_name=CLUSTER_NAME, + aws_region=AWS_REGION, + aws_profile=None, + base_os=base_os, + ) + + +def test_without_base_os_the_cluster_keeps_the_one_it_runs(monkeypatch): + # an unattended --force run must not move every module onto the CLI default + stub_db(monkeypatch, settings('rhel9', 'rhel9')) + context = FakeContext() + + assert resolve(context, None) == 'rhel9' + assert any('rhel9' in message for message in context.messages) + + +def test_disagreeing_base_os_settings_refuse_rather_than_guess(monkeypatch): + stub_db(monkeypatch, settings('rhel9', 'rocky9')) + context = FakeContext() + + with pytest.raises(SystemExit): + resolve(context, None) + + assert any( + 'rhel9' in message and 'rocky9' in message and '--base-os' in message + for message in context.messages + ) + + +def test_unreadable_cluster_settings_refuse_rather_than_guess(monkeypatch): + stub_db(monkeypatch, error=RuntimeError('access denied')) + context = FakeContext() + + with pytest.raises(SystemExit): + resolve(context, None) + + assert any('--base-os' in message for message in context.messages) + + +def test_no_base_os_setting_at_all_refuses(monkeypatch): + stub_db(monkeypatch, [{'key': 'cluster.locale', 'value': 'en_US'}]) + context = FakeContext() + + with pytest.raises(SystemExit): + resolve(context, None) + + +def test_an_explicit_base_os_change_is_announced(monkeypatch): + stub_db(monkeypatch, settings('rhel9')) + context = FakeContext() + + assert resolve(context, 'amazonlinux2023') == 'amazonlinux2023' + assert any( + 'amazonlinux2023' in message and 'rhel9' in message + for message in context.messages + ) + + +def test_an_explicit_base_os_survives_unreadable_settings(monkeypatch): + stub_db(monkeypatch, error=RuntimeError('access denied')) + context = FakeContext() + + assert resolve(context, 'rhel9') == 'rhel9' + + +def test_the_upgrade_context_uses_the_requested_region_and_profile(monkeypatch): + # ClusterConfigDB takes the explicit region; the aws client provider must not be left on + # whatever region the ambient session happens to default to + captured = {} + + def fake_context(options=None): + captured['options'] = options + return FakeContext() + + monkeypatch.setattr(app_main, 'SocaCliContext', fake_context) + stub_db(monkeypatch, error=RuntimeError('stop here')) + + result = CliRunner().invoke( + app_main.upgrade_cluster, + [ + '--cluster-name', + CLUSTER_NAME, + '--aws-region', + AWS_REGION, + '--aws-profile', + 'my-profile', + ], + ) + + assert result.exit_code != 0 + assert captured['options'].aws_region == AWS_REGION + assert captured['options'].aws_profile == 'my-profile' + assert captured['options'].enable_aws_client_provider is True + + +def test_a_failed_parallel_module_fails_the_run(monkeypatch): + # CdkInvoker signals a failed cdk run with SystemExit, which dies with the thread; the + # status check cannot see it because the module is still 'deployed' from the last release + monkeypatch.setattr(deployment_helper.time, 'sleep', lambda *_args: None) + + helper = deployment_helper.DeploymentHelper.__new__( + deployment_helper.DeploymentHelper + ) + helper.optimize_deployment = True + helper.module_ids = ['scheduler', 'vdc'] + helper.get_optimized_deployment_order = lambda: [['scheduler', 'vdc']] + + deployed = [] + + def deploy_module(module_id): + if module_id == 'vdc': + raise SystemExit(1) + deployed.append(module_id) + + helper.deploy_module = deploy_module + helper.initialize_cluster_modules = lambda: pytest.fail( + 'a failed module must fail the run before the status check' + ) + + with pytest.raises(Exception) as raised: + helper.invoke() + + assert 'vdc' in str(raised.value) + assert deployed == ['scheduler'] diff --git a/source/idea/idea-administrator/tests/test_values_file_s3_fallback.py b/source/idea/idea-administrator/tests/test_values_file_s3_fallback.py new file mode 100644 index 00000000..6c015875 --- /dev/null +++ b/source/idea/idea-administrator/tests/test_values_file_s3_fallback.py @@ -0,0 +1,198 @@ +import botocore.exceptions +import pytest + +from ideaadministrator import app_main + +CLUSTER_NAME = 'c' +AWS_REGION = 'us-east-2' +BUCKET = f'{CLUSTER_NAME}-cluster-{AWS_REGION}-123456789012' +S3_URI = f's3://{BUCKET}/values/values.yml' +VALUES_YAML = 'cluster_name: c\nbase_os: amazonlinux2\n' + + +class FakeS3: + def __init__(self, body=None, upload_error=None): + self.body = body + self.upload_error = upload_error + self.uploaded = [] + + def get_object(self, Bucket, Key): + if self.body is None: + raise botocore.exceptions.ClientError( + {'Error': {'Code': 'NoSuchKey', 'Message': 'not found'}}, 'GetObject' + ) + return {'Body': self.body} + + def upload_file(self, Bucket, Filename, Key): + if self.upload_error is not None: + raise self.upload_error + self.uploaded.append((Bucket, Filename, Key)) + + +class FakeContext: + def __init__(self, s3): + self.messages = [] + self._s3 = s3 + + def info(self, message): + self.messages.append(message) + + warning = info + error = info + success = info + + def aws(self): + return self + + def s3(self): + return self._s3 + + +def stub_db(monkeypatch): + class FakeDb: + def __init__(self, **_kwargs): + pass + + def get_config_entry(self, key): + return {'key': key, 'value': BUCKET} + + monkeypatch.setattr(app_main, 'ClusterConfigDB', FakeDb) + + +def values_path(tmp_path): + return tmp_path / '.idea' / 'clusters' / CLUSTER_NAME / AWS_REGION / 'values.yml' + + +def write_local(tmp_path, text): + path = values_path(tmp_path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + return path + + +def setup(monkeypatch, tmp_path, s3): + # AdministratorProps resolves the values file under ~/.idea + monkeypatch.setenv('HOME', str(tmp_path)) + stub_db(monkeypatch) + return FakeContext(s3) + + +def update_base_os(context, base_os='rhel9'): + return app_main._update_values_base_os( + context=context, + cluster_name=CLUSTER_NAME, + aws_region=AWS_REGION, + aws_profile=None, + base_os=base_os, + ) + + +def save_values(context, aws_profile=None): + app_main._save_values_file_to_bucket( + context=context, + cluster_name=CLUSTER_NAME, + aws_region=AWS_REGION, + aws_profile=aws_profile, + ) + + +def test_a_missing_values_file_is_restored_from_the_bucket_and_updated( + monkeypatch, tmp_path +): + context = setup(monkeypatch, tmp_path, FakeS3(body=VALUES_YAML)) + + update_base_os(context) + + written = values_path(tmp_path).read_text() + assert 'base_os: rhel9' in written + # the rest of the restored file is kept, not just the base_os line + assert 'cluster_name: c' in written + assert any(S3_URI in message for message in context.messages) + + +def test_a_local_values_file_is_not_overwritten_from_the_bucket(monkeypatch, tmp_path): + context = setup(monkeypatch, tmp_path, FakeS3(body=VALUES_YAML)) + path = write_local(tmp_path, 'cluster_name: local\nbase_os: amazonlinux2\n') + + update_base_os(context) + + assert path.read_text() == 'cluster_name: local\nbase_os: rhel9\n' + + +def test_a_local_copy_that_differs_from_the_bucket_is_reported(monkeypatch, tmp_path): + # the local file wins, so the admin has to be told the bucket holds something else + context = setup(monkeypatch, tmp_path, FakeS3(body=VALUES_YAML)) + write_local(tmp_path, 'cluster_name: local\nbase_os: amazonlinux2\n') + + update_base_os(context) + + assert any( + 'differ' in message + and 'cluster_name' in message + and S3_URI in message + and 'local copy' in message + for message in context.messages + ) + + +def test_a_values_file_in_neither_place_names_both_locations(monkeypatch, tmp_path): + context = setup(monkeypatch, tmp_path, FakeS3()) + + with pytest.raises(SystemExit): + update_base_os(context) + + errors = [m for m in context.messages if 'Values file not found' in m] + assert len(errors) == 1 + assert str(values_path(tmp_path)) in errors[0] + assert S3_URI in errors[0] + assert 'config save-values' in errors[0] + assert not values_path(tmp_path).exists() + + +def test_a_values_file_without_a_base_os_key_fails_instead_of_claiming_success( + monkeypatch, tmp_path +): + # a values.yml old enough to predate the key would otherwise be reported as updated + context = setup(monkeypatch, tmp_path, FakeS3()) + path = write_local(tmp_path, 'cluster_name: c\n') + + with pytest.raises(SystemExit): + update_base_os(context) + + assert path.read_text() == 'cluster_name: c\n' + assert any( + 'no base_os key' in message and 'rhel9' in message + for message in context.messages + ) + assert not any('Successfully updated' in message for message in context.messages) + + +def test_values_file_is_saved_to_the_bucket_after_a_successful_upgrade( + monkeypatch, tmp_path +): + s3 = FakeS3() + context = setup(monkeypatch, tmp_path, s3) + path = write_local(tmp_path, VALUES_YAML) + + save_values(context) + + assert s3.uploaded == [(BUCKET, str(path), 'values/values.yml')] + + +def test_a_failed_save_to_the_bucket_warns_with_the_command_to_run( + monkeypatch, tmp_path +): + # credentials that expire during a long upgrade break this upload first + upload_error = botocore.exceptions.ClientError( + {'Error': {'Code': 'ExpiredToken', 'Message': 'expired'}}, 'PutObject' + ) + context = setup(monkeypatch, tmp_path, FakeS3(upload_error=upload_error)) + + save_values(context, aws_profile='my-profile') + + assert any( + f'config save-values --cluster-name {CLUSTER_NAME} --aws-region {AWS_REGION} ' + f'--aws-profile my-profile' in message + for message in context.messages + ) + assert any('upgrade itself' in message for message in context.messages) diff --git a/source/idea/idea-bootstrap/_templates/linux/openpbs.jinja2 b/source/idea/idea-bootstrap/_templates/linux/openpbs.jinja2 index a2dfa522..26766f97 100644 --- a/source/idea/idea-bootstrap/_templates/linux/openpbs.jinja2 +++ b/source/idea/idea-bootstrap/_templates/linux/openpbs.jinja2 @@ -2,7 +2,7 @@ {% if context.base_os in ('amazonlinux2023', 'rhel8', 'rhel9', 'rhel10', 'rocky8', 'rocky9', 'rocky10', 'ubuntu2204', 'ubuntu2404', 'ubuntu2604') %} function install_openpbs_dependencies() { - OPENPBS_PKGS="{{ ' '.join(context.config.get_list('global-settings.package_config.openpbs.packages')) }}" + OPENPBS_PKGS="{{ ' '.join(context.config.get_list('global-settings.package_config.openpbs.packages', default=[])) }}" {% if context.base_os in ('amazonlinux2023', 'rhel8', 'rhel9', 'rhel10', 'rocky8', 'rocky9', 'rocky10') %} yum install -y --skip-broken $(echo ${OPENPBS_PKGS[*]}) {% endif %} diff --git a/source/idea/idea-bootstrap/_templates/linux/system_packages.jinja2 b/source/idea/idea-bootstrap/_templates/linux/system_packages.jinja2 index 45d5f43f..83158a61 100644 --- a/source/idea/idea-bootstrap/_templates/linux/system_packages.jinja2 +++ b/source/idea/idea-bootstrap/_templates/linux/system_packages.jinja2 @@ -1,25 +1,29 @@ # Begin: System Packages Install +{# the optional, version-suffixed lists below read with default=[] rather than + required=True. required=True does not guard a NULL row: SocaConfig.get_list() + returns None for one and only raises for a genuinely absent key, so ' '.join(None) + would kill bootstrap at instance boot. the base lists keep required=True, where a + missing key means a broken config that should fail loudly instead of silently + installing nothing. #} SYSTEM_PKGS=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.system', required=True)) }}) SYSTEM_PKGS_DEB=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.system_deb', required=True)) }}) SYSTEM_PKGS_7=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.system_7', default=[])) }}) -SYSTEM_PKGS_8=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.system_8', required=True)) }}) -SYSTEM_PKGS_9=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.system_9', required=True)) }}) -{# default=[] not required=True: a stored config without system_10 must not fail - bootstrap on non-EL10 nodes #} +SYSTEM_PKGS_8=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.system_8', default=[])) }}) +SYSTEM_PKGS_9=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.system_9', default=[])) }}) SYSTEM_PKGS_10=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.system_10', default=[])) }}) APPLICATION_PKGS=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.application', required=True)) }}) APPLICATION_PKGS_DEB=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.application_deb', required=True)) }}) -APPLICATION_PKGS_7=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.application_7', required=True)) }}) -APPLICATION_PKGS_8=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.application_8', required=True)) }}) +APPLICATION_PKGS_7=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.application_7', default=[])) }}) +APPLICATION_PKGS_8=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.application_8', default=[])) }}) SSSD_PKGS=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.sssd', required=True)) }}) SSSD_PKGS_DEB=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.sssd_deb', required=True)) }}) -SSSD_PKGS_7=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.sssd_7', required=True)) }}) +SSSD_PKGS_7=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.sssd_7', default=[])) }}) OPENLDAP_CLIENT_PKGS=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.openldap_client', required=True)) }}) OPENLDAP_CLIENT_PKGS_DEB=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.openldap_client_deb', required=True)) }}) diff --git a/source/idea/idea-bootstrap/_templates/linux/system_upgrade.jinja2 b/source/idea/idea-bootstrap/_templates/linux/system_upgrade.jinja2 index 320c59b0..ab4839aa 100644 --- a/source/idea/idea-bootstrap/_templates/linux/system_upgrade.jinja2 +++ b/source/idea/idea-bootstrap/_templates/linux/system_upgrade.jinja2 @@ -1,7 +1,7 @@ {% if context.base_os in ('ubuntu2204', 'ubuntu2404', 'ubuntu2604') %} # System upgrade with error handling to prevent infinite loops -if apt update; then - if apt upgrade -y; then +if DEBIAN_FRONTEND=noninteractive apt update; then + if DEBIAN_FRONTEND=noninteractive apt upgrade -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold; then log_info "System packages upgraded successfully" # Only set reboot required if upgrade actually succeeded set_reboot_required "Packages Upgraded" @@ -11,4 +11,16 @@ if apt update; then else log_error "apt update failed, but continuing bootstrap process" fi +{# first_boot_dnf_update is a desktop-only flag. this template also renders for + compute nodes, where no vdc key is in scope, so the read carries an explicit + default=False rather than depending on a vdc key being present. #} +{% elif context.base_os in ('amazonlinux2023', 'rhel8', 'rhel9', 'rocky8', 'rocky9') and context.config.get_bool('virtual-desktop-controller.dcv_session.first_boot_dnf_update', default=False) %} +# System upgrade with error handling to prevent infinite loops +if dnf -y update; then + log_info "System packages upgraded successfully" + # Only set reboot required if upgrade actually succeeded + set_reboot_required "Packages Upgraded" +else + log_error "dnf update failed, but continuing bootstrap process" +fi {% endif %} diff --git a/source/idea/idea-bootstrap/dcv-host-ami-builder/dcv_host_ami_builder.sh.jinja2 b/source/idea/idea-bootstrap/dcv-host-ami-builder/dcv_host_ami_builder.sh.jinja2 new file mode 100644 index 00000000..915a5fbf --- /dev/null +++ b/source/idea/idea-bootstrap/dcv-host-ami-builder/dcv_host_ami_builder.sh.jinja2 @@ -0,0 +1,94 @@ +#!/bin/bash +set -x + +source /etc/environment +if [[ -f /etc/profile.d/proxy.sh ]]; then + source /etc/profile.d/proxy.sh +fi + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/../common/bootstrap_common.sh" + +{% set TAGS = [ + {'Key':'idea:ClusterName', 'Value': context.cluster_name }, + {'Key':'idea:ModuleName', 'Value': context.module_name }, + {'Key':'idea:ModuleId', 'Value': context.module_id }, + {'Key':'Name', 'Value': context.vars.ami_name } +] %} + +{% with ebs_volume_tags = TAGS %} + {% include '_templates/linux/tag_ebs_volumes.jinja2' %} +{% endwith %} + +{% with network_interface_tags = TAGS %} + {% include '_templates/linux/tag_network_interface.jinja2' %} +{% endwith %} + +{% include '_templates/linux/disable_auto_upgrades.jinja2' %} + +{% include '_templates/linux/disable_firewall.jinja2' %} + +{% include '_templates/linux/systemd_resolved.jinja2' %} + +{% include '_templates/linux/set_kernel.jinja2' %} + +if [[ ! -f ${BOOTSTRAP_DIR}/idea_preinstalled_packages.log ]]; then + {% if context.base_os in ('rhel8', 'rocky8') %} + #drpm isn't available in rhel9 or rocky9 + yum install -y drpm + {% endif %} + {% include '_templates/linux/universe.jinja2' %} + {% include '_templates/linux/system_packages.jinja2' %} + {% include '_templates/linux/jq.jinja2' %} + {% include '_templates/linux/disable_se_linux.jinja2' %} + echo "Package installation completed at $(date)" >> ${BOOTSTRAP_DIR}/idea_preinstalled_packages.log +else + log_info "Found ${BOOTSTRAP_DIR}/idea_preinstalled_packages.log... skipping package installation..." +fi + +{% include '_templates/linux/cloudwatch_agent.jinja2' %} + +{% include '_templates/linux/chronyd.jinja2' %} + +{% include '_templates/linux/snapd.jinja2' %} + +{% include '_templates/linux/disable_ulimit.jinja2' %} + +{% include '_templates/linux/disable_strict_host_check.jinja2' %} + +{% include '_templates/linux/disable_motd_update.jinja2' %} + +{% with secure_path = '/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin' %} + {% include '_templates/linux/sudoer_secure_path.jinja2' %} +{% endwith %} + +{% with messages = [ + 'DCV Host AMI, AMI Name: ' + context.vars.ami_name + ', ModuleId: ' + context.module_id + ', Cluster: ' + context.cluster_name +] %} + {% include '_templates/linux/motd.jinja2' %} +{% endwith %} + +{% if context.is_gpu_instance_type() %} + {% include '_templates/linux/disable_nouveau_drivers.jinja2' %} +{% else %} + log_info "GPU InstanceType not detected. Skipping disabling of Nouveau Drivers..." +{% endif %} + +if [[ ! -f ${BOOTSTRAP_DIR}/idea_system_upgraded.log ]]; then + {% include '_templates/linux/system_upgrade.jinja2' %} + echo "System upgrade completed at $(date)" >> ${BOOTSTRAP_DIR}/idea_system_upgraded.log +else + log_info "Found ${BOOTSTRAP_DIR}/idea_system_upgraded.log... skipping system upgrade..." +fi + +REBOOT_REQUIRED=$(cat /root/bootstrap/reboot_required.txt) +if [[ "${REBOOT_REQUIRED}" == "yes" ]]; then + log_info "reboot required. dcv_host_ami_builder_post_reboot.sh will be executed after reboot ..." + (crontab -l; echo "@reboot /bin/bash ${SCRIPT_DIR}/dcv_host_ami_builder_post_reboot.sh >> ${IDEA_DCV_HOST_AMI_BUILDER_LOGS_DIR}/dcv_host_ami_builder_bootstrap.log 2>&1") | crontab - + reboot + exit 0 +else + mount -a + log_info "reboot not required. executing dcv_host_ami_builder_post_reboot.sh ..." + /bin/bash ${SCRIPT_DIR}/dcv_host_ami_builder_post_reboot.sh >> "${IDEA_DCV_HOST_AMI_BUILDER_LOGS_DIR}/dcv_host_ami_builder_bootstrap.log" 2>&1 +fi diff --git a/source/idea/idea-bootstrap/dcv-host-ami-builder/dcv_host_ami_builder_post_reboot.sh.jinja2 b/source/idea/idea-bootstrap/dcv-host-ami-builder/dcv_host_ami_builder_post_reboot.sh.jinja2 new file mode 100644 index 00000000..4d9441a4 --- /dev/null +++ b/source/idea/idea-bootstrap/dcv-host-ami-builder/dcv_host_ami_builder_post_reboot.sh.jinja2 @@ -0,0 +1,101 @@ +#!/bin/bash +set -x + +source /etc/environment +if [[ -f /etc/profile.d/proxy.sh ]]; then + source /etc/profile.d/proxy.sh +fi + +# reset reboot_required.txt +echo -n "no" > ${BOOTSTRAP_DIR}/reboot_required.txt + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/../common/bootstrap_common.sh" + +## -- DCV SERVER AND AGENT INSTALLATION -- ## + +{% include '_templates/linux/dcv_server.jinja2' %} + +{% include '_templates/linux/dcv_session_manager_agent.jinja2' %} + +{% if context.base_os == 'amazonlinux2023' %} +{% include '_templates/linux/dcv_server_cert.jinja2' %} +{% endif %} + +# usb support: same guarded install the session bootstrap performs +if [[ -z "$(lsmod | grep eveusb)" ]]; then + echo "Installing usb support..." + DCV_USB_DRIVER_INSTALLER=$(which dcvusbdriverinstaller) + $DCV_USB_DRIVER_INSTALLER --quiet +else + log_info "Found eveusb kernel module pre-installed... skipping installation..." +fi + +# microphone redirect: same guarded install the session bootstrap performs +{% if context.base_os in ('amazonlinux2023', 'rhel8', 'rhel9', 'rocky8', 'rocky9') %} +if [[ -z "$(rpm -qa pulseaudio-utils)" ]]; then + yum install -y pulseaudio-utils +{% elif context.base_os in ('ubuntu2204', 'ubuntu2404') %} +if [[ -z "$(dpkg -l pulseaudio-utils | grep ^ii)" ]]; then + apt install -y pulseaudio-utils +{% endif %} +else + log_info "Found pulseaudio-utils pre-installed... skipping installation..." +fi + +# a reboot may be required if GPU drivers were installed +REBOOT_REQUIRED=$(cat /root/bootstrap/reboot_required.txt) +if [[ "${REBOOT_REQUIRED}" == "yes" ]]; then + reboot + exit 0 +else + # user data customizations + if [[ -f ${IDEA_CLUSTER_HOME}/${IDEA_MODULE_ID}/ami_builder/userdata_customizations.sh ]]; then + /bin/bash ${IDEA_CLUSTER_HOME}/${IDEA_MODULE_ID}/ami_builder/userdata_customizations.sh >> ${IDEA_DCV_HOST_AMI_BUILDER_LOGS_DIR}/userdata_customizations.log 2>&1 + fi + + # services are installed but must not start before the session bootstrap configures them + systemctl disable dcvserver 2>/dev/null || true + systemctl disable dcv-session-manager-agent 2>/dev/null || true + rm -f /etc/dcv-session-manager-agent/tags/*.toml + + # clean-up: shared storage mounts + {% for name, storage in context.config.get_config('shared-storage').items() %} + {% if context.eval_shared_storage_scope(shared_storage=storage) %} + remove_from_fstab "{{ storage.mount_dir }}" + {% endif %} + {% endfor %} + + # clean-up: instance-unique state. the idea_preinstalled_packages.log and + # idea_system_upgraded.log markers are kept on purpose: the session bootstrap + # skips the steps they record. + rm -rf /var/tmp/* /tmp/* /var/crash/* + rm -rf /etc/ssh/ssh_host_* + rm -f /etc/udev/rules.d/70-persistent-net.rules + grep -l "Created by cloud-init on instance boot automatically" /etc/sysconfig/network-scripts/ifcfg-* 2>/dev/null | xargs -r rm -f + rm -rf /root/bootstrap/logs + rm -f /root/bootstrap/reboot_required.txt /root/bootstrap/reboot_history.log + + # clean-up: identity that must be unique per desktop. the ssm agent registration, the + # machine ids, cloud-init's per-instance state and the dcv server key are regenerated + # on first boot; left in place, every desktop cloned from this image would answer to + # the builder's ssm identity and send-command would land on the wrong host + rm -rf /var/lib/amazon/ssm/* + : > /etc/machine-id + rm -f /var/lib/dbus/machine-id + rm -rf /var/lib/cloud/instances/* /var/lib/cloud/instance /var/lib/cloud/data/* + find /var/log -type f -exec truncate -s 0 {} + 2>/dev/null || true + rm -f /etc/dcv/dcv.key /etc/dcv/dcv.pem + + # clear crontab to ensure new image starts from clean slate + crontab -l | grep -v 'dcv_host_ami_builder_post_reboot.sh' | crontab - + + # flush filesystem buffers before signaling ami creation + /bin/sync + + # create tag indicating ec2 instance is ready for creating AMI + aws ec2 create-tags \ + --resources "${AWS_INSTANCE_ID}" \ + --region "${AWS_REGION}" \ + --tags Key=idea:AmiBuilderStatus,Value=complete +fi diff --git a/source/idea/idea-bootstrap/dcv-host-ami-builder/setup.sh.jinja2 b/source/idea/idea-bootstrap/dcv-host-ami-builder/setup.sh.jinja2 new file mode 100644 index 00000000..586006a1 --- /dev/null +++ b/source/idea/idea-bootstrap/dcv-host-ami-builder/setup.sh.jinja2 @@ -0,0 +1,67 @@ +#!/bin/bash +set -x + +{% set path = '/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin' %} + +echo -n "no" > ${BOOTSTRAP_DIR}/reboot_required.txt +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/../common/bootstrap_common.sh" + +echo -e " +## [BEGIN] IDEA Environment Configuration - Do Not Delete +AWS_DEFAULT_REGION={{ context.aws_region }} +AWS_REGION={{ context.aws_region }} +AWS_INSTANCE_ID=$(instance_id) +IDEA_BASE_OS={{ context.base_os }} +IDEA_MODULE_NAME={{ context.module_name }} +IDEA_MODULE_ID={{ context.module_id }} +IDEA_MODULE_SET={{ context.module_set }} +IDEA_MODULE_VERSION={{ context.module_version }} +IDEA_CLUSTER_S3_BUCKET={{ context.cluster_s3_bucket }} +IDEA_CLUSTER_NAME={{ context.cluster_name }} +IDEA_CLUSTER_HOME={{ context.cluster_home_dir }} +IDEA_DCV_HOST_AMI_BUILDER_LOGS_DIR={{ context.vars.ami_dir }}/logs/$(hostname -s) +IDEA_AMI_NAME={{ context.vars.ami_name }} +BOOTSTRAP_DIR=/root/bootstrap" > /etc/environment +{% if context.https_proxy != '' %} +echo -e "IDEA_HTTPS_PROXY={{ context.https_proxy }} +IDEA_NO_PROXY={{ context.no_proxy }}" >> /etc/environment +{% endif %} +echo -e "## [END] IDEA Environment Configuration + +PATH={{ path }} +" >> /etc/environment + +source /etc/environment + +{% if context.base_os == 'amazonlinux2023' %} +dnf -y install cronie +systemctl enable crond +systemctl start crond +{% endif %} + +SOURCE="${1}" +if [[ "${SOURCE}" == "crontab" ]]; then + # clean crontab, remove current file from reboot commands + crontab -l | grep -v 'setup.sh' | crontab - +fi + +echo -n "no" > ${BOOTSTRAP_DIR}/reboot_required.txt + +{% include '_templates/linux/idea_proxy.jinja2' %} + +{% include '_templates/linux/rsyslog.jinja2' %} + +{% include '_templates/linux/idea_service_account.jinja2' %} + +{% include '_templates/linux/aws_ssm.jinja2' %} + +{% include '_templates/linux/epel_repo.jinja2' %} + +{% include '_templates/linux/nfs_utils.jinja2' %} + +{% include '_templates/linux/mount_shared_storage.jinja2' %} + +mkdir -p ${IDEA_DCV_HOST_AMI_BUILDER_LOGS_DIR} +log_info "executing dcv_host_ami_builder.sh ..." +/bin/bash ${SCRIPT_DIR}/dcv_host_ami_builder.sh >> "${IDEA_DCV_HOST_AMI_BUILDER_LOGS_DIR}/dcv_host_ami_builder_bootstrap.log" 2>&1 diff --git a/source/idea/idea-bootstrap/openldap-server/_templates/install_openldap.jinja2 b/source/idea/idea-bootstrap/openldap-server/_templates/install_openldap.jinja2 index 52090b4b..cb1e2804 100644 --- a/source/idea/idea-bootstrap/openldap-server/_templates/install_openldap.jinja2 +++ b/source/idea/idea-bootstrap/openldap-server/_templates/install_openldap.jinja2 @@ -15,8 +15,8 @@ DS_TLS_PRIVATE_KEY=$(get_secret "${LDAP_TLS_PRIVATE_KEY_SECRET_ARN}") log_info "Installing LDAP packages ..." OPENLDAP_SERVER_PKGS=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.openldap_server', required=True)) }}) OPENLDAP_SERVER_PKGS_7=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.openldap_server_7', default=[])) }}) -OPENLDAP_SERVER_PKGS_8=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.openldap_server_8', required=True)) }}) -OPENLDAP_SERVER_PKGS_9=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.openldap_server_9', required=True)) }}) +OPENLDAP_SERVER_PKGS_8=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.openldap_server_8', default=[])) }}) +OPENLDAP_SERVER_PKGS_9=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.openldap_server_9', default=[])) }}) {# default=[] not required=True: a stored config without openldap_server_10 must not fail the render for every base_os #} OPENLDAP_SERVER_PKGS_10=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.openldap_server_10', default=[])) }}) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/api_invoker.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/api_invoker.py index 2b216d6d..3e5bfec3 100644 --- a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/api_invoker.py +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/api_invoker.py @@ -37,6 +37,8 @@ from ideaclustermanager.app.api.accounts_api import AccountsAPI from ideaclustermanager.app.api.auth_api import AuthAPI from ideaclustermanager.app.api.email_templates_api import EmailTemplatesAPI +from ideaclustermanager.app.api.my_costs_api import MyCostsAPI +from ideaclustermanager.app.api.costs_api import CostsAPI from typing import Optional, Dict @@ -52,6 +54,8 @@ def __init__(self, context: ideaclustermanager.AppContext): self.auth_api = AuthAPI(context) self.accounts_api = AccountsAPI(context) self.email_templates_api = EmailTemplatesAPI(context) + self.my_costs_api = MyCostsAPI(context) + self.costs_api = CostsAPI(context) self.max_listings_for_logging = 10 self.auto_truncate_responses = { 'Accounts.ListUsers': ListUsersResult, @@ -203,3 +207,7 @@ def invoke(self, context: ApiInvocationContext): self.accounts_api.invoke(context) elif namespace.startswith('EmailTemplates.'): self.email_templates_api.invoke(context) + elif namespace.startswith('MyCosts.'): + self.my_costs_api.invoke(context) + elif namespace.startswith('Costs.'): + self.costs_api.invoke(context) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/cluster_settings_api.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/cluster_settings_api.py index e9e00942..06cf8f9a 100644 --- a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/cluster_settings_api.py +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/cluster_settings_api.py @@ -53,6 +53,10 @@ 'web_portal.custom_dashboard.enabled', 'web_portal.custom_dashboard.title', 'web_portal.custom_dashboard.url', + # maintenance banner. every user sees it, so every user has to be able to read it. + 'maintenance.enabled', + 'maintenance.message', + 'maintenance.ends_at', ], constants.MODULE_DIRECTORYSERVICE: [ 'provider', @@ -204,13 +208,20 @@ def get_allowed_settings_for_module(self, module_id: str) -> List[str]: 'dcv_session.schedule.sunday.shut_down_time', ], 'scheduler': [ - # Add scheduler settings that should be editable here + # the default image compute nodes launch from, written from the + # custom AMIs page. a flat key, so no dotted path here. + 'compute_node_ami', ], 'cluster-manager': [ # feature flag and the org-approved model catalog. edited from the # bedrock tab on the cluster settings page. 'bedrock.enabled', 'bedrock.model_ids', + # maintenance banner, edited from the maintenance tab so a window + # can be opened and closed without a redeploy. + 'maintenance.enabled', + 'maintenance.message', + 'maintenance.ends_at', ], } diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/costs_api.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/costs_api.py new file mode 100644 index 00000000..0e50487f --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/costs_api.py @@ -0,0 +1,54 @@ +import ideaclustermanager + +from ideasdk.api import ApiInvocationContext, BaseAPI +from ideadatamodel import exceptions, GetUserCostsSummaryRequest +from ideasdk.utils import Utils + +from ideaclustermanager.app.costs.my_costs_service import MyCostsService + + +class CostsAPI(BaseAPI): + """ + the same measurements MyCosts serves, for every user, to administrators only. kept a + separate class from MyCostsAPI so that a self scoped page cannot quietly become an + open one through a single invoke() holding both authorization rules. + """ + + def __init__(self, context: ideaclustermanager.AppContext): + self.context = context + self.my_costs = MyCostsService(context) + + self.SCOPE_READ = f'{self.context.module_id()}/read' + + self.acl = { + # an application token holding cluster-manager/read may list this, + # which is what reporting integrations use. + 'Costs.ListUserCosts': { + 'scope': self.SCOPE_READ, + 'method': self.list_user_costs, + }, + 'Costs.GetUserSummary': { + 'scope': self.SCOPE_READ, + 'method': self.get_user_summary, + }, + } + + def list_user_costs(self, context: ApiInvocationContext): + context.success(self.my_costs.list_user_costs()) + + def get_user_summary(self, context: ApiInvocationContext): + request = context.get_request_payload_as(GetUserCostsSummaryRequest) + if Utils.is_empty(request.username): + raise exceptions.invalid_params('username is required') + context.success(self.my_costs.get_summary(username=request.username)) + + def invoke(self, context: ApiInvocationContext): + acl_entry = Utils.get_value_as_dict(context.namespace, self.acl) + if acl_entry is None: + raise exceptions.unauthorized_access() + + acl_entry_scope = Utils.get_value_as_string('scope', acl_entry) + if context.is_authorized(elevated_access=True, scopes=[acl_entry_scope]): + acl_entry['method'](context) + else: + raise exceptions.unauthorized_access() diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/my_costs_api.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/my_costs_api.py new file mode 100644 index 00000000..b99e6630 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/my_costs_api.py @@ -0,0 +1,32 @@ +import ideaclustermanager + +from ideasdk.api import ApiInvocationContext, BaseAPI +from ideadatamodel import exceptions + +from ideaclustermanager.app.costs.my_costs_service import MyCostsService + + +class MyCostsAPI(BaseAPI): + """ + what the cluster recorded against the caller. every method is scoped to the token's + own username and the request model carries no username field, so no shape of this + request asks about somebody else. an administrator uses the admin pages instead. + """ + + def __init__(self, context: ideaclustermanager.AppContext): + self.context = context + self.my_costs = MyCostsService(context) + + def get_summary(self, context: ApiInvocationContext): + context.success(self.my_costs.get_summary(username=context.get_username())) + + def invoke(self, context: ApiInvocationContext): + # authorized, not merely authenticated: removing a user from the module + # group has to take the page away with them. + if not context.is_authorized_user(): + raise exceptions.unauthorized_access() + + if context.namespace == 'MyCosts.GetSummary': + self.get_summary(context) + else: + raise exceptions.unauthorized_access() diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/projects_api.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/projects_api.py index 7f66f5f8..f27623d2 100644 --- a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/projects_api.py +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/projects_api.py @@ -17,6 +17,7 @@ GetProjectRequest, UpdateProjectRequest, ListProjectsRequest, + ListBedrockUsageRequest, EnableProjectRequest, DisableProjectRequest, GetUserProjectsRequest, @@ -56,6 +57,10 @@ def __init__(self, context: ideaclustermanager.AppContext): 'scope': self.SCOPE_READ, 'method': self.list_projects, }, + 'Projects.ListBedrockUsage': { + 'scope': self.SCOPE_READ, + 'method': self.list_bedrock_usage, + }, 'Projects.GetUserProjects': { 'scope': self.SCOPE_READ, 'method': self.admin_get_user_projects, @@ -107,8 +112,9 @@ def apply_budget_actuals(self, project: Project): def apply_bedrock_usage(self, project: Project, username: str = None): """ - attach month to date bedrock usage, and the spend cost explorer reports for - the project tag, which trails the recorded usage by about a day. + attach trailing window bedrock usage, and the spend cost explorer reports for + the project tag over the same window, which trails the recorded usage by about + a day. """ if not project.is_bedrock_enabled(): return @@ -152,14 +158,17 @@ def apply_bedrock_budget(self, project: Project): def strip_bedrock_provisioner_fields(project: Project): """ the iam role and instance profile are server side selection inputs with no - client use, and policy errors name administrator policies. the inference - profile arns stay: they are the identifier a user passes to invoke a model. + client use, and policy and reconcile errors name administrator policies and + iam calls. the inference profile arns stay: they are the identifier a user + passes to invoke a model. """ if project is None or project.bedrock is None: return project.bedrock.role_arn = None project.bedrock.instance_profile_arn = None project.bedrock.policy_errors = None + project.bedrock.reconcile_error = None + project.bedrock.reconcile_error_on = None def get_project(self, context: ApiInvocationContext): request = context.get_request_payload_as(GetProjectRequest) @@ -171,6 +180,9 @@ def get_project(self, context: ApiInvocationContext): def get_member_project(self, context: ApiInvocationContext): """ + the project's budget figures reach its members here: a member held back by + an exhausted budget has to be able to see why. + non-elevated variant of get_project: serves the project only when the caller is a member, using the same membership resolution as Projects.GetUserProjects. non-membership and non-existence are reported @@ -213,6 +225,15 @@ def list_projects(self, context: ApiInvocationContext): self.apply_bedrock_budget(project) context.success(result) + def list_bedrock_usage(self, context: ApiInvocationContext): + """ + every bedrock project with its window usage. elevated callers only: invoke() + has no non-elevated route to this namespace, so the per user breakdown never + reaches a member. + """ + request = context.get_request_payload_as(ListBedrockUsageRequest) + context.success(self.context.projects.list_bedrock_usage(request)) + def get_user_projects(self, context: ApiInvocationContext): request = context.get_request_payload_as(GetUserProjectsRequest) request.username = context.get_username() diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/costs/__init__.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/costs/__init__.py new file mode 100644 index 00000000..3ebc345d --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/costs/__init__.py @@ -0,0 +1 @@ +from ideaclustermanager.app.costs.my_costs_service import MyCostsService diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/costs/my_costs_service.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/costs/my_costs_service.py new file mode 100644 index 00000000..bc68fde7 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/costs/my_costs_service.py @@ -0,0 +1,926 @@ +""" +what one user cost the cluster over the trailing usage window, from what IDEA itself +recorded: attributed bedrock tokens, scheduler job cost estimates and indexed desktop +uptime. the only billed figure read here is the project's bedrock spend, by cost +allocation tag, and it is only ever used to apportion, never shown whole. +""" + +__all__ = ( + 'MyCostsService', + 'LIVE_SESSION_STATES', + 'RECENT_JOBS', + 'MAX_SESSIONS', + 'MAX_ADMIN_SESSIONS', + 'SESSION_FIELDS', + 'TERMINATED_DISPLAY_STATE', + 'TERMINATED_SESSION_STATES', + 'USER_BUCKETS', +) + +from ideadatamodel import ( + constants, + exceptions, + locale, + GetUserProjectsRequest, + ProjectBedrockUsage, + MyCostsAi, + MyCostsAiModel, + MyCostsAiProject, + MyCostsDesktopSession, + MyCostsDesktops, + MyCostsJob, + MyCostsJobGroup, + MyCostsJobs, + GetMyCostsSummaryResult, + ListUserCostsResult, + UserCosts, +) +from ideasdk.utils import Utils + +from boto3.dynamodb.conditions import Key + +from ideaclustermanager.app.projects.bedrock_usage_service import ( + USAGE_WINDOW, + get_project_window_usage, + list_bedrock_projects, + usage_window_dates, +) + +import arrow +from typing import Dict, List, Optional + +# states in which the instance behind a desktop is still running and still costing +LIVE_SESSION_STATES = frozenset( + {'PROVISIONING', 'CREATING', 'INITIALIZING', 'READY', 'RESUMING'} +) + +# a deleted desktop still cost what it cost while it ran. the controller hard deletes +# the table row, so the session index is the only remaining record of it. +TERMINATED_SESSION_STATES = frozenset({'DELETING', 'DELETED'}) + +# "DELETED" is the controller's word for a row that is gone; a cost page reader wants +# to know the desktop was terminated. +TERMINATED_DISPLAY_STATE = 'Terminated' + +# Known limitation: the administrator view scans the history table. a cluster that +# outgrows one row per terminated desktop wants an index on the deletion time. +HISTORY_SCAN_PAGES = 1000 + +RECENT_JOBS = 20 +GROUP_BUCKETS = 50 +# Known limitation: one bucket per user in the admin listing. a larger cluster loses the +# tail of the listing and wants a composite aggregation instead. +USER_BUCKETS = 1000 +# Known limitation: one page of sessions; paginate the search if a user ever exceeds it. +MAX_SESSIONS = 200 +# every desktop session in the window across every user. only the fields the hours are +# computed from are fetched, so a page this size stays small. +MAX_ADMIN_SESSIONS = 10000 + +# the fields a session's hours and price are computed from; without this the query +# returns whole session documents, none of the rest of which is read. +SESSION_FIELDS = [ + 'idea_session_id', + 'name', + 'owner', + 'base_os', + 'state', + 'created_on', + 'updated_on', + 'stopped_on', + 'cleanup_warning_stop_time', + 'server.instance_type', +] + +COST_FIELD = 'estimated_bom_cost.total.amount' +# the scheduler sets this when it could not price the instance hours. such a job still +# carries a real amount, so it cannot be detected from the amount being zero. +PRICE_UNAVAILABLE_FIELD = 'estimated_bom_cost.price_unavailable' +UNAVAILABLE_AGG = {'filter': {'term': {PRICE_UNAVAILABLE_FIELD: True}}} + + +def _round(value: float) -> float: + return round(Utils.get_as_float(value, 0.0), 4) + + +class MyCostsService: + def __init__(self, context): + self.context = context + self.logger = context.logger('my-costs') + + # opensearch + + def _os_client(self): + analytics = self.context.analytics_service() + if analytics is None: + return None + return analytics.os_client.os_client + + def _search(self, index: str, body: Dict) -> Optional[Dict]: + os_client = self._os_client() + if os_client is None: + return None + return os_client.search(index=index, body=body) + + # summary + + def get_summary(self, username: str) -> GetMyCostsSummaryResult: + # get_username() is Optional, and an empty owner filter reads as every user + # further down. there is no sensible all-users answer to "what did I cost". + if Utils.is_empty(username): + raise exceptions.unauthorized_access() + + start_date, end_date = usage_window_dates() + window_start = arrow.get(start_date).floor('day') + window_end = arrow.utcnow() + + return GetMyCostsSummaryResult( + username=username, + window=USAGE_WINDOW, + start_date=start_date, + end_date=end_date, + currency=locale.get_currency_code(), + ai=self._ai(username, start_date, end_date), + jobs=self._jobs(username, window_start, window_end), + desktops=self._desktops(username, window_start, window_end), + ) + + # ai + + def _ai(self, username: str, start_date: str, end_date: str) -> MyCostsAi: + try: + projects = self.context.projects.get_user_projects( + GetUserProjectsRequest(username=username) + ).projects + except Exception as e: + self.logger.warning(f'failed to list projects for {username}: {e}') + return MyCostsAi(is_unavailable=True) + + rows: List[MyCostsAiProject] = [] + for project in Utils.get_as_list(projects, []): + row = self._ai_project(username, project, start_date, end_date) + if row is not None: + rows.append(row) + rows.sort(key=lambda entry: entry.total_tokens, reverse=True) + + return MyCostsAi( + invocations=sum(row.invocations for row in rows), + total_tokens=sum(row.total_tokens for row in rows), + cost=_round(sum(row.cost for row in rows)), + estimated=True, + projects=rows, + ) + + def _usage_rows(self, project_id: str, start_date: str, end_date: str): + """ + the window's day rows for one project, read once: the same page answers both + what the caller used and what the project used in total. + """ + projects = self.context.projects + if not projects.bedrock_provisioner.is_enabled(): + return None + dao = projects.bedrock_usage_dao + if dao.table is None: + return None + return dao.query_day_rows(project_id, start_date, end_date) + + @staticmethod + def _total_tokens(rows) -> int: + return sum( + Utils.get_value_as_int('input_tokens', row, 0) + + Utils.get_value_as_int('output_tokens', row, 0) + for row in rows + ) + + def _project_spend(self, project_name: str): + """ + the project's window spend on its own. not lifted off an unscoped usage read: + that carries every user's totals, which do not belong on a per caller page. + """ + scratch = ProjectBedrockUsage() + self.context.projects.apply_bedrock_spend(scratch, project_name) + return scratch.spend + + def _ai_project( + self, username: str, project, start_date: str, end_date: str + ) -> Optional[MyCostsAiProject]: + try: + rows = self._usage_rows(project.project_id, start_date, end_date) + if rows is None: + return None + # the caller's own tokens, from the page already in hand. the aggregation + # filters by username, so no other user's row or name reaches the response. + mine = get_project_window_usage( + self.context.projects.bedrock_usage_dao, + project.project_id, + start_date, + end_date, + username=username, + rows=rows, + ) + except Exception as e: + self.logger.warning( + f'failed to read bedrock usage for project {project.project_id}: {e}' + ) + return None + if mine is None or Utils.get_as_int(mine.total_tokens, 0) == 0: + return None + + row = MyCostsAiProject( + project_id=project.project_id, + project_name=project.name, + project_title=project.title, + invocations=Utils.get_as_int(mine.invocations, 0), + input_tokens=Utils.get_as_int(mine.input_tokens, 0), + output_tokens=Utils.get_as_int(mine.output_tokens, 0), + total_tokens=Utils.get_as_int(mine.total_tokens, 0), + cost=0.0, + estimated=True, + by_model=[ + MyCostsAiModel( + model_id=model.model_id, + invocations=Utils.get_as_int(model.invocations, 0), + input_tokens=Utils.get_as_int(model.input_tokens, 0), + output_tokens=Utils.get_as_int(model.output_tokens, 0), + total_tokens=Utils.get_as_int(model.total_tokens, 0), + cost=0.0, + estimated=True, + ) + for model in Utils.get_as_list(mine.by_model, []) + ], + ) + try: + spend = self._project_spend(project.name) + except Exception as e: + self.logger.warning( + f'failed to read project bedrock spend for {project.project_id}: {e}' + ) + spend = None + + self._apportion(row, self._total_tokens(rows), spend) + return row + + @staticmethod + def _apportion(row: MyCostsAiProject, project_tokens: int, spend): + """ + the project's bedrock spend over the same window, split by the caller's token + share and then by their share per model. a spend that cannot be read is marked + unavailable rather than priced at zero: "no charge" must not follow a failed call. + """ + if spend is None or project_tokens <= 0: + row.cost_unavailable = True + row.cost = 0.0 + for model in row.by_model: + model.cost = None + return + + share = row.total_tokens / project_tokens + row.cost = _round(Utils.get_as_float(spend.amount, 0.0) * share) + for model in row.by_model: + model.cost = _round( + Utils.get_as_float(spend.amount, 0.0) + * (model.total_tokens / project_tokens) + ) + + # jobs + + def _jobs_index(self) -> Optional[str]: + config = self.context.config() + if not config.is_module_enabled(constants.MODULE_SCHEDULER): + return None + module_id = config.get_module_id(constants.MODULE_SCHEDULER) + return f'{self.context.cluster_name()}_{module_id}_jobs' + + def _jobs(self, username: str, window_start, window_end) -> MyCostsJobs: + """ + completed jobs the caller owns, from the index the scheduler writes. the per + project and per queue totals are aggregated in opensearch so they cover every + job in the window, not just the recent ones listed. + """ + try: + index = self._jobs_index() + if index is None: + return MyCostsJobs(is_unavailable=True) + group = { + 'terms': {'field': 'project.raw', 'size': GROUP_BUCKETS}, + 'aggs': {'cost': {'sum': {'field': COST_FIELD}}}, + } + response = self._search( + index, + { + 'size': RECENT_JOBS, + 'track_total_hits': True, + 'sort': [{'end_time': {'order': 'desc'}}], + '_source': [ + 'job_id', + 'name', + 'queue', + 'project', + 'end_time', + 'estimated_bom_cost', + ], + 'query': { + 'bool': { + 'filter': [ + {'term': {'owner.raw': username}}, + { + 'range': { + 'end_time': { + 'gte': window_start.isoformat(), + 'lte': window_end.isoformat(), + } + } + }, + ] + } + }, + 'aggs': { + 'cost': {'sum': {'field': COST_FIELD}}, + # jobs that carry an estimate. a sum over a missing field is + # zero, so an unpriced job would otherwise look like a free one. + 'priced': {'value_count': {'field': COST_FIELD}}, + # jobs whose estimate is missing its compute. they carry an + # amount, so 'priced' counts them and only this finds them. + 'unavailable': UNAVAILABLE_AGG, + 'by_project': group, + 'by_queue': { + 'terms': {'field': 'queue.raw', 'size': GROUP_BUCKETS}, + 'aggs': {'cost': {'sum': {'field': COST_FIELD}}}, + }, + }, + }, + ) + except Exception as e: + self.logger.warning(f'failed to read jobs for {username}: {e}') + return MyCostsJobs(is_unavailable=True) + + if response is None: + return MyCostsJobs(is_unavailable=True) + + hits = Utils.get_value_as_dict('hits', response, {}) + aggregations = Utils.get_value_as_dict('aggregations', response, {}) + job_count = Utils.get_value_as_int( + 'value', Utils.get_value_as_dict('total', hits, {}), 0 + ) + priced = Utils.get_value_as_int( + 'value', Utils.get_value_as_dict('priced', aggregations, {}), 0 + ) + unavailable = Utils.get_value_as_int( + 'doc_count', Utils.get_value_as_dict('unavailable', aggregations, {}), 0 + ) + return MyCostsJobs( + job_count=job_count, + cost=_round( + Utils.get_value_as_float( + 'value', Utils.get_value_as_dict('cost', aggregations, {}), 0.0 + ) + ), + unpriced_jobs=max(job_count - priced, 0), + cost_unavailable=unavailable > 0, + estimated=True, + by_project=self._job_groups(aggregations, 'by_project'), + by_queue=self._job_groups(aggregations, 'by_queue'), + recent_jobs=[ + self._job(Utils.get_value_as_dict('_source', hit, {})) + for hit in Utils.get_value_as_list('hits', hits, []) + ], + ) + + @staticmethod + def _job_groups(aggregations: Dict, key: str) -> List[MyCostsJobGroup]: + buckets = Utils.get_value_as_list( + 'buckets', Utils.get_value_as_dict(key, aggregations, {}), [] + ) + return [ + MyCostsJobGroup( + name=Utils.get_value_as_string('key', bucket), + job_count=Utils.get_value_as_int('doc_count', bucket, 0), + cost=_round( + Utils.get_value_as_float( + 'value', Utils.get_value_as_dict('cost', bucket, {}), 0.0 + ) + ), + ) + for bucket in buckets + ] + + @staticmethod + def _job(source: Dict) -> MyCostsJob: + bom = Utils.get_value_as_dict('estimated_bom_cost', source, {}) + total = Utils.get_value_as_dict('total', bom, {}) + return MyCostsJob( + job_id=Utils.get_value_as_string('job_id', source), + name=Utils.get_value_as_string('name', source), + queue=Utils.get_value_as_string('queue', source), + project=Utils.get_value_as_string('project', source), + end_time=Utils.get_value_as_string('end_time', source), + cost=_round(Utils.get_value_as_float('amount', total, 0.0)), + # only from the flag: a job cancelled before it ran was priced normally, + # and its zero is the real answer. + cost_unavailable=Utils.get_value_as_bool('price_unavailable', bom, False), + ) + + # desktops + + def _desktop_hits( + self, + username: Optional[str], + start_ms: int, + end_ms: int, + size: int = None, + all_users: bool = False, + ): + """ + raw session documents overlapping the window. the admin listing asks for every + user by saying so; a missing username is a bug, never a wildcard. no state is + filtered: a desktop deleted after running in the window still cost what it cost. + """ + if not all_users and Utils.is_empty(username): + raise exceptions.unauthorized_access() + index = self.context.config().get_string( + 'virtual-desktop-controller.opensearch.dcv_session.alias' + ) + if Utils.is_empty(index): + return None + owner = [] if all_users else [{'term': {'owner.raw': username}}] + response = self._search( + index, + { + 'size': size if size is not None else MAX_SESSIONS, + '_source': SESSION_FIELDS, + 'sort': [{'created_on': {'order': 'desc'}}], + 'query': { + 'bool': { + 'filter': owner + + [ + {'range': {'created_on': {'lte': end_ms}}}, + {'range': {'updated_on': {'gte': start_ms}}}, + ] + } + }, + }, + ) + if response is None: + return None + return Utils.get_value_as_list( + 'hits', Utils.get_value_as_dict('hits', response, {}), [] + ) + + @staticmethod + def _desktop_totals(sessions: List[MyCostsDesktopSession]) -> MyCostsDesktops: + """ + cost is the subtotal of the sessions that had a price. unpriced_sessions says + how many are missing from it, so a partial subtotal is never read as the whole. + """ + return MyCostsDesktops( + session_count=len(sessions), + hours=_round(sum(session.hours for session in sessions)), + cost=_round( + sum(Utils.get_as_float(session.cost, 0.0) for session in sessions) + ), + unpriced_sessions=sum( + 1 for session in sessions if session.price_unavailable + ), + estimated=True, + sessions=sessions, + ) + + def _desktops(self, username: str, window_start, window_end) -> MyCostsDesktops: + """ + the caller's desktop sessions overlapping the window, priced at the recorded + hours times the on-demand rate. an unpriced instance type reports hours only. + """ + start_ms = window_start.int_timestamp * 1000 + end_ms = window_end.int_timestamp * 1000 + try: + hits = self._desktop_hits(username, start_ms, end_ms) + except Exception as e: + self.logger.warning(f'failed to read desktop sessions for {username}: {e}') + return MyCostsDesktops(is_unavailable=True) + + if hits is None: + return MyCostsDesktops(is_unavailable=True) + + hits = list(hits) + self._history_hits(username, start_ms, end_ms) + return self._desktop_totals(self._sessions_from(hits, start_ms, end_ms)) + + def _history_table(self): + """ + the controller's record of terminated desktops, absent until the controller + writes one, which is not an error. + """ + config = self.context.config() + if not config.is_module_enabled(constants.MODULE_VIRTUAL_DESKTOP_CONTROLLER): + return None + module_id = config.get_module_id(constants.MODULE_VIRTUAL_DESKTOP_CONTROLLER) + name = f'{self.context.cluster_name()}.{module_id}.controller.session-history' + return self.context.aws().dynamodb_table().Table(name) + + def _history_hits( + self, + username: Optional[str], + start_ms: int, + end_ms: int, + all_users: bool = False, + ) -> List[Dict]: + """ + desktops that ran in the window and have since been deleted; this record is the + only evidence left of them. shaped like a search hit so one builder serves both. + """ + if not all_users and Utils.is_empty(username): + raise exceptions.unauthorized_access() + try: + table = self._history_table() + if table is None: + return [] + if all_users: + rows = self._scan_history(table) + else: + rows = Utils.get_value_as_list( + 'Items', + table.query(KeyConditionExpression=Key('owner').eq(username)), + [], + ) + except Exception as e: + # additive: a history read that fails leaves the live sessions intact + self.logger.warning(f'failed to read desktop session history: {e}') + return [] + + hits = [] + for row in rows: + created_on = Utils.get_value_as_int('created_on', row, 0) + deleted_on = Utils.get_value_as_int('deleted_on', row, 0) + # the desktop has to have existed during the window to have cost anything + if deleted_on < start_ms or (created_on > 0 and created_on > end_ms): + continue + hits.append( + { + '_history': True, + '_source': { + 'idea_session_id': Utils.get_value_as_string( + 'idea_session_id', row + ), + 'name': Utils.get_value_as_string('name', row), + 'owner': Utils.get_value_as_string('owner', row), + 'base_os': Utils.get_value_as_string('base_os', row), + # the record only exists because the desktop was terminated + 'state': 'DELETED', + 'created_on': created_on, + 'updated_on': deleted_on, + 'stopped_on': Utils.get_value_as_int('stopped_on', row, 0), + 'server': { + 'instance_type': Utils.get_value_as_string( + 'instance_type', row + ) + }, + }, + } + ) + return hits + + @staticmethod + def _scan_history(table) -> List[Dict]: + rows: List[Dict] = [] + last_key = None + for _ in range(HISTORY_SCAN_PAGES): + kwargs = {} if last_key is None else {'ExclusiveStartKey': last_key} + result = table.scan(**kwargs) + rows.extend(Utils.get_value_as_list('Items', result, [])) + last_key = Utils.get_value_as_dict('LastEvaluatedKey', result, None) + if last_key is None: + break + return rows + + @staticmethod + def _newest_per_session(hits) -> List[Dict]: + """ + one document per desktop, the most recently written. + + the alias spans every generation of the versioned index, so a reindexed desktop + has a document under more than one and counting both would bill it twice. a + history record always wins: it is written at the deletion and is complete. + """ + newest: Dict[str, Dict] = {} + from_history: Dict[str, bool] = {} + for hit in hits: + source = Utils.get_value_as_dict('_source', hit, {}) + session_id = Utils.get_value_as_string('idea_session_id', source, '') + if Utils.is_empty(session_id): + continue + is_history = Utils.get_value_as_bool('_history', hit, False) + seen = newest.get(session_id) + if seen is None: + newest[session_id] = source + from_history[session_id] = is_history + continue + if from_history.get(session_id) and not is_history: + continue + if is_history and not from_history.get(session_id): + newest[session_id] = source + from_history[session_id] = True + continue + if Utils.get_value_as_int('updated_on', source, 0) >= ( + Utils.get_value_as_int('updated_on', seen, 0) + ): + newest[session_id] = source + from_history[session_id] = is_history + return list(newest.values()) + + def _sessions_from( + self, hits, start_ms: int, end_ms: int + ) -> List[MyCostsDesktopSession]: + sessions = [ + self._desktop_session(source, start_ms, end_ms) + for source in self._newest_per_session(hits) + ] + return [session for session in sessions if session is not None] + + def _desktop_session( + self, source: Dict, start_ms: int, end_ms: int + ) -> Optional[MyCostsDesktopSession]: + created_on = Utils.get_value_as_int('created_on', source, 0) + updated_on = Utils.get_value_as_int('updated_on', source, 0) + state = Utils.get_value_as_string('state', source, '') + + stopped_at, stop_time_estimated = self._stop_time( + source, state, updated_on, end_ms + ) + started = max(created_on, start_ms) + stopped = min(stopped_at, end_ms) + if stopped <= started: + return None + + hours = (stopped - started) / 3600000.0 + server = Utils.get_value_as_dict('server', source, {}) + instance_type = Utils.get_value_as_string('instance_type', server) + + session = MyCostsDesktopSession( + idea_session_id=Utils.get_value_as_string('idea_session_id', source), + name=Utils.get_value_as_string('name', source), + instance_type=instance_type, + base_os=Utils.get_value_as_string('base_os', source), + state=TERMINATED_DISPLAY_STATE + if state in TERMINATED_SESSION_STATES + else state, + started_on=arrow.get(started / 1000).isoformat(), + ended_on=arrow.get(stopped / 1000).isoformat(), + hours=_round(hours), + estimated=True, + ) + if stop_time_estimated: + session.stop_time_estimated = True + + price = self._ondemand_price(instance_type) + if price is None: + session.price_unavailable = True + else: + session.cost = _round(hours * price) + return session + + # admin listing + + def list_user_costs(self) -> ListUserCostsResult: + """ + one row per user with any measured cost in the window, across the same three + sources the per user summary reads. an unreadable source is reported as + unavailable rather than leaving rows silently short. + """ + start_date, end_date = usage_window_dates() + window_start = arrow.get(start_date).floor('day') + window_end = arrow.utcnow() + start_ms = window_start.int_timestamp * 1000 + end_ms = window_end.int_timestamp * 1000 + + rows: Dict[str, UserCosts] = {} + + def row_for(username: str) -> UserCosts: + return rows.setdefault( + username, + UserCosts( + username=username, + ai_requests=0, + ai_tokens=0, + ai_cost=0.0, + desktop_session_count=0, + desktop_hours=0.0, + desktop_cost=0.0, + desktop_unpriced_sessions=0, + job_count=0, + job_cost=0.0, + job_unpriced_jobs=0, + total_cost=0.0, + ), + ) + + ai_unavailable = self._ai_by_user(row_for) + jobs_unavailable = self._jobs_by_user(row_for, window_start, window_end) + desktops_unavailable = self._desktops_by_user(row_for, start_ms, end_ms) + + listing = list(rows.values()) + for row in listing: + row.ai_cost = _round(row.ai_cost) + row.desktop_cost = _round(row.desktop_cost) + row.job_cost = _round(row.job_cost) + row.total_cost = _round(row.ai_cost + row.desktop_cost + row.job_cost) + listing.sort(key=lambda entry: entry.total_cost, reverse=True) + + return ListUserCostsResult( + window=USAGE_WINDOW, + start_date=start_date, + end_date=end_date, + currency=locale.get_currency_code(), + listing=listing, + ai_unavailable=ai_unavailable, + jobs_unavailable=jobs_unavailable, + desktops_unavailable=desktops_unavailable, + ) + + def _ai_by_user(self, row_for) -> bool: + try: + projects = list_bedrock_projects(self.context.projects.projects_dao) + except Exception as e: + self.logger.warning(f'failed to list bedrock projects: {e}') + return True + + for project in projects: + try: + # unscoped read: the per project total plus its by_user split, the + # same call the projects page makes. Known limitation: by_user is + # capped at max_users_per_project, so a larger project loses its tail. + usage = self.context.projects.get_project_bedrock_usage( + project_id=project.project_id, project_name=project.name + ) + except Exception as e: + self.logger.warning( + f'failed to read bedrock usage for project {project.project_id}: {e}' + ) + continue + if usage is None: + continue + + project_tokens = Utils.get_as_int(usage.total_tokens, 0) + spend = usage.spend + for entry in Utils.get_as_list(usage.by_user, []): + row = row_for(entry.username) + row.ai_requests += Utils.get_as_int(entry.invocations, 0) + row.ai_tokens += Utils.get_as_int(entry.total_tokens, 0) + if spend is None or project_tokens <= 0: + row.ai_cost_unavailable = True + continue + row.ai_cost += ( + Utils.get_as_float(spend.amount, 0.0) + * Utils.get_as_int(entry.total_tokens, 0) + / project_tokens + ) + return False + + def _jobs_by_user(self, row_for, window_start, window_end) -> bool: + try: + index = self._jobs_index() + if index is None: + return True + response = self._search( + index, + { + 'size': 0, + 'query': { + 'bool': { + 'filter': [ + { + 'range': { + 'end_time': { + 'gte': window_start.isoformat(), + 'lte': window_end.isoformat(), + } + } + } + ] + } + }, + 'aggs': { + 'by_user': { + 'terms': {'field': 'owner.raw', 'size': USER_BUCKETS}, + 'aggs': { + 'cost': {'sum': {'field': COST_FIELD}}, + 'priced': {'value_count': {'field': COST_FIELD}}, + 'unavailable': UNAVAILABLE_AGG, + }, + } + }, + }, + ) + except Exception as e: + self.logger.warning(f'failed to aggregate jobs by user: {e}') + return True + + if response is None: + return True + + buckets = Utils.get_value_as_list( + 'buckets', + Utils.get_value_as_dict( + 'by_user', Utils.get_value_as_dict('aggregations', response, {}), {} + ), + [], + ) + for bucket in buckets: + username = Utils.get_value_as_string('key', bucket) + if Utils.is_empty(username): + continue + row = row_for(username) + count = Utils.get_value_as_int('doc_count', bucket, 0) + priced = Utils.get_value_as_int( + 'value', Utils.get_value_as_dict('priced', bucket, {}), 0 + ) + row.job_count += count + row.job_unpriced_jobs += max(count - priced, 0) + row.job_cost += Utils.get_value_as_float( + 'value', Utils.get_value_as_dict('cost', bucket, {}), 0.0 + ) + if ( + Utils.get_value_as_int( + 'doc_count', Utils.get_value_as_dict('unavailable', bucket, {}), 0 + ) + > 0 + ): + row.job_cost_unavailable = True + return False + + def _desktops_by_user(self, row_for, start_ms: int, end_ms: int) -> bool: + try: + hits = self._desktop_hits( + None, start_ms, end_ms, size=MAX_ADMIN_SESSIONS, all_users=True + ) + except Exception as e: + self.logger.warning(f'failed to read desktop sessions for all users: {e}') + return True + + if hits is None: + return True + + hits = list(hits) + self._history_hits(None, start_ms, end_ms, all_users=True) + + by_owner: Dict[str, List] = {} + for hit in hits: + source = Utils.get_value_as_dict('_source', hit, {}) + owner = Utils.get_value_as_string('owner', source, '') + if Utils.is_empty(owner): + continue + by_owner.setdefault(owner, []).append(hit) + + for owner, owner_hits in by_owner.items(): + totals = self._desktop_totals( + self._sessions_from(owner_hits, start_ms, end_ms) + ) + row = row_for(owner) + row.desktop_session_count += Utils.get_as_int(totals.session_count, 0) + row.desktop_hours += Utils.get_as_float(totals.hours, 0.0) + row.desktop_cost += Utils.get_as_float(totals.cost, 0.0) + row.desktop_unpriced_sessions += Utils.get_as_int( + totals.unpriced_sessions, 0 + ) + return False + + @staticmethod + def _stop_time(source: Dict, state: str, updated_on: int, end_ms: int): + """ + when the desktop stopped costing, and whether that had to be inferred. + + a live desktop bills to the end of the window; a stopped or terminated one to + the recorded stop time. updated_on is a last resort because it moves on any + write, so a desktop stopped on day one and renamed on day twenty five would + otherwise bill the gap. it is reported as an estimate whenever it is used. + """ + if state in LIVE_SESSION_STATES: + return end_ms, False + + stopped_on = Utils.get_value_as_int('stopped_on', source, 0) + if stopped_on > 0: + return stopped_on, False + + # older sessions stopped before the stop time was recorded. the cleanup notice + # carries a real ec2 stop time for some of them; either way it is a guess. + cleanup_stop = Utils.get_value_as_int('cleanup_warning_stop_time', source, 0) + if cleanup_stop > 0 and updated_on > 0: + return min(cleanup_stop, updated_on), True + if cleanup_stop > 0: + return cleanup_stop, True + return updated_on, True + + def _ondemand_price(self, instance_type: str) -> Optional[float]: + if Utils.is_empty(instance_type): + return None + try: + unit_price = self.context.aws_util().get_ec2_instance_type_unit_price( + instance_type + ) + except Exception as e: + self.logger.warning(f'failed to price instance type {instance_type}: {e}') + return None + # no answer at all outside the commercial partition, or when the lookup failed. + if unit_price is None: + return None + ondemand = Utils.get_as_float(getattr(unit_price, 'ondemand', None), 0.0) + if ondemand <= 0: + return None + return ondemand diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/bedrock_provisioner.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/bedrock_provisioner.py index ed079c97..2c11f2d2 100644 --- a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/bedrock_provisioner.py +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/bedrock_provisioner.py @@ -421,21 +421,37 @@ def reconcile_project( else: self._teardown(project) except botocore.exceptions.ClientError as e: - if not self._is_access_denied(e): - raise e + code = e.response.get('Error', {}).get('Code', 'ClientError') + denied = self._is_access_denied(e) + # one line per pass, not one per model: the per model reasons are + # already recorded on the project. + self.logger.error( + f'bedrock reconcile failed for project {project.project_id}: ' + f'{code} on {e.operation_name}: {e}' + ) reason = ( f'denied: {e}. redeploy the cluster-manager module so its role ' f'carries the bedrock provisioner permissions.' ) - self.logger.error( - f'bedrock reconcile for project {project.name} was {reason}' + self._record_reconcile_failure( + project, + # the code and the failing call are what an administrator acts + # on; the full message stays in the log. + error=f'{code} on {e.operation_name}', + policy_errors={e.operation_name: reason} if denied else None, ) - self._record_access_denied(project, e.operation_name, reason) + # a denial clears with a module redeploy, so raising would only fail the + # task on every retry. anything else may be transient and has to keep + # failing it so the queue redelivers. + if not denied: + raise e - def _record_access_denied(self, project: Project, operation: str, reason: str): + def _record_reconcile_failure( + self, project: Project, error: str, policy_errors: Optional[Dict[str, str]] + ): """ - recorded against the project like a refused policy, so the administrator sees - it in the project view. the provisioned fields are carried over unchanged. + recorded against the project like a refused policy, so a broken integration is + visible in the project view. the provisioned fields are carried over unchanged. """ bedrock = project.bedrock self._save_provisioner_fields( @@ -448,7 +464,8 @@ def _record_access_denied(self, project: Project, operation: str, reason: str): bedrock.inference_profile_arns if bedrock is not None else None, {} ), model_errors=bedrock.model_errors if bedrock is not None else None, - policy_errors={operation: reason}, + policy_errors=policy_errors, + reconcile_error=error, ) @staticmethod @@ -1281,6 +1298,7 @@ def _save_provisioner_fields( inference_profile_arns: Dict[str, str], model_errors: Optional[Dict[str, str]] = None, policy_errors: Optional[Dict[str, str]] = None, + reconcile_error: Optional[str] = None, ): # enabled and model_ids are re-read from storage: an administrator edit # that landed during this reconcile must not be reverted by the write back. @@ -1309,6 +1327,9 @@ def _save_provisioner_fields( bedrock['model_errors'] = model_errors if policy_errors: bedrock['policy_errors'] = policy_errors + if reconcile_error: + bedrock['reconcile_error'] = reconcile_error + bedrock['reconcile_error_on'] = Utils.current_time_ms() self.projects_dao.update_project( {'project_id': project.project_id, 'bedrock': bedrock} diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/bedrock_usage_service.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/bedrock_usage_service.py index 35a597bb..79893536 100644 --- a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/bedrock_usage_service.py +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/bedrock_usage_service.py @@ -3,17 +3,25 @@ 'INVOCATION_QUERY', 'CALLER_ARN_RE', 'INSTANCE_ID_RE', + 'USAGE_WINDOW', + 'USAGE_WINDOW_DAYS', 'parse_caller_arn', 'row_to_dict', - 'get_project_usage_by_model', + 'usage_window_dates', + 'get_project_window_usage', + 'list_bedrock_projects', + 'apportion_spend', ) from ideadatamodel import ( constants, exceptions, BedrockModelUsage, + BedrockUserUsage, ListProjectsRequest, Project, + ProjectBedrockUsage, + SocaAmount, ) from ideasdk.context import SocaContext from ideasdk.service import SocaService @@ -53,6 +61,11 @@ | sort invocations desc | limit {limit}""" +# a trailing window rather than the calendar month, so a project last used a few weeks +# ago still reads as used on the first of a month. +USAGE_WINDOW_DAYS = 30 +USAGE_WINDOW = 'last_30_days' + QUERY_POLL_INTERVAL_SECONDS = 2 DESCRIBE_INSTANCES_CHUNK_SIZE = 200 INSTANCE_CACHE_MAX_ENTRIES = 20000 @@ -91,35 +104,140 @@ def _row_counters(row: Dict) -> Dict[str, int]: } -def get_project_usage_by_model( - usage_dao, project_id: str, period: str -) -> List[BedrockModelUsage]: +def usage_window_dates(days: int = USAGE_WINDOW_DAYS) -> Tuple[str, str]: + """the inclusive [today - (days - 1), today] utc dates the day rows are keyed by.""" + today = arrow.utcnow() + return ( + today.shift(days=-(days - 1)).format('YYYY-MM-DD'), + today.format('YYYY-MM-DD'), + ) + + +def _usage_entry(counters: Dict[str, int]) -> Dict[str, int]: + return { + 'invocations': counters['invocations'], + 'input_tokens': counters['input_tokens'], + 'output_tokens': counters['output_tokens'], + 'total_tokens': counters['input_tokens'] + counters['output_tokens'], + } + + +def get_project_window_usage( + usage_dao, + project_id: str, + start_date: str, + end_date: str, + username: str = None, + max_users: int = 50, + rows: Optional[List[Dict]] = None, +) -> Optional[ProjectBedrockUsage]: """ - per model totals for one period, summed across users. the stored rollups are per - user and per job, so the model dimension is aggregated on read from the day rows. - takes the dao because the read path holds one but not the usage service itself. + project usage over a trailing window, summed from the stored day rows. the month + rollups are left as they are, so budgets and anything else keyed on project#YYYY-MM + are unaffected. rows may be passed in by a caller that already holds the window, so + one page can be aggregated more than one way without being read twice. """ + if rows is None: + rows = usage_dao.query_day_rows(project_id, start_date, end_date) + if Utils.is_not_empty(username): + rows = [ + row + for row in rows + if Utils.get_value_as_string('username', row) == username + ] + if len(rows) == 0: + return None + + totals = _new_counters() + by_user: Dict[str, Dict[str, int]] = {} by_model: Dict[str, Dict[str, int]] = {} - for row in usage_dao.query_day_rows(project_id, f'{period}-01', f'{period}-31'): + user_model_tokens: Dict[Tuple[str, str], int] = {} + updated_on = 0 + for row in rows: + counters = _row_counters(row) + _add_counters(totals, counters) + row_user = Utils.get_value_as_string('username', row, UNATTRIBUTED_USER) + _add_counters(by_user.setdefault(row_user, _new_counters()), counters) model_id = Utils.get_value_as_string('model_id', row) - if Utils.is_empty(model_id): - continue - _add_counters( - by_model.setdefault(model_id, _new_counters()), _row_counters(row) - ) + if Utils.is_not_empty(model_id): + _add_counters(by_model.setdefault(model_id, _new_counters()), counters) + key = (row_user, model_id) + user_model_tokens[key] = user_model_tokens.get(key, 0) + ( + counters['input_tokens'] + counters['output_tokens'] + ) + updated_on = max(updated_on, Utils.get_value_as_int('updated_on', row, 0)) - entries = [ - BedrockModelUsage( - model_id=model_id, - invocations=counters['invocations'], - input_tokens=counters['input_tokens'], - output_tokens=counters['output_tokens'], - total_tokens=counters['input_tokens'] + counters['output_tokens'], - ) + models = [ + BedrockModelUsage(model_id=model_id, **_usage_entry(counters)) for model_id, counters in by_model.items() ] - entries.sort(key=lambda entry: entry.total_tokens, reverse=True) - return entries + models.sort(key=lambda entry: entry.total_tokens, reverse=True) + + usage = ProjectBedrockUsage( + window=USAGE_WINDOW, + username=username, + updated_on=arrow.get(updated_on / 1000).datetime if updated_on else None, + by_model=models, + **_usage_entry(totals), + ) + if Utils.is_empty(username): + users = [ + BedrockUserUsage( + username=name, + top_model_id=_top_model(user_model_tokens, name), + **_usage_entry(counters), + ) + for name, counters in by_user.items() + ] + users.sort(key=lambda entry: entry.total_tokens, reverse=True) + usage.by_user = users[:max_users] + return usage + + +def _top_model(user_model_tokens: Dict[Tuple[str, str], int], username: str): + models = [ + (tokens, model_id) + for (name, model_id), tokens in user_model_tokens.items() + if name == username + ] + if len(models) == 0: + return None + return max(models)[1] + + +def apportion_spend(usage: ProjectBedrockUsage): + """ + split the project's window spend across its models and users by token share, marked + estimated. cost explorer prices a cost allocation tag, never a model or a caller, so + a share is the only breakdown there is. nothing is written when the project spend is + unknown: an estimate of an unknown is not zero. + + Known limitation: token share, not per model pricing. if cost explorer ever reports + bedrock spend per model for a tag, price from that and drop the estimated flag. + """ + total_tokens = Utils.get_as_int(usage.total_tokens, 0) + if usage.spend is None or total_tokens <= 0: + return + amount = Utils.get_as_float(usage.spend.amount, 0.0) + for entry in list(usage.by_model or []) + list(usage.by_user or []): + share = Utils.get_as_int(entry.total_tokens, 0) / total_tokens + entry.spend = SocaAmount(amount=round(amount * share, 2)) + entry.spend_is_estimated = True + + +def list_bedrock_projects(projects_dao) -> List[Project]: + """every project carrying a bedrock configuration, paged out of the projects table.""" + projects = [] + cursor = None + for _ in range(MAX_PROJECT_PAGES): + result = projects_dao.list_projects(ListProjectsRequest(cursor=cursor)) + for project in Utils.get_as_list(result.listing, []): + if project.bedrock is not None: + projects.append(project) + cursor = result.paginator.cursor if result.paginator is not None else None + if Utils.is_empty(cursor): + break + return projects class BedrockUsageService(SocaService): @@ -155,7 +273,7 @@ def is_enabled(self) -> bool: def get_interval_seconds(self) -> int: minutes = self.context.config().get_int( - self._config_key('usage.interval_minutes'), 60 + self._config_key('usage.interval_minutes'), 15 ) return max(5, minutes) * 60 @@ -169,14 +287,23 @@ def get_max_query_results(self) -> int: self._config_key('usage.max_query_results'), 10000 ) + def is_truncated(self, records: List) -> bool: + """whether the query hit its row limit, so what came back is a partial window.""" + return len(records) >= self.get_max_query_results() + def get_query_timeout_seconds(self) -> int: return self.context.config().get_int( self._config_key('usage.query_timeout_seconds'), 300 ) def get_retention_days(self) -> int: - return self.context.config().get_int( - self._config_key('usage.retention_days'), 400 + # the ttl on every usage row must outlive USAGE_WINDOW_DAYS, or the reported + # window would read days dynamodb has already expired. + return max( + USAGE_WINDOW_DAYS + 15, + self.context.config().get_int( + self._config_key('usage.retention_days'), 400 + ), ) def get_row_ttl(self) -> int: @@ -255,22 +382,33 @@ def aggregate(self): f'project. account and region invocation logging captures every ' f'bedrock caller, not only idea hosts.' ) - self.store(projects, aggregates, days, job_aggregates=job_aggregates) - def list_bedrock_projects(self) -> List[Project]: - projects = [] - cursor = None - for _ in range(MAX_PROJECT_PAGES): - result = self.projects_service.projects_dao.list_projects( - ListProjectsRequest(cursor=cursor) + # a query that answered with nothing, or was cut off, cannot say what is stale: + # a missing log group reads exactly like a quiet window. stored usage is updated + # and left in place rather than reconciled away, because nothing would restore it. + truncated = self.is_truncated(records) + if len(records) == 0: + self.logger.info( + 'the model invocation log query returned no records for this window. ' + 'usage rows are left unchanged: an empty answer is not evidence that ' + 'recorded usage is gone.' ) - for project in Utils.get_as_list(result.listing, []): - if project.bedrock is not None: - projects.append(project) - cursor = result.paginator.cursor if result.paginator is not None else None - if Utils.is_empty(cursor): - break - return projects + elif truncated: + self.logger.info( + 'the model invocation log query was truncated, so usage rows are ' + 'updated but not reconciled: a partial answer cannot establish that a ' + 'stored row is stale.' + ) + self.store( + projects, + aggregates, + days, + job_aggregates=job_aggregates, + reconcile=not truncated, + ) + + def list_bedrock_projects(self) -> List[Project]: + return list_bedrock_projects(self.projects_service.projects_dao) @staticmethod def build_indexes(projects: List[Project]): @@ -323,7 +461,7 @@ def query_invocation_records( status = Utils.get_value_as_string('status', result) if status == 'Complete': results = Utils.get_value_as_list('results', result, []) - if len(results) >= self.get_max_query_results(): + if self.is_truncated(results): self.logger.warning( f'model invocation log query returned the maximum of ' f'{self.get_max_query_results()} rows. usage for this window is ' @@ -512,11 +650,21 @@ def store( aggregates: Dict[str, Dict[Tuple[str, str, str], Dict[str, int]]], days: List[str], job_aggregates: Dict[str, Dict[Tuple[str, str, str], Dict[str, int]]] = None, + reconcile: bool = True, ): periods = sorted({day[:7] for day in days}) ttl = self.get_row_ttl() updated_on = Utils.current_time_ms() + # the days the query actually returned usage for. a day it said nothing about is + # left alone rather than emptied, because logging can have been off for part of + # a window and not the rest. + days_with_data = { + usage_date + for project_rows in aggregates.values() + for usage_date, _, _ in project_rows + } + for project in projects: project_id = project.project_id desired = aggregates.get(project_id, {}) @@ -568,14 +716,18 @@ def store( } ) - existing = self.usage_dao.query_day_rows( - project_id, days[0], days[-1] - ) + self.usage_dao.query_day_job_rows(project_id, days[0], days[-1]) - existing_keys = { - Utils.get_value_as_string('usage_id', row) for row in existing - } self.usage_dao.put_rows(rows) - self.usage_dao.delete_rows(project_id, existing_keys - desired_keys) + + if reconcile and len(days_with_data) > 0: + existing = self.usage_dao.query_day_rows( + project_id, days[0], days[-1] + ) + self.usage_dao.query_day_job_rows(project_id, days[0], days[-1]) + stale = { + Utils.get_value_as_string('usage_id', row) + for row in existing + if Utils.get_value_as_string('usage_date', row) in days_with_data + } - desired_keys + self.usage_dao.delete_rows(project_id, stale) for period in periods: self.rebuild_rollups(project_id, period, ttl, updated_on) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/db/projects_dao.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/db/projects_dao.py index c360b754..8b202797 100644 --- a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/db/projects_dao.py +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/db/projects_dao.py @@ -79,6 +79,9 @@ def convert_from_db(project: Dict) -> Project: db_bedrock = Utils.get_value_as_dict('bedrock', project) bedrock = None if db_bedrock is not None: + reconcile_error_on = Utils.get_value_as_int( + 'reconcile_error_on', db_bedrock + ) bedrock = ProjectBedrockConfig( enabled=Utils.get_value_as_bool('enabled', db_bedrock, False), model_ids=Utils.get_value_as_list('model_ids', db_bedrock, []), @@ -91,6 +94,12 @@ def convert_from_db(project: Dict) -> Project: ), model_errors=Utils.get_value_as_dict('model_errors', db_bedrock), policy_errors=Utils.get_value_as_dict('policy_errors', db_bedrock), + reconcile_error=Utils.get_value_as_string( + 'reconcile_error', db_bedrock + ), + reconcile_error_on=arrow.get(reconcile_error_on).datetime + if reconcile_error_on is not None + else None, ) db_tags = Utils.get_value_as_dict('tags', project) @@ -172,6 +181,12 @@ def convert_to_db(project: Project) -> Dict: db_bedrock['model_errors'] = project.bedrock.model_errors if project.bedrock.policy_errors is not None: db_bedrock['policy_errors'] = project.bedrock.policy_errors + if project.bedrock.reconcile_error is not None: + db_bedrock['reconcile_error'] = project.bedrock.reconcile_error + if project.bedrock.reconcile_error_on is not None: + db_bedrock['reconcile_error_on'] = Utils.to_milliseconds( + project.bedrock.reconcile_error_on + ) db_project['bedrock'] = db_bedrock return db_project diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/projects_service.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/projects_service.py index 508b657b..227b9dc7 100644 --- a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/projects_service.py +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/projects_service.py @@ -20,13 +20,14 @@ UpdateProjectResult, ListProjectsRequest, ListProjectsResult, + ListBedrockUsageRequest, + ListBedrockUsageResult, EnableProjectRequest, EnableProjectResult, DisableProjectRequest, DisableProjectResult, GetUserProjectsRequest, GetUserProjectsResult, - BedrockUserUsage, Project, ProjectBedrockBudget, ProjectBedrockUsage, @@ -43,7 +44,12 @@ is_bedrock_service, ) from ideaclustermanager.app.projects.bedrock_usage_service import ( - get_project_usage_by_model, + USAGE_WINDOW, + USAGE_WINDOW_DAYS, + apportion_spend, + get_project_window_usage, + list_bedrock_projects, + usage_window_dates, ) from ideaclustermanager.app.projects.db.bedrock_usage_dao import BedrockUsageDAO from ideaclustermanager.app.projects.db.projects_dao import ProjectsDAO @@ -51,7 +57,6 @@ from ideaclustermanager.app.accounts.accounts_service import AccountsService from ideaclustermanager.app.tasks.task_manager import TaskManager -import arrow from typing import Dict, List, Optional, Tuple @@ -104,72 +109,49 @@ def is_bedrock_usage_enabled(self) -> bool: def get_project_bedrock_usage( self, project_id: str, - period: str = None, username: str = None, project_name: str = None, ) -> Optional[ProjectBedrockUsage]: """ - month to date usage for a project, or for one caller within it. read from - the rollups the usage service writes; returns None when nothing is recorded. - project_name is the cost allocation tag value the spend figure is read for. + trailing window usage for a project, or for one caller within it, summed from + the day rows; None when nothing is recorded in the window. the calendar month + rollups are left alone, so budgets are unaffected. project_name is the cost + allocation tag value the spend figure is read for. """ if not self.bedrock_provisioner.is_enabled(): return None if self.bedrock_usage_dao.table is None: return None - if Utils.is_empty(period): - period = arrow.utcnow().format('YYYY-MM') - if Utils.is_not_empty(username): - item = self.bedrock_usage_dao.get_user_rollup( - project_id=project_id, period=period, username=username - ) - if item is None: - return None - return self.build_bedrock_usage(item, period, username=username) - - item = self.bedrock_usage_dao.get_project_rollup( - project_id=project_id, period=period - ) - if item is None: - return None - - usage = self.build_bedrock_usage(item, period) - max_users = self.context.config().get_int( - f'{self.context.module_id()}.bedrock.usage.max_users_per_project', 50 - ) - user_rows = self.bedrock_usage_dao.query_user_rollups( - project_id=project_id, period=period - ) - user_rows.sort( - key=lambda row: Utils.get_value_as_int('total_tokens', row, 0), reverse=True - ) - usage.by_user = [ - BedrockUserUsage( - username=Utils.get_value_as_string('username', row), - invocations=Utils.get_value_as_int('invocations', row, 0), - input_tokens=Utils.get_value_as_int('input_tokens', row, 0), - output_tokens=Utils.get_value_as_int('output_tokens', row, 0), - total_tokens=Utils.get_value_as_int('total_tokens', row, 0), - ) - for row in user_rows[:max_users] - ] - usage.by_model = get_project_usage_by_model( - self.bedrock_usage_dao, project_id, period + start_date, end_date = usage_window_dates() + usage = get_project_window_usage( + self.bedrock_usage_dao, + project_id, + start_date, + end_date, + username=username, + max_users=self.context.config().get_int( + f'{self.context.module_id()}.bedrock.usage.max_users_per_project', 50 + ), ) + if usage is None or Utils.is_not_empty(username): + return usage self.apply_bedrock_spend(usage, project_name) return usage def apply_bedrock_spend(self, usage: ProjectBedrockUsage, project_name: str): """ - month to date bedrock cost for the project cost allocation tag. no answer is + bedrock cost for the project cost allocation tag over the same window the tokens + cover, so the two figures beside each other mean the same thing. no answer is recorded as unavailable, never as zero: an empty result is priced nothing yet. """ spend = None if Utils.is_not_empty(project_name): try: spend = self.context.aws_util().cost_explorer_get_tagged_service_spend( - constants.IDEA_TAG_PROJECT, project_name + constants.IDEA_TAG_PROJECT, + project_name, + days=USAGE_WINDOW_DAYS, ) except Exception as e: self.logger.warning( @@ -188,6 +170,23 @@ def apply_bedrock_spend(self, usage: ProjectBedrockUsage, project_name: str): 2, ) ) + apportion_spend(usage) + + def list_bedrock_usage( + self, request: ListBedrockUsageRequest + ) -> ListBedrockUsageResult: + """ + window usage for every bedrock project in one call, so the AI usage page does + not fan out a request per project. budgets are not evaluated: nothing on that + page reads them and each evaluation is a billed read. + """ + listing = [] + for project in list_bedrock_projects(self.projects_dao): + project.bedrock_usage = self.get_project_bedrock_usage( + project_id=project.project_id, project_name=project.name + ) + listing.append(project) + return ListBedrockUsageResult(listing=listing, window=USAGE_WINDOW) def get_project_bedrock_budget( self, project: Project @@ -198,21 +197,6 @@ def get_project_bedrock_budget( """ return self.bedrock_budget.evaluate(project) - @staticmethod - def build_bedrock_usage( - item: Dict, period: str, username: str = None - ) -> ProjectBedrockUsage: - updated_on = Utils.get_value_as_int('updated_on', item, 0) - return ProjectBedrockUsage( - period=period, - username=username, - invocations=Utils.get_value_as_int('invocations', item, 0), - input_tokens=Utils.get_value_as_int('input_tokens', item, 0), - output_tokens=Utils.get_value_as_int('output_tokens', item, 0), - total_tokens=Utils.get_value_as_int('total_tokens', item, 0), - updated_on=arrow.get(updated_on / 1000).datetime if updated_on else None, - ) - @staticmethod def has_bedrock_provisioner_fields(stored: Optional[Dict]) -> bool: bedrock = Utils.get_value_as_dict('bedrock', stored, {}) if stored else {} diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/web_portal.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/web_portal.py index 7ae26276..17edf9f9 100644 --- a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/web_portal.py +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/web_portal.py @@ -142,6 +142,21 @@ def build_app_init_data(self, http_request) -> Dict: 'session_management': self.context.config().get_string( 'cluster-manager.web_portal.session_management', 'in-memory' ), + # the maintenance notice travels with the page so it renders on the sign-in + # screen, where there is no token to call the settings api with. once signed in + # the portal re-reads it from the api every minute; this value is only as fresh + # as the page load. + 'maintenance': { + 'enabled': self.context.config().get_bool( + 'cluster-manager.maintenance.enabled', False + ), + 'message': self.context.config().get_string( + 'cluster-manager.maintenance.message', '' + ), + 'ends_at': self.context.config().get_string( + 'cluster-manager.maintenance.ends_at', '' + ), + }, } if sso_enabled: diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager_meta/__init__.py b/source/idea/idea-cluster-manager/src/ideaclustermanager_meta/__init__.py index 54c75c73..159e3d31 100644 --- a/source/idea/idea-cluster-manager/src/ideaclustermanager_meta/__init__.py +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager_meta/__init__.py @@ -10,4 +10,4 @@ # and limitations under the License. __name__ = 'idea-cluster-manager' -__version__ = '26.08.0' +__version__ = '26.09.0' diff --git a/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_bedrock_provisioner.py b/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_bedrock_provisioner.py index fd49349b..e4d294bc 100644 --- a/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_bedrock_provisioner.py +++ b/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_bedrock_provisioner.py @@ -1737,3 +1737,122 @@ def test_discovery_actions_stay_inside_the_project_role_boundary(): } assert get_actions_outside_boundary(document, boundary) == [] + + +# a reconcile that could not finish + + +OTHER_PROJECT_ID = 'a1b2c3d4-0000-4000-8000-000000000002' + + +class TwoProjectDAO(FakeProjectsDAO): + """serves more than one project, so a pass over several can be driven.""" + + def __init__(self, projects): + self.projects = {project['project_id']: project for project in projects} + self.updates = [] + self.log = [] + + def get_project_by_id(self, project_id): + stored = self.projects.get(project_id) + return None if stored is None else dict(stored) + + def update_project(self, project): + self.updates.append(project) + project_id = project['project_id'] + self.projects[project_id] = {**self.projects[project_id], **project} + return self.projects[project_id] + + +def denied_for(project_id): + """an iam whose grant does not reach one project's resources.""" + + class DeniedIam(FakeIam): + def get_policy(self, PolicyArn): + if project_id in PolicyArn: + raise access_denied('GetPolicy') + return super().get_policy(PolicyArn) + + return DeniedIam() + + +def test_a_reconcile_denial_is_recorded_on_the_project(): + # without this a broken integration reads exactly like a project nobody enabled + provisioner, _iam, _bedrock, dao = build_provisioner(iam=denied_for(PROJECT_ID)) + logger = RecordingLogger() + provisioner.logger = logger + provisioner.reconcile_project(PROJECT_ID) + + saved = dao.updates[-1]['bedrock'] + assert saved['reconcile_error'] == 'AccessDenied on GetPolicy' + assert saved['reconcile_error_on'] > 0 + # one line for the pass, naming the project, not one per model + assert len(logger.errors) == 1 + assert PROJECT_ID in logger.errors[0] + + +def test_a_denied_project_does_not_cost_the_pass_its_other_projects(): + other = {**db_project(), 'project_id': OTHER_PROJECT_ID} + provisioner, _iam, _bedrock, _dao = build_provisioner(iam=denied_for(PROJECT_ID)) + dao = TwoProjectDAO([db_project(), other]) + provisioner.projects_dao = dao + + for project_id in (PROJECT_ID, OTHER_PROJECT_ID): + provisioner.reconcile_project(project_id) + + saved = {update['project_id']: update['bedrock'] for update in dao.updates} + assert saved[PROJECT_ID]['reconcile_error'] == 'AccessDenied on GetPolicy' + assert 'reconcile_error' not in saved[OTHER_PROJECT_ID] + assert saved[OTHER_PROJECT_ID]['role_arn'] + + +def test_the_reconcile_error_clears_on_the_next_pass_that_completes(): + provisioner, _iam, _bedrock, dao = build_provisioner(iam=denied_for(PROJECT_ID)) + provisioner.reconcile_project(PROJECT_ID) + assert 'reconcile_error' in dao.updates[-1]['bedrock'] + + # same project, a cluster-manager role that now carries the grant + provisioner, _iam2, _bedrock2, dao2 = build_provisioner(project=dao.db_project) + provisioner.reconcile_project(PROJECT_ID) + + saved = dao2.updates[-1]['bedrock'] + assert 'reconcile_error' not in saved + assert 'reconcile_error_on' not in saved + assert saved['role_arn'] + + +def test_a_non_denial_client_error_is_recorded_and_still_raised(): + # throttling has to keep failing the task so the queue redelivers it, but the + # project still carries why the last pass did not finish. + class ThrottledIam(FakeIam): + def get_policy(self, PolicyArn): + raise botocore.exceptions.ClientError( + {'Error': {'Code': 'ThrottlingException', 'Message': 'slow down'}}, + 'GetPolicy', + ) + + provisioner, _iam, _bedrock, dao = build_provisioner(iam=ThrottledIam()) + with pytest.raises(botocore.exceptions.ClientError): + provisioner.reconcile_project(PROJECT_ID) + + saved = dao.updates[-1]['bedrock'] + assert saved['reconcile_error'] == 'ThrottlingException on GetPolicy' + assert 'policy_errors' not in saved + + +def test_a_recorded_reconcile_error_survives_a_project_update(): + stored = db_project( + bedrock={ + 'enabled': True, + 'model_ids': CATALOG, + 'reconcile_error': 'AccessDenied on CreateRole', + 'reconcile_error_on': 1756000000000, + } + ) + project = ProjectsDAO.convert_from_db(stored) + + assert project.bedrock.reconcile_error == 'AccessDenied on CreateRole' + assert project.bedrock.reconcile_error_on is not None + round_tripped = ProjectsDAO.convert_to_db(project)['bedrock'] + assert round_tripped['reconcile_error'] == 'AccessDenied on CreateRole' + assert round_tripped['reconcile_error_on'] == 1756000000000 diff --git a/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_bedrock_usage.py b/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_bedrock_usage.py index 11982b27..fd9bb7c5 100644 --- a/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_bedrock_usage.py +++ b/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_bedrock_usage.py @@ -18,13 +18,18 @@ from ideaclustermanager.app.projects.bedrock_usage_service import ( BedrockUsageService, INVOCATION_QUERY, - get_project_usage_by_model, + USAGE_WINDOW, + USAGE_WINDOW_DAYS, + get_project_window_usage, parse_caller_arn, + usage_window_dates, ) from ideaclustermanager.app.projects.db.bedrock_usage_dao import ( BedrockInstanceOwnerDAO, BedrockUsageDAO, UNATTRIBUTED_USER, + build_day_key, + build_project_key, ) from ideaclustermanager.app.api.projects_api import ProjectsAPI from ideaclustermanager.app.projects.db.projects_dao import ProjectsDAO @@ -34,6 +39,7 @@ constants, exceptions, locale, + ListBedrockUsageRequest, ListProjectsResult, Project, ProjectBedrockConfig, @@ -42,6 +48,7 @@ ) import arrow +import botocore.exceptions import pytest @@ -158,10 +165,13 @@ def put_model_invocation_logging_configuration(self, loggingConfig): class FakeLogs: - def __init__(self, results=None, status='Complete'): + def __init__(self, results=None, status='Complete', start_query_error=None): self.calls = [] self.results = results if results is not None else [] self.status = status + # an error code start_query raises, e.g. the ResourceNotFoundException a log + # group that does not exist yet answers with. + self.start_query_error = start_query_error def start_query(self, logGroupName, startTime, endTime, queryString): self.calls.append( @@ -175,6 +185,11 @@ def start_query(self, logGroupName, startTime, endTime, queryString): }, ) ) + if self.start_query_error is not None: + raise botocore.exceptions.ClientError( + {'Error': {'Code': self.start_query_error, 'Message': 'not found'}}, + 'StartQuery', + ) return {'queryId': 'q-1'} def get_query_results(self, queryId): @@ -331,8 +346,8 @@ def __init__(self): self.raises = False self.calls = [] - def cost_explorer_get_tagged_service_spend(self, tag_key, tag_value): - self.calls.append((tag_key, tag_value)) + def cost_explorer_get_tagged_service_spend(self, tag_key, tag_value, days=None): + self.calls.append((tag_key, tag_value, days)) if self.raises: raise RuntimeError('cost explorer is not reachable') return self.spend @@ -750,8 +765,13 @@ def test_recompute_removes_rows_that_left_the_window(): assert rows[f'project#{PERIOD}']['total_tokens'] == 15 -def test_usage_dropping_to_zero_removes_the_project_rollup(): - service, _, logs, _, usage_dao = build_service( +def test_an_empty_query_leaves_stored_usage_alone(): + """ + an empty answer cannot be told apart from a missing log group, or from logging + enabled after the fact, so the rows stay and expire on their own ttl rather than + being deleted on no evidence. + """ + service, context, logs, _, usage_dao = build_service( results=[insights_row(caller_arn(), MODEL_B, TODAY, 10, 10, 1)] ) service.aggregate() @@ -759,7 +779,16 @@ def test_usage_dropping_to_zero_removes_the_project_rollup(): logs.results = [] service.aggregate() - assert rows_by_usage_id(usage_dao) == {} + rows = rows_by_usage_id(usage_dao) + + assert rows[f'day#{TODAY}#alice#{MODEL_B}']['total_tokens'] == 20 + assert rows[f'user#{PERIOD}#alice']['total_tokens'] == 20 + assert rows[f'project#{PERIOD}']['total_tokens'] == 20 + assert any( + 'empty answer is not evidence' in message + for level, message in context.logger().messages + if level == 'info' + ) def test_instance_owner_is_resolved_once_and_cached_in_dynamodb(): @@ -850,15 +879,16 @@ def test_run_once_takes_the_distributed_lock(): class StubProjectsService: - def __init__(self, context, usage_dao, enabled=True): + def __init__(self, context, usage_dao, enabled=True, projects=None): self.context = context self.logger = context.logger() self.bedrock_usage_dao = usage_dao + self.projects_dao = FakeProjectsDAO(projects if projects is not None else []) self.bedrock_provisioner = type('P', (), {'is_enabled': lambda self: enabled})() - build_bedrock_usage = staticmethod(ProjectsService.build_bedrock_usage) get_project_bedrock_usage = ProjectsService.get_project_bedrock_usage apply_bedrock_spend = ProjectsService.apply_bedrock_spend + list_bedrock_usage = ProjectsService.list_bedrock_usage def test_get_project_bedrock_usage_returns_totals_and_users(): @@ -873,7 +903,8 @@ def test_get_project_bedrock_usage_returns_totals_and_users(): stub = StubProjectsService(context, usage_dao) usage = ProjectsService.get_project_bedrock_usage(stub, project_id=PROJECT_ID) - assert usage.period == PERIOD + assert usage.window == USAGE_WINDOW + assert usage.period is None assert usage.username is None assert usage.total_tokens == 40 assert [entry.username for entry in usage.by_user] == ['bob', 'alice'] @@ -927,7 +958,10 @@ def test_project_usage_by_model_sums_across_users(): ) service.aggregate() - by_model = get_project_usage_by_model(usage_dao, PROJECT_ID, PERIOD) + start_date, end_date = usage_window_dates() + by_model = get_project_window_usage( + usage_dao, PROJECT_ID, start_date, end_date + ).by_model assert [entry.model_id for entry in by_model] == [MODEL_A, MODEL_B] assert by_model[0].total_tokens == 40 @@ -935,6 +969,190 @@ def test_project_usage_by_model_sums_across_users(): assert by_model[1].total_tokens == 2 +# ------------------------------------------------------------------ usage window + +# on the first of a month a project used 11 days earlier read as unused, because the +# column asked for the calendar month rollup. +FROZEN_NOW = arrow.get('2026-09-01T12:00:00+00:00') +IN_WINDOW_DAY = '2026-08-21' +OUT_OF_WINDOW_DAY = '2026-08-02' +WINDOW_MONTH = '2026-08' + + +def freeze_today(monkeypatch): + monkeypatch.setattr(arrow, 'utcnow', lambda: FROZEN_NOW) + + +def seed_day_row( + usage_dao, + usage_date, + username, + model_id, + tokens, + invocations=1, + project_id=PROJECT_ID, +): + usage_dao.table.put_item( + Item={ + 'project_id': project_id, + 'usage_id': build_day_key(usage_date, username, model_id), + 'usage_date': usage_date, + 'period': usage_date[:7], + 'username': username, + 'model_id': model_id, + 'invocations': invocations, + 'input_tokens': tokens, + 'output_tokens': 0, + 'total_tokens': tokens, + 'updated_on': 0, + } + ) + + +# ------------------------------------------------- ai usage page read (list + detail) + + +def build_two_project_usage(monkeypatch, spend=None): + """two bedrock projects, two users and two models inside the window""" + projects = [ + build_project(), + build_project(project_id=PROJECT_ID_2, name='physics', profiles={}), + ] + service, context, _, _, usage_dao = build_service(results=[], projects=projects) + seed_day_row(usage_dao, IN_WINDOW_DAY, 'alice', MODEL_A, 1000, invocations=4) + seed_day_row(usage_dao, IN_WINDOW_DAY, 'bob', MODEL_B, 500, invocations=1) + seed_day_row(usage_dao, OUT_OF_WINDOW_DAY, 'alice', MODEL_A, 9999) + seed_day_row( + usage_dao, IN_WINDOW_DAY, 'alice', MODEL_A, 200, project_id=PROJECT_ID_2 + ) + context.aws_util().spend = spend + freeze_today(monkeypatch) + stub = StubProjectsService(context, usage_dao, projects=projects) + return ProjectsService.list_bedrock_usage(stub, ListBedrockUsageRequest()), context + + +def test_list_bedrock_usage_reports_every_bedrock_project_in_one_call(monkeypatch): + result, _ = build_two_project_usage(monkeypatch) + + assert result.window == USAGE_WINDOW + assert [project.project_id for project in result.listing] == [ + PROJECT_ID, + PROJECT_ID_2, + ] + first = result.listing[0].bedrock_usage + # the out of window day is excluded from the project as it is from the column + assert first.total_tokens == 1500 + assert first.invocations == 5 + assert result.listing[1].bedrock_usage.total_tokens == 200 + + +def test_list_bedrock_usage_breaks_a_project_down_per_model_and_per_user(monkeypatch): + result, _ = build_two_project_usage(monkeypatch) + usage = result.listing[0].bedrock_usage + + assert [(entry.model_id, entry.total_tokens) for entry in usage.by_model] == [ + (MODEL_A, 1000), + (MODEL_B, 500), + ] + assert [(entry.username, entry.total_tokens) for entry in usage.by_user] == [ + ('alice', 1000), + ('bob', 500), + ] + assert [entry.top_model_id for entry in usage.by_user] == [MODEL_A, MODEL_B] + assert usage.by_model[0].input_tokens == 1000 + assert usage.by_model[0].output_tokens == 0 + assert usage.by_model[0].invocations == 4 + + +def test_model_and_user_cost_is_apportioned_by_token_share_and_marked_estimated( + monkeypatch, +): + result, _ = build_two_project_usage(monkeypatch, spend={'Amazon Bedrock': 3.0}) + usage = result.listing[0].bedrock_usage + + assert usage.spend.amount == 3.0 + # 1000 and 500 of 1500 tokens + assert [entry.spend.amount for entry in usage.by_model] == [2.0, 1.0] + assert [entry.spend.amount for entry in usage.by_user] == [2.0, 1.0] + assert all(entry.spend_is_estimated for entry in usage.by_model) + assert all(entry.spend_is_estimated for entry in usage.by_user) + # the project figure is priced by cost explorer, not apportioned, so it carries no + # estimated flag at all: the field only exists on the breakdown entries + assert usage.spend_is_unavailable is None + + +def test_an_unpriced_project_gets_no_estimated_breakdown(monkeypatch): + """no answer from cost explorer must not become an estimate of zero""" + result, _ = build_two_project_usage(monkeypatch, spend=None) + usage = result.listing[0].bedrock_usage + + assert usage.spend_is_unavailable is True + assert all(entry.spend is None for entry in usage.by_model) + assert all(entry.spend_is_estimated is None for entry in usage.by_user) + + +def test_the_usage_window_is_thirty_days_inclusive_of_today(monkeypatch): + freeze_today(monkeypatch) + assert usage_window_dates() == ('2026-08-03', '2026-09-01') + assert USAGE_WINDOW_DAYS == 30 + + +def test_usage_reports_the_trailing_window_not_the_calendar_month(monkeypatch): + service, context, _, _, usage_dao = build_service(results=[]) + seed_day_row(usage_dao, IN_WINDOW_DAY, 'clusteradmin', MODEL_A, 1359) + seed_day_row(usage_dao, OUT_OF_WINDOW_DAY, 'clusteradmin', MODEL_B, 794) + usage_dao.table.put_item( + Item={ + 'project_id': PROJECT_ID, + 'usage_id': build_project_key(WINDOW_MONTH), + 'period': WINDOW_MONTH, + 'invocations': 2, + 'input_tokens': 2153, + 'output_tokens': 0, + 'total_tokens': 2153, + } + ) + freeze_today(monkeypatch) + + stub = StubProjectsService(context, usage_dao) + usage = ProjectsService.get_project_bedrock_usage(stub, project_id=PROJECT_ID) + + assert usage.window == USAGE_WINDOW + assert usage.total_tokens == 1359 + assert usage.invocations == 1 + assert [entry.model_id for entry in usage.by_model] == [MODEL_A] + assert [entry.username for entry in usage.by_user] == ['clusteradmin'] + assert usage.by_user[0].total_tokens == 1359 + + +def test_the_month_rollup_is_untouched_by_the_window_read(monkeypatch): + """budgets and anything else keyed on project#YYYY-MM keep reading what they read""" + service, context, _, _, usage_dao = build_service(results=[]) + seed_day_row(usage_dao, IN_WINDOW_DAY, 'clusteradmin', MODEL_A, 1359) + seed_day_row(usage_dao, OUT_OF_WINDOW_DAY, 'clusteradmin', MODEL_B, 794) + usage_dao.table.put_item( + Item={ + 'project_id': PROJECT_ID, + 'usage_id': build_project_key(WINDOW_MONTH), + 'period': WINDOW_MONTH, + 'total_tokens': 2153, + } + ) + freeze_today(monkeypatch) + + stub = StubProjectsService(context, usage_dao) + ProjectsService.get_project_bedrock_usage(stub, project_id=PROJECT_ID) + + rollup = usage_dao.get_project_rollup(PROJECT_ID, WINDOW_MONTH) + assert rollup['total_tokens'] == 2153 + assert ( + rows_by_usage_id(usage_dao)[ + build_day_key(OUT_OF_WINDOW_DAY, 'clusteradmin', MODEL_B) + ]['total_tokens'] + == 794 + ) + + # ------------------------------------------------------------------ spend @@ -957,7 +1175,10 @@ def test_bedrock_spend_is_unavailable_when_cost_explorer_has_no_answer(): assert usage.spend_is_unavailable is True assert usage.spend is None - assert context.aws_util().calls == [(constants.IDEA_TAG_PROJECT, 'research')] + # priced over the same window the tokens cover, not the calendar month to date + assert context.aws_util().calls == [ + (constants.IDEA_TAG_PROJECT, 'research', USAGE_WINDOW_DAYS) + ] def test_bedrock_spend_is_zero_when_nothing_is_priced_yet(): @@ -1021,7 +1242,7 @@ def test_api_hydration_survives_a_usage_read_failure(): class Exploding: table = usage_dao.table - def get_project_rollup(self, **kwargs): + def query_day_rows(self, *args, **kwargs): raise RuntimeError('table unavailable') stub_projects = StubProjectsService(context, Exploding()) @@ -1114,6 +1335,54 @@ def test_a_job_spanning_days_keeps_every_day_in_its_month_rollup(): assert rows[f'job#{PERIOD}#4242']['invocations'] == 3 +def test_a_missing_log_group_leaves_stored_usage_alone(): + """the same path start_query takes when the log group does not exist yet""" + service, _, logs, _, usage_dao = build_service( + results=[insights_row(caller_arn(), MODEL_A, TODAY, 10, 5, 1)], + owners={INSTANCE_ONE: 'alice'}, + ) + service.aggregate() + + logs.start_query_error = 'ResourceNotFoundException' + service.aggregate() + + assert ( + rows_by_usage_id(usage_dao)[f'day#{TODAY}#alice#{MODEL_A}']['total_tokens'] + == 15 + ) + + +def test_a_truncated_query_upserts_without_deleting(): + """ + a truncated answer is a partial view of every day it covers, so it can say what is + new but never that a stored row is stale. + """ + service, context, logs, _, usage_dao = build_service( + results=[ + insights_row(caller_arn(), MODEL_A, TODAY, 10, 5, 1), + insights_row(caller_arn(), MODEL_B, TODAY, 20, 5, 2), + ], + owners={INSTANCE_ONE: 'alice'}, + ) + service.aggregate() + + # the next run only sees one of the two models, and hits the row limit doing it + logs.results = [insights_row(caller_arn(), MODEL_A, TODAY, 30, 5, 3)] + service.context.config().values[f'{MODULE_ID}.bedrock.usage.max_query_results'] = 1 + service.aggregate() + rows = rows_by_usage_id(usage_dao) + + # upserted + assert rows[f'day#{TODAY}#alice#{MODEL_A}']['total_tokens'] == 35 + # not deleted, even though the truncated answer never mentioned it + assert rows[f'day#{TODAY}#alice#{MODEL_B}']['total_tokens'] == 25 + assert any( + 'cannot establish that a stored row is stale' in message + for level, message in context.logger().messages + if level == 'info' + ) + + def test_a_reattributed_job_loses_its_stale_day_rows_and_rollup(): service, _, _, ec2, usage_dao = build_service( results=[insights_row(caller_arn(), MODEL_A, TODAY, 10, 5, 1)], diff --git a/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_cluster_settings_scoping.py b/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_cluster_settings_scoping.py index 646b5d02..e1fdc1d0 100644 --- a/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_cluster_settings_scoping.py +++ b/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_cluster_settings_scoping.py @@ -66,6 +66,11 @@ 'url': 'https://dashboard.example.com/view', }, }, + 'maintenance': { + 'enabled': True, + 'message': 'Scheduler closed for the 26.09 upgrade.', + 'ends_at': '2026-09-15T18:00:00Z', + }, } GLOBAL_SETTINGS = { @@ -208,7 +213,7 @@ def test_secrets_shaped_keys_never_serialize_for_users(self): ) def test_user_sees_exact_cluster_manager_projection(self): - """cluster-manager projects only the optional dashboard embed keys""" + """cluster-manager projects the optional dashboard embed and the maintenance banner""" settings = self.invoke_get_module_settings( 'cluster-manager', CLUSTER_MANAGER_SETTINGS, elevated=False ) @@ -225,8 +230,17 @@ def test_user_sees_exact_cluster_manager_projection(self): 'url': 'https://dashboard.example.com/view', } }, + # every user has to read these three: the banner is shown to all of them + 'maintenance': { + 'enabled': True, + 'message': 'Scheduler closed for the 26.09 upgrade.', + 'ends_at': '2026-09-15T18:00:00Z', + }, }, ) + # the client id and secret sit beside them in the same module and must not ride along + self.assertNotIn('client_id', settings) + self.assertNotIn('client_secret', settings) def test_unknown_module_projects_to_empty(self): """Modules with no allowlist entry serialize as an empty dict for users""" @@ -251,11 +265,15 @@ def test_user_sees_only_the_bedrock_feature_flag(self): settings = self.invoke_get_module_settings( 'cluster-manager', CLUSTER_MANAGER_BEDROCK_SETTINGS, elevated=False ) - # the custom-dashboard paths are allowlisted for this module as well, so - # their empty parent scaffold rides along. + # the custom-dashboard and maintenance paths are allowlisted for this module as + # well, so their empty parent scaffolds ride along. self.assertEqual( settings, - {'bedrock': {'enabled': True}, 'web_portal': {'custom_dashboard': {}}}, + { + 'bedrock': {'enabled': True}, + 'web_portal': {'custom_dashboard': {}}, + 'maintenance': {}, + }, ) def test_admin_sees_the_bedrock_catalog(self): diff --git a/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_cluster_settings_validation.py b/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_cluster_settings_validation.py index 77ed2a68..cc4be59c 100644 --- a/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_cluster_settings_validation.py +++ b/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_cluster_settings_validation.py @@ -182,6 +182,27 @@ def test_nested_path_extraction(self): self.assertIn('invalid_nested', str(context.exception)) + def test_the_scheduler_default_image_is_allowed(self): + """the custom amis page writes this flat key when a build is adopted""" + + self.api.validate_settings_allowed( + 'scheduler', {'compute_node_ami': 'ami-0123'} + ) + + def test_another_scheduler_setting_is_still_rejected(self): + """the allowance is one key wide, not the whole scheduler module""" + + with self.assertRaises(exceptions.SocaException) as context: + self.api.validate_settings_allowed( + 'scheduler', + {'compute_node_ami': 'ami-0123', 'compute_node_os': 'rocky9'}, + ) + + self.assertIn( + 'not allowed to be updated via web UI: compute_node_os.', + str(context.exception), + ) + if __name__ == '__main__': print('Testing cluster settings validation...') diff --git a/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_costs_api.py b/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_costs_api.py new file mode 100644 index 00000000..400f5729 --- /dev/null +++ b/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_costs_api.py @@ -0,0 +1,190 @@ +""" +Test Cases for CostsAPI and MyCostsAPI ACL dispatch + +These isolate the authorization routing from token decoding and OpenSearch by faking +the AppContext/ApiInvocationContext collaborators. What is under test is who is allowed +to ask about whom, which is the whole reason the admin API is a separate class. +""" + +from ideaclustermanager.app.api.costs_api import CostsAPI +from ideaclustermanager.app.api.my_costs_api import MyCostsAPI +from ideadatamodel import exceptions, GetMyCostsSummaryResult, ListUserCostsResult + +import pytest + + +class FakeLogger: + def warning(self, *_args, **_kwargs): + pass + + +class FakeAppContext: + def module_id(self) -> str: + return 'cluster-manager' + + def logger(self, _name=None): + return FakeLogger() + + +class FakeApiInvocationContext: + """ + Minimal stand-in for ideasdk.api.ApiInvocationContext, implementing only the + surface the two invoke() methods touch. + """ + + def __init__( + self, namespace, username, payload=None, elevated=False, authorized_user=True + ): + self.namespace = namespace + self._username = username + self._payload = payload if payload is not None else {} + self._elevated = elevated + self._authorized_user = authorized_user + self.response_payload = None + + def is_authorized(self, elevated_access: bool, scopes=None) -> bool: + return elevated_access and self._elevated + + def is_authenticated_user(self) -> bool: + return True + + def is_authorized_user(self) -> bool: + return self._authorized_user + + def get_username(self) -> str: + return self._username + + def get_request_payload_as(self, payload_type): + return payload_type(**self._payload) + + def success(self, payload): + self.response_payload = payload + + +class RecordingService: + """records who the service was asked about, which is what these tests assert on.""" + + def __init__(self): + self.summary_calls = [] + self.list_calls = 0 + + def get_summary(self, username): + self.summary_calls.append(username) + return GetMyCostsSummaryResult(username=username) + + def list_user_costs(self): + self.list_calls += 1 + return ListUserCostsResult(listing=[]) + + +def build_costs_api(): + api = CostsAPI(context=FakeAppContext()) + service = RecordingService() + api.my_costs = service + return api, service + + +def build_my_costs_api(): + api = MyCostsAPI(context=FakeAppContext()) + service = RecordingService() + api.my_costs = service + return api, service + + +# admin api + + +def test_get_user_summary_rejects_a_non_elevated_caller(): + api, service = build_costs_api() + context = FakeApiInvocationContext( + 'Costs.GetUserSummary', 'user-a', {'username': 'user-b'}, elevated=False + ) + + with pytest.raises(exceptions.SocaException): + api.invoke(context) + + # the read must not have happened at all + assert service.summary_calls == [] + + +def test_list_user_costs_rejects_a_non_elevated_caller(): + api, service = build_costs_api() + context = FakeApiInvocationContext('Costs.ListUserCosts', 'user-a', elevated=False) + + with pytest.raises(exceptions.SocaException): + api.invoke(context) + + assert service.list_calls == 0 + + +def test_get_user_summary_serves_the_named_user_to_an_elevated_caller(): + api, service = build_costs_api() + context = FakeApiInvocationContext( + 'Costs.GetUserSummary', 'admin', {'username': 'user-b'}, elevated=True + ) + + api.invoke(context) + + assert service.summary_calls == ['user-b'] + assert context.response_payload.username == 'user-b' + + +def test_get_user_summary_requires_a_username(): + api, _ = build_costs_api() + context = FakeApiInvocationContext( + 'Costs.GetUserSummary', 'admin', {}, elevated=True + ) + + with pytest.raises(exceptions.SocaException): + api.invoke(context) + + +def test_costs_api_rejects_an_unknown_namespace(): + api, _ = build_costs_api() + context = FakeApiInvocationContext('Costs.Whatever', 'admin', elevated=True) + + with pytest.raises(exceptions.SocaException): + api.invoke(context) + + +# self scoped api + + +def test_my_costs_serves_the_caller_and_ignores_any_username_in_the_payload(): + api, service = build_my_costs_api() + # a caller-supplied username must not change whose costs come back. the request + # model has no such field, so this pins the dispatch itself. + context = FakeApiInvocationContext( + 'MyCosts.GetSummary', 'user-a', {'username': 'user-b'}, elevated=False + ) + + api.invoke(context) + + assert service.summary_calls == ['user-a'] + assert context.response_payload.username == 'user-a' + + +def test_my_costs_rejects_the_admin_namespaces(): + api, service = build_my_costs_api() + # the self scoped class must not answer for the admin namespace even if the + # invoker ever routed one to it by mistake. + for namespace in ('Costs.ListUserCosts', 'Costs.GetUserSummary'): + context = FakeApiInvocationContext(namespace, 'user-a', elevated=True) + with pytest.raises(exceptions.SocaException): + api.invoke(context) + + assert service.summary_calls == [] + + +def test_my_costs_rejects_a_user_who_is_not_in_the_module_group(): + api, service = build_my_costs_api() + # authenticated but no longer authorized: taking someone out of the users group + # has to take the page with it + context = FakeApiInvocationContext( + 'MyCosts.GetSummary', 'user-a', elevated=False, authorized_user=False + ) + + with pytest.raises(exceptions.SocaException): + api.invoke(context) + + assert service.summary_calls == [] diff --git a/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_my_costs.py b/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_my_costs.py new file mode 100644 index 00000000..feaf593d --- /dev/null +++ b/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_my_costs.py @@ -0,0 +1,1297 @@ +""" +Test Cases for the per user cost summary + +The service runs against fakes for the projects service, the opensearch client and the +pricing helper. The bedrock read path is the real one, so the usage rows are filtered by +username by the same code the projects page uses. +""" + +from ideaclustermanager.app.costs.my_costs_service import MyCostsService +from ideaclustermanager.app.projects.bedrock_usage_service import ( + get_project_window_usage, + usage_window_dates, +) +from ideaclustermanager.app.projects.db.bedrock_usage_dao import build_day_key + +from ideadatamodel import ( + exceptions, + locale, + ListProjectsResult, + Project, + ProjectBedrockConfig, + SocaAmount, +) + +import arrow +import pytest + +USER = 'user-a' +OTHER_USER = 'user-b' +PROJECT_ID = 'a1b2c3d4-0000-4000-8000-000000000001' +PROJECT_NAME = 'project-a' +MODEL_A = 'vendor-a.model-1' +MODEL_B = 'vendor-b.model-9' + + +@pytest.fixture(autouse=True) +def initialized_locale(): + # SocaAmount reads the currency code the app context normally initializes. + try: + locale.get_currency_code() + except Exception: + locale.init('C') + + +class FakeUsageDAO: + """day rows keyed the way the real dao keys them, queried by date range.""" + + def __init__(self): + self.rows = [] + self.table = 'fake-usage-table' + # the self scoped path must read the window once per project, not twice + self.query_calls = 0 + + def add(self, username, model_id, invocations, input_tokens, output_tokens, day): + self.rows.append( + { + 'project_id': PROJECT_ID, + 'usage_id': build_day_key(day, username, model_id), + 'usage_date': day, + 'username': username, + 'model_id': model_id, + 'invocations': invocations, + 'input_tokens': input_tokens, + 'output_tokens': output_tokens, + 'updated_on': 0, + } + ) + + def query_day_rows(self, project_id, start_date, end_date): + self.query_calls += 1 + return [ + row + for row in self.rows + if row['project_id'] == project_id + and start_date <= row['usage_date'] <= end_date + ] + + +class FakeProjectsDAO: + def __init__(self, projects): + self.projects = projects + + def list_projects(self, request): + return ListProjectsResult(listing=self.projects, paginator=None) + + +class FakeBedrockProvisioner: + @staticmethod + def is_enabled(): + return True + + +class FakeProjectsService: + """ + stands in for the projects service. get_project_bedrock_usage delegates to the real + window read so the username filter under test is the shipped one. + """ + + def __init__(self, usage_dao, projects, spend=None): + self.usage_dao = usage_dao + # the self scoped path reads the dao itself so one page serves both views + self.bedrock_usage_dao = usage_dao + self.bedrock_provisioner = FakeBedrockProvisioner() + self.projects = projects + self.spend = spend + self.projects_dao = FakeProjectsDAO(projects) + + def apply_bedrock_spend(self, usage, project_name): + if self.spend is not None: + usage.spend = SocaAmount(amount=self.spend) + else: + usage.spend_is_unavailable = True + + def get_user_projects(self, request): + assert request.username == USER + return type('Result', (), {'projects': self.projects})() + + def get_project_bedrock_usage(self, project_id, username=None, project_name=None): + start_date, end_date = usage_window_dates() + usage = get_project_window_usage( + self.usage_dao, project_id, start_date, end_date, username=username + ) + if usage is None or username is not None: + return usage + if self.spend is not None: + usage.spend = SocaAmount(amount=self.spend) + else: + usage.spend_is_unavailable = True + return usage + + +class FakeConfig: + def __init__(self, values, scheduler_enabled=True): + self.values = dict(values) + self.scheduler_enabled = scheduler_enabled + + def get_string(self, key, default=None, required=False, module_id=None): + return self.values.get(key, default) + + def is_module_enabled(self, module_name): + return self.scheduler_enabled + + def get_module_id(self, module_name): + return module_name + + +class FakeOsClient: + def __init__(self, responses): + self.responses = responses + self.queries = [] + + def search(self, index, body): + self.queries.append((index, body)) + for prefix, response in self.responses.items(): + if index.startswith(prefix) or prefix in index: + if isinstance(response, Exception): + raise response + return response + return {'hits': {'total': {'value': 0}, 'hits': []}, 'aggregations': {}} + + +class FakeAnalyticsService: + def __init__(self, os_client): + self.os_client = type('Wrapper', (), {'os_client': os_client})() + + +class FakeAwsUtil: + def __init__(self, prices): + self.prices = prices + + def get_ec2_instance_type_unit_price(self, instance_type): + # the shipped helper reports an unanswered lookup as 0.0, not as an error. + return type('Price', (), {'ondemand': self.prices.get(instance_type, 0.0)})() + + +class FakeLogger: + def warning(self, *_args, **_kwargs): + pass + + def info(self, *_args, **_kwargs): + pass + + +class FakeHistoryTable: + def __init__(self, rows): + self.rows = rows + + def query(self, **kwargs): + return {'Items': list(self.rows)} + + def scan(self, **kwargs): + return {'Items': list(self.rows)} + + +class FakeDynamoDB: + def __init__(self, rows): + self.rows = rows + + def Table(self, _name): + return FakeHistoryTable(self.rows) + + +class FakeAws: + def __init__(self, rows): + self._rows = rows + + def dynamodb_table(self): + return FakeDynamoDB(self._rows) + + +class FakeContext: + def __init__( + self, projects, config, analytics_service, aws_util, history_rows=None + ): + self.projects = projects + self._config = config + self._analytics_service = analytics_service + self._aws_util = aws_util + self._aws = FakeAws(history_rows if history_rows is not None else []) + + def aws(self): + return self._aws + + def config(self): + return self._config + + def analytics_service(self): + return self._analytics_service + + def aws_util(self): + return self._aws_util + + def cluster_name(self): + return 'idea-test' + + def module_id(self): + return 'cluster-manager' + + def logger(self, _name=None): + return FakeLogger() + + +def build_project(): + return Project( + project_id=PROJECT_ID, + name=PROJECT_NAME, + title='Project A', + bedrock=ProjectBedrockConfig(), + ) + + +def build_service( + usage_dao=None, + projects=None, + spend=None, + os_responses=None, + prices=None, + config_values=None, + scheduler_enabled=True, + history_rows=None, +): + usage_dao = usage_dao if usage_dao is not None else FakeUsageDAO() + projects = projects if projects is not None else [build_project()] + os_client = FakeOsClient(os_responses or {}) + context = FakeContext( + projects=FakeProjectsService(usage_dao, projects, spend=spend), + config=FakeConfig( + config_values + if config_values is not None + else { + 'virtual-desktop-controller.opensearch.dcv_session.alias': 'idea-test_vdc_user_sessions' + }, + scheduler_enabled=scheduler_enabled, + ), + analytics_service=FakeAnalyticsService(os_client), + aws_util=FakeAwsUtil(prices or {}), + history_rows=history_rows, + ) + return MyCostsService(context), os_client + + +def today(): + return arrow.utcnow().format('YYYY-MM-DD') + + +# ai + + +def test_ai_sums_only_the_callers_rows(): + usage_dao = FakeUsageDAO() + usage_dao.add(USER, MODEL_A, 2, 100, 50, today()) + usage_dao.add(OTHER_USER, MODEL_A, 8, 700, 350, today()) + + service, _ = build_service(usage_dao=usage_dao, spend=120.0) + ai = service.get_summary(USER).ai + + assert len(ai.projects) == 1 + row = ai.projects[0] + assert row.total_tokens == 150 + assert row.invocations == 2 + assert ai.total_tokens == 150 + # the other user's 1050 tokens are in the project total but not in the caller's + assert row.cost == pytest.approx(120.0 * 150 / 1200) + assert row.estimated is True + assert ai.estimated is True + + +def test_ai_apportions_per_model_within_the_project(): + usage_dao = FakeUsageDAO() + usage_dao.add(USER, MODEL_A, 1, 100, 100, today()) + usage_dao.add(USER, MODEL_B, 1, 50, 50, today()) + + service, _ = build_service(usage_dao=usage_dao, spend=60.0) + row = service.get_summary(USER).ai.projects[0] + + by_model = {model.model_id: model for model in row.by_model} + assert by_model[MODEL_A].total_tokens == 200 + assert by_model[MODEL_B].total_tokens == 100 + assert by_model[MODEL_A].cost == pytest.approx(40.0) + assert by_model[MODEL_B].cost == pytest.approx(20.0) + assert row.cost == pytest.approx(60.0) + + +def test_ai_marks_cost_unavailable_rather_than_zero_when_spend_is_missing(): + usage_dao = FakeUsageDAO() + usage_dao.add(USER, MODEL_A, 1, 10, 10, today()) + + service, _ = build_service(usage_dao=usage_dao, spend=None) + row = service.get_summary(USER).ai.projects[0] + + assert row.cost_unavailable is True + assert row.total_tokens == 20 + assert row.by_model[0].cost is None + + +def test_ai_skips_projects_the_caller_never_used(): + usage_dao = FakeUsageDAO() + usage_dao.add(OTHER_USER, MODEL_A, 5, 500, 500, today()) + + service, _ = build_service(usage_dao=usage_dao, spend=10.0) + ai = service.get_summary(USER).ai + + assert ai.projects == [] + assert ai.total_tokens == 0 + + +# jobs + + +JOBS_RESPONSE = { + 'hits': { + 'total': {'value': 3}, + 'hits': [ + { + '_source': { + 'job_id': '101', + 'name': 'solve', + 'queue': 'normal', + 'project': PROJECT_NAME, + 'end_time': '2026-08-30T10:00:00+00:00', + 'estimated_bom_cost': {'total': {'amount': 4.5, 'unit': 'USD'}}, + } + } + ], + }, + 'aggregations': { + 'cost': {'value': 12.75}, + 'by_project': { + 'buckets': [{'key': PROJECT_NAME, 'doc_count': 3, 'cost': {'value': 12.75}}] + }, + 'by_queue': { + 'buckets': [ + {'key': 'normal', 'doc_count': 2, 'cost': {'value': 9.0}}, + {'key': 'high', 'doc_count': 1, 'cost': {'value': 3.75}}, + ] + }, + }, +} + + +def test_jobs_reports_totals_breakdowns_and_recent_jobs(): + service, os_client = build_service(os_responses={'_scheduler_jobs': JOBS_RESPONSE}) + jobs = service.get_summary(USER).jobs + + assert jobs.job_count == 3 + assert jobs.cost == pytest.approx(12.75) + assert jobs.estimated is True + assert [group.name for group in jobs.by_queue] == ['normal', 'high'] + assert jobs.by_project[0].job_count == 3 + assert jobs.recent_jobs[0].job_id == '101' + assert jobs.recent_jobs[0].cost == pytest.approx(4.5) + + +def test_jobs_query_is_scoped_to_the_caller(): + service, os_client = build_service(os_responses={'_scheduler_jobs': JOBS_RESPONSE}) + service.get_summary(USER) + + index, body = next( + query for query in os_client.queries if query[0].endswith('_jobs') + ) + assert index == 'idea-test_scheduler_jobs' + filters = body['query']['bool']['filter'] + assert {'term': {'owner.raw': USER}} in filters + + +def test_jobs_unavailable_when_the_scheduler_is_not_deployed(): + service, _ = build_service(scheduler_enabled=False) + assert service.get_summary(USER).jobs.is_unavailable is True + + +def test_jobs_unavailable_when_the_search_fails(): + service, _ = build_service( + os_responses={'_scheduler_jobs': RuntimeError('index_not_found_exception')} + ) + assert service.get_summary(USER).jobs.is_unavailable is True + + +# desktops + + +def session_hit( + session_id, + state, + instance_type, + created_on, + updated_on, + stopped_on=None, + cleanup_warning_stop_time=None, +): + source = { + 'idea_session_id': session_id, + 'name': session_id, + 'base_os': 'amazonlinux2023', + 'state': state, + 'created_on': created_on, + 'updated_on': updated_on, + 'server': {'instance_type': instance_type}, + } + if stopped_on is not None: + source['stopped_on'] = stopped_on + if cleanup_warning_stop_time is not None: + source['cleanup_warning_stop_time'] = cleanup_warning_stop_time + return {'_source': source} + + +def test_desktops_prices_recorded_hours_at_the_ondemand_rate(): + now = arrow.utcnow() + created = now.shift(hours=-4).int_timestamp * 1000 + updated = now.shift(hours=-2).int_timestamp * 1000 + + service, _ = build_service( + os_responses={ + 'user_sessions': { + 'hits': { + 'hits': [ + session_hit('sess-1', 'STOPPED', 'm5.large', created, updated) + ] + } + } + }, + prices={'m5.large': 0.1}, + ) + desktops = service.get_summary(USER).desktops + + assert desktops.session_count == 1 + session = desktops.sessions[0] + assert session.hours == pytest.approx(2.0, abs=0.01) + assert session.cost == pytest.approx(0.2, abs=0.01) + assert session.estimated is True + assert session.price_unavailable is None + assert desktops.hours == pytest.approx(2.0, abs=0.01) + + +def test_desktops_running_session_is_measured_to_now(): + now = arrow.utcnow() + created = now.shift(hours=-3).int_timestamp * 1000 + updated = now.shift(hours=-3).int_timestamp * 1000 + + service, _ = build_service( + os_responses={ + 'user_sessions': { + 'hits': { + 'hits': [ + session_hit('sess-2', 'READY', 'm5.large', created, updated) + ] + } + } + }, + prices={'m5.large': 0.1}, + ) + session = service.get_summary(USER).desktops.sessions[0] + + assert session.hours == pytest.approx(3.0, abs=0.01) + + +def test_desktops_report_hours_without_cost_when_the_price_is_unavailable(): + now = arrow.utcnow() + created = now.shift(hours=-5).int_timestamp * 1000 + updated = now.shift(hours=-4).int_timestamp * 1000 + + service, _ = build_service( + os_responses={ + 'user_sessions': { + 'hits': { + 'hits': [ + session_hit('sess-3', 'STOPPED', 'zz.unknown', created, updated) + ] + } + } + }, + prices={}, + ) + desktops = service.get_summary(USER).desktops + session = desktops.sessions[0] + + assert session.price_unavailable is True + assert session.cost is None + assert session.hours == pytest.approx(1.0, abs=0.01) + assert desktops.cost == 0.0 + + +def test_desktops_query_is_scoped_to_the_caller(): + service, os_client = build_service( + os_responses={'user_sessions': {'hits': {'hits': []}}} + ) + service.get_summary(USER) + + index, body = next( + query for query in os_client.queries if 'user_sessions' in query[0] + ) + filters = body['query']['bool']['filter'] + assert {'term': {'owner.raw': USER}} in filters + + +def test_desktops_unavailable_when_the_alias_is_not_configured(): + service, _ = build_service(config_values={}) + assert service.get_summary(USER).desktops.is_unavailable is True + + +# summary + + +def test_summary_reports_the_trailing_window_and_the_caller(): + service, _ = build_service() + result = service.get_summary(USER) + + start_date, end_date = usage_window_dates() + assert result.username == USER + assert result.window == 'last_30_days' + assert result.start_date == start_date + assert result.end_date == end_date + assert result.ai is not None + assert result.jobs is not None + assert result.desktops is not None + + +# unpriced rules + + +def test_desktops_counts_sessions_that_could_not_be_priced(): + now = arrow.utcnow() + created = now.shift(hours=-4).int_timestamp * 1000 + updated = now.shift(hours=-2).int_timestamp * 1000 + + service, _ = build_service( + os_responses={ + 'user_sessions': { + 'hits': { + 'hits': [ + session_hit('sess-1', 'STOPPED', 'm5.large', created, updated), + session_hit( + 'sess-2', 'STOPPED', 'zz.unknown', created, updated + ), + ] + } + } + }, + prices={'m5.large': 0.1}, + ) + desktops = service.get_summary(USER).desktops + + assert desktops.session_count == 2 + assert desktops.unpriced_sessions == 1 + # the subtotal covers only the session that had a price + assert desktops.cost == pytest.approx(0.2, abs=0.01) + + +def test_desktops_report_every_session_unpriced_when_no_price_is_known(): + now = arrow.utcnow() + created = now.shift(hours=-4).int_timestamp * 1000 + updated = now.shift(hours=-2).int_timestamp * 1000 + + service, _ = build_service( + os_responses={ + 'user_sessions': { + 'hits': { + 'hits': [ + session_hit( + 'sess-1', 'STOPPED', 'm6g.xlarge', created, updated + ), + session_hit( + 'sess-2', 'STOPPED', 'g4dn.2xlarge', created, updated + ), + ] + } + } + }, + prices={}, + ) + desktops = service.get_summary(USER).desktops + + # real hours with no prices: the caller must be able to tell that from a genuine + # zero. + assert desktops.session_count == 2 + assert desktops.unpriced_sessions == 2 + assert desktops.hours > 0 + + +def test_jobs_counts_jobs_the_scheduler_never_priced(): + response = dict(JOBS_RESPONSE) + response['aggregations'] = dict(JOBS_RESPONSE['aggregations']) + response['aggregations']['priced'] = {'value': 2} + + service, _ = build_service(os_responses={'_scheduler_jobs': response}) + jobs = service.get_summary(USER).jobs + + assert jobs.job_count == 3 + assert jobs.unpriced_jobs == 1 + + +def test_a_job_whose_instance_hours_could_not_be_priced_is_not_reported_as_free(): + """ + the scheduler still records an estimate, it is just missing the compute. the amount + it carries has to read as unavailable rather than as what the job cost. + """ + response = dict(JOBS_RESPONSE) + response['hits'] = { + 'total': {'value': 1}, + 'hits': [ + { + '_source': { + 'job_id': '102', + 'name': 'solve', + 'queue': 'normal', + 'project': PROJECT_NAME, + 'end_time': '2026-08-30T10:00:00+00:00', + 'estimated_bom_cost': { + 'total': {'amount': 0.8, 'unit': 'USD'}, + 'price_unavailable': True, + }, + } + } + ], + } + response['aggregations'] = dict(JOBS_RESPONSE['aggregations']) + response['aggregations']['unavailable'] = {'doc_count': 1} + + service, _ = build_service(os_responses={'_scheduler_jobs': response}) + jobs = service.get_summary(USER).jobs + + assert jobs.recent_jobs[0].cost_unavailable is True + assert jobs.cost_unavailable is True + + +def test_a_job_that_really_cost_nothing_still_reads_as_zero(): + """ + a job cancelled or failed before it ran was priced normally. unavailability is read + from the flag the scheduler wrote, never inferred from a zero amount. + """ + response = dict(JOBS_RESPONSE) + response['hits'] = { + 'total': {'value': 1}, + 'hits': [ + { + '_source': { + 'job_id': '103', + 'name': 'cancelled', + 'queue': 'normal', + 'project': PROJECT_NAME, + 'end_time': '2026-08-30T10:00:00+00:00', + 'estimated_bom_cost': {'total': {'amount': 0.0, 'unit': 'USD'}}, + } + } + ], + } + + service, _ = build_service(os_responses={'_scheduler_jobs': response}) + jobs = service.get_summary(USER).jobs + + assert jobs.recent_jobs[0].cost == 0.0 + assert jobs.recent_jobs[0].cost_unavailable is False + assert jobs.cost_unavailable is False + + +def test_the_jobs_query_counts_the_estimates_that_are_missing_their_compute(): + """ + such a job does carry an amount, so the existing priced count finds nothing wrong + with it and only this aggregation can. + """ + service, os_client = build_service(os_responses={'_scheduler_jobs': JOBS_RESPONSE}) + service.get_summary(USER) + + _, body = next(query for query in os_client.queries if query[0].endswith('_jobs')) + assert body['aggs']['unavailable'] == { + 'filter': {'term': {'estimated_bom_cost.price_unavailable': True}} + } + + +# admin listing + + +def test_the_admin_listing_marks_a_user_whose_job_cost_is_incomplete(): + jobs = { + 'hits': {'total': {'value': 0}, 'hits': []}, + 'aggregations': { + 'by_user': { + 'buckets': [ + { + 'key': USER, + 'doc_count': 2, + 'cost': {'value': 1.5}, + 'priced': {'value': 2}, + 'unavailable': {'doc_count': 1}, + }, + { + 'key': OTHER_USER, + 'doc_count': 2, + 'cost': {'value': 9.0}, + 'priced': {'value': 2}, + 'unavailable': {'doc_count': 0}, + }, + ] + } + }, + } + + service, _ = build_service(os_responses={'_scheduler_jobs': jobs}, config_values={}) + rows = {row.username: row for row in service.list_user_costs().listing} + + assert rows[USER].job_cost_unavailable is True + # every job of theirs was priced, so their total is the whole story + assert rows[OTHER_USER].job_cost_unavailable is None + assert rows[OTHER_USER].job_cost == pytest.approx(9.0) + + +def test_list_user_costs_aggregates_across_the_three_sources(): + usage_dao = FakeUsageDAO() + usage_dao.add(USER, MODEL_A, 2, 100, 50, today()) + usage_dao.add(OTHER_USER, MODEL_A, 8, 700, 350, today()) + + now = arrow.utcnow() + created = now.shift(hours=-4).int_timestamp * 1000 + updated = now.shift(hours=-2).int_timestamp * 1000 + sessions = { + 'hits': { + 'hits': [ + dict( + session_hit('sess-1', 'STOPPED', 'm5.large', created, updated), + _source=dict( + session_hit('sess-1', 'STOPPED', 'm5.large', created, updated)[ + '_source' + ], + owner=USER, + ), + ) + ] + } + } + jobs = { + 'hits': {'total': {'value': 0}, 'hits': []}, + 'aggregations': { + 'by_user': { + 'buckets': [ + { + 'key': OTHER_USER, + 'doc_count': 4, + 'cost': {'value': 10.0}, + 'priced': {'value': 4}, + } + ] + } + }, + } + + service, _ = build_service( + usage_dao=usage_dao, + spend=120.0, + os_responses={'_scheduler_jobs': jobs, 'user_sessions': sessions}, + prices={'m5.large': 0.1}, + ) + result = service.list_user_costs() + rows = {row.username: row for row in result.listing} + + # both users appear: one from bedrock and desktops, one from bedrock and jobs + assert set(rows) == {USER, OTHER_USER} + assert rows[USER].ai_tokens == 150 + assert rows[USER].desktop_hours == pytest.approx(2.0, abs=0.01) + assert rows[USER].desktop_cost == pytest.approx(0.2, abs=0.01) + assert rows[USER].job_count == 0 + assert rows[OTHER_USER].ai_tokens == 1050 + assert rows[OTHER_USER].job_count == 4 + assert rows[OTHER_USER].job_cost == pytest.approx(10.0) + # the ai split follows token share of the project spend + assert rows[USER].ai_cost == pytest.approx(120.0 * 150 / 1200) + assert rows[OTHER_USER].ai_cost == pytest.approx(120.0 * 1050 / 1200) + # total is the sum of the three, and the listing leads with the biggest spender + assert rows[OTHER_USER].total_cost == pytest.approx( + rows[OTHER_USER].ai_cost + rows[OTHER_USER].job_cost + ) + assert result.listing[0].username == OTHER_USER + + +def test_list_user_costs_reports_a_source_it_could_not_read(): + usage_dao = FakeUsageDAO() + usage_dao.add(USER, MODEL_A, 1, 10, 10, today()) + + service, _ = build_service( + usage_dao=usage_dao, + spend=10.0, + os_responses={'_scheduler_jobs': RuntimeError('index_not_found_exception')}, + config_values={}, + ) + result = service.list_user_costs() + + # a read that failed is flagged, never silently rendered as nobody spending + assert result.jobs_unavailable is True + assert result.desktops_unavailable is True + assert result.ai_unavailable is False + assert result.listing[0].username == USER + + +# username guard + + +def test_get_summary_refuses_a_missing_username(): + service, _ = build_service() + + # get_username() is Optional, and an empty owner filter would read as every user. + for missing in (None, ''): + with pytest.raises(exceptions.SocaException): + service.get_summary(missing) + + +def test_desktop_hits_refuses_a_missing_username_unless_all_users_is_asked_for(): + service, os_client = build_service( + os_responses={'user_sessions': {'hits': {'hits': []}}} + ) + + with pytest.raises(exceptions.SocaException): + service._desktop_hits(None, 0, 1) + + # the admin listing gets the same query by saying so explicitly + service._desktop_hits(None, 0, 1, all_users=True) + _, body = next(query for query in os_client.queries if 'user_sessions' in query[0]) + assert not any('term' in entry for entry in body['query']['bool']['filter']) + + +# one scoped read per project + + +def test_ai_reads_the_window_once_per_project(): + usage_dao = FakeUsageDAO() + usage_dao.add(USER, MODEL_A, 2, 100, 50, today()) + usage_dao.add(OTHER_USER, MODEL_A, 8, 700, 350, today()) + + service, _ = build_service(usage_dao=usage_dao, spend=120.0) + service.get_summary(USER) + + # one project in play: the caller's totals and the project total come off the + # same page, so the apportionment costs no extra read + assert usage_dao.query_calls == 1 + + +def test_ai_response_never_carries_another_username(): + usage_dao = FakeUsageDAO() + usage_dao.add(USER, MODEL_A, 2, 100, 50, today()) + usage_dao.add(OTHER_USER, MODEL_A, 8, 700, 350, today()) + + service, _ = build_service(usage_dao=usage_dao, spend=120.0) + result = service.get_summary(USER) + + # the apportionment needs the project total, not a per user split. nobody else's + # name may reach the caller. + assert OTHER_USER not in result.model_dump_json() + assert result.ai.projects[0].total_tokens == 150 + assert result.ai.projects[0].cost == pytest.approx(120.0 * 150 / 1200) + + +# stop time + + +def test_desktop_bills_to_the_recorded_stop_time_not_the_last_write(): + now = arrow.utcnow() + created = now.shift(days=-26).int_timestamp * 1000 + stopped = now.shift(days=-25).int_timestamp * 1000 + # renamed on day 25: updated_on moved, the desktop did not start costing again + touched = now.shift(days=-1).int_timestamp * 1000 + + service, _ = build_service( + os_responses={ + 'user_sessions': { + 'hits': { + 'hits': [ + session_hit( + 'sess-1', + 'STOPPED', + 'm5.large', + created, + touched, + stopped_on=stopped, + ) + ] + } + } + }, + prices={'m5.large': 0.1}, + ) + session = service.get_summary(USER).desktops.sessions[0] + + # one day of running, not the twenty four days of sitting stopped + assert session.hours == pytest.approx(24.0, abs=0.1) + assert session.stop_time_estimated is None + assert session.cost == pytest.approx(2.4, abs=0.05) + + +def test_live_desktop_still_bills_to_now(): + now = arrow.utcnow() + created = now.shift(hours=-3).int_timestamp * 1000 + + service, _ = build_service( + os_responses={ + 'user_sessions': { + 'hits': { + 'hits': [ + session_hit('sess-2', 'READY', 'm5.large', created, created) + ] + } + } + }, + prices={'m5.large': 0.1}, + ) + session = service.get_summary(USER).desktops.sessions[0] + + # a running desktop has no stop time and must not be cut short by one + assert session.hours == pytest.approx(3.0, abs=0.01) + assert session.stop_time_estimated is None + + +def test_legacy_desktop_falls_back_to_the_last_write_and_says_so(): + now = arrow.utcnow() + created = now.shift(hours=-5).int_timestamp * 1000 + touched = now.shift(hours=-4).int_timestamp * 1000 + + service, _ = build_service( + os_responses={ + 'user_sessions': { + 'hits': { + 'hits': [ + session_hit('sess-3', 'STOPPED', 'm5.large', created, touched) + ] + } + } + }, + prices={'m5.large': 0.1}, + ) + session = service.get_summary(USER).desktops.sessions[0] + + # stopped before the stop time was recorded: still an upper bound, and flagged + assert session.hours == pytest.approx(1.0, abs=0.01) + assert session.stop_time_estimated is True + + +def test_legacy_desktop_prefers_the_cleanup_notice_stop_time_to_the_last_write(): + now = arrow.utcnow() + created = now.shift(days=-10).int_timestamp * 1000 + real_stop = now.shift(days=-9).int_timestamp * 1000 + touched = now.shift(days=-1).int_timestamp * 1000 + + service, _ = build_service( + os_responses={ + 'user_sessions': { + 'hits': { + 'hits': [ + session_hit( + 'sess-4', + 'STOPPED', + 'm5.large', + created, + touched, + cleanup_warning_stop_time=real_stop, + ) + ] + } + } + }, + prices={'m5.large': 0.1}, + ) + session = service.get_summary(USER).desktops.sessions[0] + + # the cleanup notice carries a real ec2 stop time, which beats the last write + assert session.hours == pytest.approx(24.0, abs=0.1) + assert session.stop_time_estimated is True + + +# terminated desktops + + +def test_terminated_desktop_inside_the_window_is_billed_to_its_stop_time(): + now = arrow.utcnow() + created = now.shift(days=-10).int_timestamp * 1000 + stopped = now.shift(days=-9).int_timestamp * 1000 + # the deletion wrote the record again, days after the desktop stopped costing + deleted = now.shift(days=-2).int_timestamp * 1000 + + service, _ = build_service( + os_responses={ + 'user_sessions': { + 'hits': { + 'hits': [ + session_hit( + 'sess-1', + 'DELETED', + 'm5.large', + created, + deleted, + stopped_on=stopped, + ) + ] + } + } + }, + prices={'m5.large': 0.1}, + ) + desktops = service.get_summary(USER).desktops + session = desktops.sessions[0] + + # a desktop that ran and was later deleted still cost what it cost while running + assert desktops.session_count == 1 + assert session.hours == pytest.approx(24.0, abs=0.1) + assert session.cost == pytest.approx(2.4, abs=0.05) + assert session.stop_time_estimated is None + # the controller's word is DELETED; what a cost reader wants is Terminated + assert session.state == 'Terminated' + + +def test_a_desktop_terminated_from_running_bills_to_the_deletion(): + now = arrow.utcnow() + created = now.shift(hours=-6).int_timestamp * 1000 + deleted = now.shift(hours=-4).int_timestamp * 1000 + + service, _ = build_service( + os_responses={ + 'user_sessions': { + 'hits': { + 'hits': [ + session_hit('sess-2', 'DELETED', 'm5.large', created, deleted) + ] + } + } + }, + prices={'m5.large': 0.1}, + ) + session = service.get_summary(USER).desktops.sessions[0] + + # no recorded stop time on an older row: the deletion write is the best answer + # there is, and it is reported as an estimate rather than as a measurement + assert session.hours == pytest.approx(2.0, abs=0.01) + assert session.stop_time_estimated is True + assert session.state == 'Terminated' + + +def test_a_desktop_deleted_before_the_window_is_excluded(): + now = arrow.utcnow() + created = now.shift(days=-60).int_timestamp * 1000 + deleted = now.shift(days=-45).int_timestamp * 1000 + + service, os_client = build_service( + os_responses={'user_sessions': {'hits': {'hits': []}}}, + prices={'m5.large': 0.1}, + ) + service.get_summary(USER) + + # the query itself excludes it: nothing that stopped before the window starts is + # asked for, so a long dead desktop never reaches the page + _, body = next(query for query in os_client.queries if 'user_sessions' in query[0]) + ranges = [entry for entry in body['query']['bool']['filter'] if 'range' in entry] + updated_lower_bound = next( + entry['range']['updated_on']['gte'] + for entry in ranges + if 'updated_on' in entry['range'] + ) + assert deleted < updated_lower_bound + assert created < updated_lower_bound + + +def test_a_terminating_desktop_reads_as_terminated_too(): + now = arrow.utcnow() + created = now.shift(hours=-3).int_timestamp * 1000 + deleting = now.shift(hours=-1).int_timestamp * 1000 + + service, _ = build_service( + os_responses={ + 'user_sessions': { + 'hits': { + 'hits': [ + session_hit('sess-3', 'DELETING', 'm5.large', created, deleting) + ] + } + } + }, + prices={'m5.large': 0.1}, + ) + session = service.get_summary(USER).desktops.sessions[0] + + assert session.state == 'Terminated' + assert session.hours == pytest.approx(2.0, abs=0.01) + + +def test_terminated_desktops_count_toward_the_admin_listing(): + now = arrow.utcnow() + created = now.shift(hours=-5).int_timestamp * 1000 + stopped = now.shift(hours=-3).int_timestamp * 1000 + hit = session_hit( + 'sess-4', 'DELETED', 'm5.large', created, created, stopped_on=stopped + ) + hit['_source']['owner'] = USER + + service, _ = build_service( + os_responses={ + 'user_sessions': {'hits': {'hits': [hit]}}, + '_scheduler_jobs': { + 'hits': {'total': {'value': 0}, 'hits': []}, + 'aggregations': {'by_user': {'buckets': []}}, + }, + }, + prices={'m5.large': 0.1}, + ) + rows = {row.username: row for row in service.list_user_costs().listing} + + # the admin view reads the same sessions, so a terminated desktop counts there too + assert rows[USER].desktop_session_count == 1 + assert rows[USER].desktop_hours == pytest.approx(2.0, abs=0.01) + + +def test_a_session_indexed_under_two_generations_is_counted_once(): + """ + the controller writes and deletes under a versioned index name while this reads the + alias, which spans every generation of it. a reindexed desktop has a document under + more than one, and billing both would double its hours. + """ + now = arrow.utcnow() + created = now.shift(hours=-4).int_timestamp * 1000 + stopped = now.shift(hours=-2).int_timestamp * 1000 + + stale = session_hit('sess-1', 'STOPPED', 'm5.large', created, created) + current = session_hit( + 'sess-1', 'STOPPED', 'm5.large', created, stopped, stopped_on=stopped + ) + + service, _ = build_service( + os_responses={'user_sessions': {'hits': {'hits': [stale, current]}}}, + prices={'m5.large': 0.1}, + ) + desktops = service.get_summary(USER).desktops + + assert desktops.session_count == 1 + # the newer document wins, so the recorded stop time is the one that counts + assert desktops.sessions[0].hours == pytest.approx(2.0, abs=0.01) + assert desktops.sessions[0].stop_time_estimated is None + + +# terminated desktops kept as history + + +def history_row( + session_id, owner, created_on, stopped_on, deleted_on, instance_type='m5.large' +): + return { + 'owner': owner, + 'idea_session_id': session_id, + 'name': session_id, + 'base_os': 'amazonlinux2023', + 'instance_type': instance_type, + 'project_id': 'project-1', + 'created_on': created_on, + 'stopped_on': stopped_on, + 'deleted_on': deleted_on, + } + + +def test_a_terminated_desktop_is_costed_from_its_history_record(): + now = arrow.utcnow() + created = now.shift(hours=-6).int_timestamp * 1000 + stopped = now.shift(hours=-4).int_timestamp * 1000 + deleted = now.shift(hours=-1).int_timestamp * 1000 + + service, _ = build_service( + os_responses={'user_sessions': {'hits': {'hits': []}}}, + prices={'m5.large': 0.1}, + history_rows=[history_row('sess-1', USER, created, stopped, deleted)], + ) + desktops = service.get_summary(USER).desktops + + # the session row and its search document are both long gone + assert desktops.session_count == 1 + session = desktops.sessions[0] + assert session.state == 'Terminated' + assert session.hours == pytest.approx(2.0, abs=0.01) + assert session.cost == pytest.approx(0.2, abs=0.01) + assert session.stop_time_estimated is None + + +def test_a_desktop_deleted_before_the_window_is_left_out_of_history(): + now = arrow.utcnow() + created = now.shift(days=-90).int_timestamp * 1000 + stopped = now.shift(days=-89).int_timestamp * 1000 + deleted = now.shift(days=-88).int_timestamp * 1000 + + service, _ = build_service( + os_responses={'user_sessions': {'hits': {'hits': []}}}, + prices={'m5.large': 0.1}, + history_rows=[history_row('sess-old', USER, created, stopped, deleted)], + ) + desktops = service.get_summary(USER).desktops + + # the retention window is far longer than the costing window, so old rows are read + # and then dropped rather than billed + assert desktops.session_count == 0 + + +def test_history_wins_over_a_search_document_that_was_never_removed(): + now = arrow.utcnow() + created = now.shift(hours=-8).int_timestamp * 1000 + stopped = now.shift(hours=-6).int_timestamp * 1000 + deleted = now.shift(hours=-5).int_timestamp * 1000 + + # the leaked document says the desktop is still stopped and was written later + leaked = session_hit('sess-1', 'STOPPED', 'm5.large', created, deleted) + + service, _ = build_service( + os_responses={'user_sessions': {'hits': {'hits': [leaked]}}}, + prices={'m5.large': 0.1}, + history_rows=[history_row('sess-1', USER, created, stopped, deleted)], + ) + desktops = service.get_summary(USER).desktops + + # counted once, and the record written at the deletion is the one believed + assert desktops.session_count == 1 + assert desktops.sessions[0].state == 'Terminated' + assert desktops.sessions[0].hours == pytest.approx(2.0, abs=0.01) + + +def test_terminated_desktops_reach_the_administrator_listing(): + now = arrow.utcnow() + created = now.shift(hours=-5).int_timestamp * 1000 + stopped = now.shift(hours=-3).int_timestamp * 1000 + deleted = now.shift(hours=-2).int_timestamp * 1000 + + service, _ = build_service( + os_responses={ + 'user_sessions': {'hits': {'hits': []}}, + '_scheduler_jobs': { + 'hits': {'total': {'value': 0}, 'hits': []}, + 'aggregations': {'by_user': {'buckets': []}}, + }, + }, + prices={'m5.large': 0.1}, + history_rows=[history_row('sess-1', OTHER_USER, created, stopped, deleted)], + ) + rows = {row.username: row for row in service.list_user_costs().listing} + + assert rows[OTHER_USER].desktop_session_count == 1 + assert rows[OTHER_USER].desktop_hours == pytest.approx(2.0, abs=0.01) + + +def test_a_history_read_that_fails_leaves_the_live_desktops_alone(): + now = arrow.utcnow() + created = now.shift(hours=-3).int_timestamp * 1000 + + service, _ = build_service( + os_responses={ + 'user_sessions': { + 'hits': { + 'hits': [ + session_hit('sess-live', 'READY', 'm5.large', created, created) + ] + } + } + }, + prices={'m5.large': 0.1}, + ) + # the fake context has no history table wired for this case; the read fails and is + # swallowed, because history is additive and must not take the page down + desktops = service.get_summary(USER).desktops + + assert desktops.session_count == 1 + assert desktops.sessions[0].name == 'sess-live' diff --git a/source/idea/idea-cluster-manager/webapp/.env b/source/idea/idea-cluster-manager/webapp/.env index b5ae6b7a..8d82758f 100644 --- a/source/idea/idea-cluster-manager/webapp/.env +++ b/source/idea/idea-cluster-manager/webapp/.env @@ -1,4 +1,4 @@ REACT_APP_IDEA_HTTP_ENDPOINT="http://localhost:8080" REACT_APP_IDEA_ALB_ENDPOINT="http://localhost:8080" REACT_APP_IDEA_HTTP_API_SUFFIX="/api/v1" -REACT_APP_IDEA_RELEASE_VERSION="26.08.0" +REACT_APP_IDEA_RELEASE_VERSION="26.09.0" diff --git a/source/idea/idea-cluster-manager/webapp/package.json b/source/idea/idea-cluster-manager/webapp/package.json index 79eee1cf..6ae272ed 100644 --- a/source/idea/idea-cluster-manager/webapp/package.json +++ b/source/idea/idea-cluster-manager/webapp/package.json @@ -1,6 +1,6 @@ { "name": "web-portal", - "version": "26.08.0", + "version": "26.09.0", "private": true, "dependencies": { "@cloudscape-design/collection-hooks": "^1.0.0", diff --git a/source/idea/idea-cluster-manager/webapp/src/App.tsx b/source/idea/idea-cluster-manager/webapp/src/App.tsx index 7589fa9e..4809e497 100644 --- a/source/idea/idea-cluster-manager/webapp/src/App.tsx +++ b/source/idea/idea-cluster-manager/webapp/src/App.tsx @@ -35,14 +35,18 @@ import UpdateHpcApplication from "./pages/hpc/update-hpc-application"; import SubmitJob from "./pages/hpc/submit-job"; import AccountSettings from "./pages/account/account-settings"; import SSHAccess from "./pages/home/ssh-access"; +import MyCosts from "./pages/home/my-costs"; import CustomDashboard from "./pages/home/custom-dashboard"; import ClusterSettings from "./pages/cluster-admin/cluster-settings"; import ClusterStatus from "./pages/cluster-admin/cluster-status"; import Projects from "./pages/cluster-admin/projects"; +import AiUsage from "./pages/cluster-admin/ai-usage"; +import UserCostsPage from "./pages/cluster-admin/user-costs"; import {Box, HelpPanel, SideNavigationProps, StatusIndicator} from "@cloudscape-design/components"; import {NonCancelableCustomEvent} from "@cloudscape-design/components/internal/events"; import {FlashbarProps} from "@cloudscape-design/components/flashbar/interfaces"; import HpcLicenses from "./pages/hpc/hpc-licenses"; +import HpcCustomAmis from "./pages/hpc/hpc-custom-amis"; import EmailTemplates from "./pages/cluster-admin/email-templates"; import ReactMarkdown from "react-markdown"; import remarkGfm from "remark-gfm"; @@ -446,6 +450,22 @@ class IdeaWebPortalApp extends Component }/> + + + + }/> }/> + + + + }/> }/> + + + + }/> + + + + }/> void): IdeaApiInvoker { +function buildInvoker(postMessage: (message: any, transfer: any[]) => void, timeout?: number): IdeaApiInvoker { initTestAppData(); return new IdeaApiInvoker({ name: 'test-invoker', url: 'http://localhost:8080/api/v1', + timeout: timeout, serviceWorkerRegistration: { active: { postMessage: postMessage @@ -50,4 +51,23 @@ describe('idea api invoker', () => { const result: any = await invoker.invoke_alt('Scheduler.SubmitJob', { dry_run: false }); expect(result.job.job_id).toBe('101'); }); + + it('honors a caller-supplied timeout instead of the hardcoded default', async () => { + vi.useFakeTimers(); + const invoker = buildInvoker(() => {}, 5000); + + const request = invoker.invoke_alt('Scheduler.SubmitJob', { dry_run: false }); + const settled = request.then( + () => 'resolved', + (error) => error + ); + + await vi.advanceTimersByTimeAsync(4999); + expect(await Promise.race([settled, Promise.resolve('pending')])).toBe('pending'); + + await vi.advanceTimersByTimeAsync(2); + const error: any = await settled; + expect(error.errorCode).toBe('REQUEST_TIMEOUT'); + expect(error.message).toBe('Request timed-out'); + }); }); diff --git a/source/idea/idea-cluster-manager/webapp/src/client/idea-api-invoker.ts b/source/idea/idea-cluster-manager/webapp/src/client/idea-api-invoker.ts index aa259618..44a6401d 100644 --- a/source/idea/idea-cluster-manager/webapp/src/client/idea-api-invoker.ts +++ b/source/idea/idea-cluster-manager/webapp/src/client/idea-api-invoker.ts @@ -21,6 +21,7 @@ import {AUTH_TOKEN_EXPIRED, REQUEST_TIMEOUT} from "../common/error-codes"; // the service worker owns the network timeout, so this only bounds the case where it never // replies at all. it stays above that timeout so it cannot pre-empt a real answer. +// Used as the default when the invoker is constructed without an explicit timeout. const SERVICE_WORKER_REPLY_TIMEOUT = 660000 export interface IdeaHeader { @@ -86,7 +87,7 @@ export class IdeaApiInvoker { message: 'Request timed-out' } }) - }, SERVICE_WORKER_REPLY_TIMEOUT) + }, this.props.timeout ?? SERVICE_WORKER_REPLY_TIMEOUT) messageChannel.port1.onmessage = (event) => { clearTimeout(timeout) if(event.data.error) { diff --git a/source/idea/idea-cluster-manager/webapp/src/client/my-costs-client.ts b/source/idea/idea-cluster-manager/webapp/src/client/my-costs-client.ts new file mode 100644 index 00000000..4033bb7b --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/client/my-costs-client.ts @@ -0,0 +1,40 @@ +import { + GetMyCostsSummaryRequest, + GetMyCostsSummaryResult, + GetUserCostsSummaryRequest, + ListUserCostsRequest, + ListUserCostsResult +} from './data-model' +import IdeaBaseClient, {IdeaBaseClientProps} from "./base-client"; + +export interface MyCostsClientProps extends IdeaBaseClientProps { +} + +class MyCostsClient extends IdeaBaseClient { + + getSummary(req: GetMyCostsSummaryRequest): Promise { + return this.apiInvoker.invoke_alt( + 'MyCosts.GetSummary', + req + ) + } + + // Admin only, enforced by the server. One row per user with a measured cost in the window. + listUserCosts(req: ListUserCostsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Costs.ListUserCosts', + req + ) + } + + // Admin only, enforced by the server. The same summary as MyCosts.GetSummary, for the named user. + getUserSummary(req: GetUserCostsSummaryRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Costs.GetUserSummary', + req + ) + } + +} + +export default MyCostsClient diff --git a/source/idea/idea-cluster-manager/webapp/src/client/projects-client.ts b/source/idea/idea-cluster-manager/webapp/src/client/projects-client.ts index fd598a65..b6bcbee6 100644 --- a/source/idea/idea-cluster-manager/webapp/src/client/projects-client.ts +++ b/source/idea/idea-cluster-manager/webapp/src/client/projects-client.ts @@ -22,6 +22,8 @@ import { UpdateProjectResult, ListProjectsRequest, ListProjectsResult, + ListBedrockUsageRequest, + ListBedrockUsageResult, EnableProjectRequest, EnableProjectResult, DisableProjectRequest, @@ -71,6 +73,13 @@ class ProjectsClient extends IdeaBaseClient { ) } + listBedrockUsage(req: ListBedrockUsageRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Projects.ListBedrockUsage', + req + ) + } + getUserProjects(req: GetUserProjectsRequest): Promise { return this.apiInvoker.invoke_alt( 'Projects.GetUserProjects', diff --git a/source/idea/idea-cluster-manager/webapp/src/client/scheduler-admin-client.test.ts b/source/idea/idea-cluster-manager/webapp/src/client/scheduler-admin-client.test.ts new file mode 100644 index 00000000..a8466439 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/client/scheduler-admin-client.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it, vi } from 'vitest' +import SchedulerAdminClient from './scheduler-admin-client' + +describe('SchedulerAdminClient', () => { + it('listComputeImages invokes its namespace', async () => { + const invoke_alt = vi.fn().mockResolvedValue({ listing: [] }) + const client = Object.create(SchedulerAdminClient.prototype) as SchedulerAdminClient + ;(client as any).apiInvoker = { invoke_alt } + + const response = await client.listComputeImages({}) + + expect(invoke_alt).toHaveBeenCalledWith('SchedulerAdmin.ListComputeImages', {}) + expect(response.listing).toEqual([]) + }) + + it('buildComputeImage carries the request through', async () => { + const invoke_alt = vi.fn().mockResolvedValue({ record: { status: 'building' } }) + const client = Object.create(SchedulerAdminClient.prototype) as SchedulerAdminClient + ;(client as any).apiInvoker = { invoke_alt } + + const response = await client.buildComputeImage({ base_os: 'rocky9', enable_drivers: ['efa'] }) + + expect(invoke_alt).toHaveBeenCalledWith('SchedulerAdmin.BuildComputeImage', { base_os: 'rocky9', enable_drivers: ['efa'] }) + expect(response.record?.status).toEqual('building') + }) +}) diff --git a/source/idea/idea-cluster-manager/webapp/src/client/scheduler-admin-client.ts b/source/idea/idea-cluster-manager/webapp/src/client/scheduler-admin-client.ts index d43726d9..0c02cd09 100644 --- a/source/idea/idea-cluster-manager/webapp/src/client/scheduler-admin-client.ts +++ b/source/idea/idea-cluster-manager/webapp/src/client/scheduler-admin-client.ts @@ -61,7 +61,11 @@ import { CheckHpcLicenseResourceAvailabilityRequest, CheckHpcLicenseResourceAvailabilityResult, DeleteJobRequest, - DeleteJobResult + DeleteJobResult, + ListComputeImagesRequest, + ListComputeImagesResult, + BuildComputeImageRequest, + BuildComputeImageResult } from './data-model' import IdeaBaseClient, {IdeaBaseClientProps} from "./base-client"; @@ -253,6 +257,20 @@ class SchedulerAdminClient extends IdeaBaseClient { } + + listComputeImages(req: ListComputeImagesRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.ListComputeImages', + req + ) + } + + buildComputeImage(req: BuildComputeImageRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.BuildComputeImage', + req + ) + } } export default SchedulerAdminClient diff --git a/source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-admin-client.test.ts b/source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-admin-client.test.ts new file mode 100644 index 00000000..8fa56185 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-admin-client.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it, vi } from 'vitest' +import VirtualDesktopAdminClient from './virtual-desktop-admin-client' + +describe('VirtualDesktopAdminClient', () => { + it('refreshBaseSoftwareStackAmis invokes its namespace', async () => { + const invoke_alt = vi.fn().mockResolvedValue({ results: [] }) + const client = Object.create(VirtualDesktopAdminClient.prototype) as VirtualDesktopAdminClient + ;(client as any).apiInvoker = { invoke_alt } + + const response = await client.refreshBaseSoftwareStackAmis({}) + + expect(invoke_alt).toHaveBeenCalledWith('VirtualDesktopAdmin.RefreshBaseSoftwareStackAmis', {}) + expect(response.results).toEqual([]) + }) + + it('carries stack_ids through', async () => { + const invoke_alt = vi.fn().mockResolvedValue({ results: [] }) + const client = Object.create(VirtualDesktopAdminClient.prototype) as VirtualDesktopAdminClient + ;(client as any).apiInvoker = { invoke_alt } + + await client.refreshBaseSoftwareStackAmis({ stack_ids: ['ss-base-a', 'ss-base-b'] }) + + expect(invoke_alt).toHaveBeenCalledWith('VirtualDesktopAdmin.RefreshBaseSoftwareStackAmis', { stack_ids: ['ss-base-a', 'ss-base-b'] }) + }) + + it('listDesktopImages and buildDesktopImage invoke their namespaces', async () => { + const invoke_alt = vi.fn().mockResolvedValue({ listing: [], record: { status: 'building' } }) + const client = Object.create(VirtualDesktopAdminClient.prototype) as VirtualDesktopAdminClient + ;(client as any).apiInvoker = { invoke_alt } + + await client.listDesktopImages({}) + await client.buildDesktopImage({ base_os: 'rocky9', architecture: 'x86_64', update_stack: true }) + + expect(invoke_alt).toHaveBeenCalledWith('VirtualDesktopAdmin.ListDesktopImages', {}) + expect(invoke_alt).toHaveBeenCalledWith('VirtualDesktopAdmin.BuildDesktopImage', { base_os: 'rocky9', architecture: 'x86_64', update_stack: true }) + }) + + it('useBuiltDesktopImages invokes its namespace', async () => { + const invoke_alt = vi.fn().mockResolvedValue({ results: [] }) + const client = Object.create(VirtualDesktopAdminClient.prototype) as VirtualDesktopAdminClient + ;(client as any).apiInvoker = { invoke_alt } + + await client.useBuiltDesktopImages({ stack_ids: ['ss-base-a'] }) + + expect(invoke_alt).toHaveBeenCalledWith('VirtualDesktopAdmin.UseBuiltDesktopImages', { stack_ids: ['ss-base-a'] }) + }) +}) diff --git a/source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-admin-client.ts b/source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-admin-client.ts index ea0dacbc..7256f2fe 100644 --- a/source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-admin-client.ts +++ b/source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-admin-client.ts @@ -20,6 +20,8 @@ import { UpdateSessionResponse, SetSessionCleanupExemptionRequest, SetSessionCleanupExemptionResponse, + RefreshBaseSoftwareStackAmisRequest, + RefreshBaseSoftwareStackAmisResponse, DeleteSessionRequest, DeleteSessionResponse, ListSessionsRequest, @@ -57,7 +59,15 @@ import { UpdatePermissionProfileResponse, UpdatePermissionProfileRequest, UpdateSessionPermissionRequest, - UpdateSessionPermissionResponse + UpdateSessionPermissionResponse, + ListDesktopImagesRequest, + ListDesktopImagesResponse, + BuildDesktopImageRequest, + BuildDesktopImageResponse, + BuildAllDesktopImagesRequest, + BuildAllDesktopImagesResponse, + UseBuiltDesktopImagesRequest, + UseBuiltDesktopImagesResponse } from './data-model' import IdeaBaseClient, {IdeaBaseClientProps} from "./base-client"; @@ -157,6 +167,13 @@ class VirtualDesktopAdminClient extends IdeaBaseClient { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.RefreshBaseSoftwareStackAmis', + req + ) + } + createSoftwareStack(req: CreateSoftwareStackRequest): Promise { return this.apiInvoker.invoke_alt( 'VirtualDesktopAdmin.CreateSoftwareStack', @@ -259,6 +276,34 @@ class VirtualDesktopAdminClient extends IdeaBaseClient { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.ListDesktopImages', + req + ) + } + + buildDesktopImage(req: BuildDesktopImageRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.BuildDesktopImage', + req + ) + } + + buildAllDesktopImages(req: BuildAllDesktopImagesRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.BuildAllDesktopImages', + req + ) + } + + useBuiltDesktopImages(req: UseBuiltDesktopImagesRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.UseBuiltDesktopImages', + req + ) + } } export default VirtualDesktopAdminClient diff --git a/source/idea/idea-cluster-manager/webapp/src/components/app-layout/app-layout.test.tsx b/source/idea/idea-cluster-manager/webapp/src/components/app-layout/app-layout.test.tsx new file mode 100644 index 00000000..fdf4c5cb --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/components/app-layout/app-layout.test.tsx @@ -0,0 +1,89 @@ +import {render, screen, waitFor} from '@testing-library/react'; +import {MemoryRouter} from 'react-router-dom'; +import moment from 'moment'; +import {vi} from 'vitest'; +import IdeaAppLayout from './index'; +import {initTestAppContext} from '../../test-support'; +import {MaintenanceSettings} from '../../service/cluster-settings-service'; + +// Every signed-in page routes through this layout, which is where the banner is rendered. + +const layoutProps = { + ideaPageId: 'test-page', + toolsOpen: false, + tools: null, + onToolsChange: () => {}, + onPageChange: () => {}, + sideNavHeader: {text: 'IDEA', href: '#/'}, + sideNavItems: [], + onSideNavChange: () => {}, + onFlashbarChange: () => {}, + flashbarItems: [], + content:

page body

+} as any; + +function renderLayout(maintenance: MaintenanceSettings) { + const context = initTestAppContext(); + const clusterSettings = context.getClusterSettingsService(); + // What the page was served with, and what the poll reads back. + clusterSettings.maintenance = maintenance; + vi.spyOn(clusterSettings, 'fetchMaintenance').mockResolvedValue(maintenance); + return render( + + + + ); +} + +describe('maintenance banner', () => { + it('shows the message while the window is open', async () => { + renderLayout({enabled: true, message: 'Scheduler closed for the 26.09 upgrade.', ends_at: ''}); + + expect(await screen.findByText('Cluster maintenance')).not.toBeNull(); + expect(screen.getByText('Scheduler closed for the 26.09 upgrade.')).not.toBeNull(); + }); + + it('shows nothing while the window is closed', async () => { + renderLayout({enabled: false, message: 'Scheduler closed for the 26.09 upgrade.', ends_at: ''}); + + // The page itself still renders; only the banner is absent. + expect(await screen.findByText('page body')).not.toBeNull(); + expect(screen.queryByText('Cluster maintenance')).toBeNull(); + expect(screen.queryByText('Scheduler closed for the 26.09 upgrade.')).toBeNull(); + }); + + it('adds the end of the window in the reader timezone', async () => { + renderLayout({enabled: true, message: 'Scheduler closed.', ends_at: '2026-09-15T18:00:00Z'}); + + // Rendered in the browser timezone, so the expected text is derived the same way. + const localEnd = moment.utc('2026-09-15T18:00:00Z').local().format('lll'); + expect(await screen.findByText(`Scheduler closed. until ${localEnd}`)).not.toBeNull(); + }); + + it('falls back to a generic message when the window is opened without one', async () => { + renderLayout({enabled: true, message: '', ends_at: ''}); + + expect(await screen.findByText('This cluster is undergoing maintenance.')).not.toBeNull(); + }); + + it('drops an unparsable end time instead of rendering "Invalid date"', async () => { + renderLayout({enabled: true, message: 'Scheduler closed.', ends_at: 'next tuesday'}); + + expect(await screen.findByText('Scheduler closed.')).not.toBeNull(); + }); + + it('takes the banner down when the poll reports the window closed', async () => { + const context = initTestAppContext(); + const clusterSettings = context.getClusterSettingsService(); + clusterSettings.maintenance = {enabled: true, message: 'Scheduler closed.', ends_at: ''}; + vi.spyOn(clusterSettings, 'fetchMaintenance').mockResolvedValue({enabled: false, message: '', ends_at: ''}); + + render( + + + + ); + + await waitFor(() => expect(screen.queryByText('Cluster maintenance')).toBeNull()); + }); +}); diff --git a/source/idea/idea-cluster-manager/webapp/src/components/app-layout/app-layout.tsx b/source/idea/idea-cluster-manager/webapp/src/components/app-layout/app-layout.tsx index f9d73944..c9ce7b39 100644 --- a/source/idea/idea-cluster-manager/webapp/src/components/app-layout/app-layout.tsx +++ b/source/idea/idea-cluster-manager/webapp/src/components/app-layout/app-layout.tsx @@ -24,6 +24,11 @@ import {withRouter} from "../../navigation/navigation-utils"; import IdeaSideNavigation, {IdeaSideNavigationProps} from "../side-navigation"; import IdeaNavbar from "../navbar"; import Utils from "../../common/utils"; +import {MaintenanceSettings, maintenanceFlashbarItems} from "../../service/cluster-settings-service"; + +// How often the banner is re-read while a page is open. Every page mounts this component, so a +// navigation picks up a change as well. +const MAINTENANCE_POLL_INTERVAL_MS = 60000 export interface IdeaAppLayoutProps extends IdeaSideNavigationProps { ideaPageId: string @@ -50,17 +55,50 @@ export interface IdeaAppLayoutProps extends IdeaSideNavigationProps { } export interface IdeaAppLayoutState { + maintenance: MaintenanceSettings } class IdeaAppLayout extends Component { + private maintenancePoll?: ReturnType + private mounted: boolean = false + + constructor(props: IdeaAppLayoutProps) { + super(props) + // The page is served with the window as it stood, so the banner is on the first render + // rather than appearing when the first poll comes back. + this.state = { + maintenance: AppContext.get().getClusterSettingsService().getMaintenance() + } + } + componentDidMount() { + this.mounted = true + Utils.hideLoadingAnimation() this.props.onPageChange({ pageId: this.props.ideaPageId }) + + this.refreshMaintenance() + this.maintenancePoll = setInterval(this.refreshMaintenance, MAINTENANCE_POLL_INTERVAL_MS) + } + + componentWillUnmount() { + this.mounted = false + if (this.maintenancePoll) { + clearInterval(this.maintenancePoll) + } + } + + refreshMaintenance = () => { + AppContext.get().getClusterSettingsService().fetchMaintenance().then(maintenance => { + if (this.mounted) { + this.setState({maintenance: maintenance}) + } + }) } buildBreadCrumbs() { @@ -74,8 +112,9 @@ class IdeaAppLayout extends Component { /> } + // The maintenance notice is prepended to whatever the page is already showing. buildNotifications() { - return + return } buildFooter() { diff --git a/source/idea/idea-cluster-manager/webapp/src/components/cost-sections.tsx b/source/idea/idea-cluster-manager/webapp/src/components/cost-sections.tsx new file mode 100644 index 00000000..6264547b --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/components/cost-sections.tsx @@ -0,0 +1,303 @@ +import React, {useState} from "react"; +import {Badge, Box, Button, ColumnLayout, Container, Header, SpaceBetween, Table} from "@cloudscape-design/components"; +import {TableProps} from "@cloudscape-design/components/table/interfaces"; +import {GetMyCostsSummaryResult, MyCostsDesktopSession, MyCostsJob, MyCostsJobGroup} from "../client/data-model"; +import Utils from "../common/utils"; + +/** + * The three cost sections, rendered from a summary payload. Shared by the self scoped My Costs + * page and the admin drill-in so both show a user the same way. + */ + +export const ESTIMATED_NOTE = 'These are the costs IDEA measured, not the AWS bill. Treat them as estimates.' + +export const money = (value?: number): string => (value == null ? '-' : Utils.getFormattedAmount({amount: value})) + +export const number = (value?: number): string => (value == null ? '-' : value.toLocaleString()) + +export const hours = (value?: number): string => (value == null ? '-' : `${value.toFixed(2)} h`) + +const estimatedBadge = () => Estimated + +/** + * Formats a subtotal. Reads "Not available" when no row could be priced, and carries a marker + * when only some rows could, so an incomplete total never reads as the whole spend. + */ +export const summaryCost = (cost?: number, rowCount?: number, unpriced?: number): string => { + const rows = rowCount ?? 0 + const missing = unpriced ?? 0 + if (rows > 0 && missing >= rows) { + return 'Not available' + } + return missing > 0 ? `${money(cost)} *` : money(cost) +} + +export const unpricedNote = (unpriced?: number, rowCount?: number, noun?: string) => { + const rows = rowCount ?? 0 + const missing = unpriced ?? 0 + if (missing === 0 || missing >= rows) { + return null + } + return ( + + {`* ${missing} of ${rows} ${noun} could not be priced and are not in this total.`} + + ) +} + +const empty = (title: string, message: string) => ( + + {title} + {message} + +) + +interface AiRow { + id: string + label: string + total_tokens: number + invocations: number + cost?: number + cost_unavailable?: boolean + children?: AiRow[] +} + +const AI_COLUMNS: TableProps.ColumnDefinition[] = [ + {id: 'label', header: 'Project / model', cell: (item) => item.label}, + {id: 'requests', header: 'Requests', cell: (item) => number(item.invocations)}, + {id: 'tokens', header: 'Tokens', cell: (item) => number(item.total_tokens)}, + {id: 'cost', header: 'Cost', cell: (item) => (item.cost_unavailable ? 'Not available' : money(item.cost))} +] + +const JOB_COLUMNS: TableProps.ColumnDefinition[] = [ + {id: 'job_id', header: 'Job', cell: (item) => item.job_id}, + {id: 'name', header: 'Name', cell: (item) => item.name}, + {id: 'queue', header: 'Queue', cell: (item) => item.queue}, + {id: 'project', header: 'Project', cell: (item) => item.project}, + {id: 'end_time', header: 'Finished', cell: (item) => item.end_time}, + // A job the scheduler could not price must not read as a free job. + {id: 'cost', header: 'Cost', cell: (item) => (item.cost_unavailable ? 'Price not available' : money(item.cost))} +] + +const GROUP_COLUMNS = (header: string): TableProps.ColumnDefinition[] => [ + {id: 'name', header: header, cell: (item) => item.name}, + {id: 'job_count', header: 'Jobs', cell: (item) => number(item.job_count)}, + {id: 'cost', header: 'Cost', cell: (item) => money(item.cost)} +] + +const DESKTOP_COLUMNS: TableProps.ColumnDefinition[] = [ + {id: 'name', header: 'Session', cell: (item) => item.name}, + {id: 'instance_type', header: 'Instance type', cell: (item) => item.instance_type}, + {id: 'state', header: 'State', cell: (item) => item.state}, + { + id: 'hours', + header: 'Hours', + // An inferred stop time is an upper bound, so the hours are marked estimated. + cell: (item) => (item.stop_time_estimated ? `${hours(item.hours)} (estimated)` : hours(item.hours)) + }, + {id: 'cost', header: 'Cost', cell: (item) => (item.price_unavailable ? 'Price not available' : money(item.cost))} +] + +export interface CostSectionsProps { + summary: GetMyCostsSummaryResult | null + loading: boolean + error?: string | null + onRetry?: () => void + /** Empty state wording: "You have not" reads wrong on another user's page. */ + subject?: 'self' | 'user' +} + +/** + * Hides the AI section for a user with no tokens in the window, rather than showing it empty. A + * failed read still shows the section, because unavailable is not the same as unused. + */ +export const showAiSection = (summary: GetMyCostsSummaryResult | null, loading: boolean): boolean => { + const ai = summary?.ai + if (loading || ai?.is_unavailable) { + return true + } + return (ai?.total_tokens ?? 0) > 0 || (ai?.invocations ?? 0) > 0 +} + +const CostSections: React.FC = ({summary, loading, error, onRetry, subject = 'self'}) => { + + const [expandedAi, setExpandedAi] = useState([]) + + const who = subject === 'self' ? 'You have' : 'This user has' + + const aiRows = (): AiRow[] => (summary?.ai?.projects ?? []).map((project) => ({ + id: project.project_id ?? '', + label: project.project_title ?? project.project_name ?? '', + total_tokens: project.total_tokens ?? 0, + invocations: project.invocations ?? 0, + cost: project.cost, + cost_unavailable: project.cost_unavailable, + children: (project.by_model ?? []).map((model) => ({ + id: `${project.project_id}/${model.model_id}`, + label: model.model_id ?? '', + total_tokens: model.total_tokens ?? 0, + invocations: model.invocations ?? 0, + cost: model.cost, + cost_unavailable: project.cost_unavailable + })) + })) + + const sectionHeader = (title: string, description: string) => ( +
{title}
+ ) + + const tableEmpty = (title: string, message: string) => { + if (error != null) { + return ( + + Could not load these costs + {error} + {onRetry != null && } + + ) + } + return empty(title, message) + } + + const ai = summary?.ai + const jobs = summary?.jobs + const desktops = summary?.desktops + + const renderAi = () => ( + + + +
+ Requests + {number(ai?.invocations)} +
+
+ Tokens + {number(ai?.total_tokens)} +
+
+ Cost + {money(ai?.cost)} +
+ + `Show models for ${item.label}`, + collapseButtonLabel: (item) => `Hide models for ${item.label}` + }} + expandableRows={{ + getItemChildren: (item) => item.children ?? [], + isItemExpandable: (item) => (item.children?.length ?? 0) > 0, + expandedItems: expandedAi, + onExpandableItemToggle: ({detail}) => { + const rest = expandedAi.filter((row) => row.id !== detail.item.id) + setExpandedAi(detail.expanded ? [...rest, detail.item] : rest) + } + }} + empty={ai?.is_unavailable + ? empty('AI usage is not available', 'IDEA could not read the Bedrock usage.') + : tableEmpty('No AI usage', `${who} not invoked a model in a project in the last 30 days.`)} + /> + + + ) + + const renderDesktops = () => ( + + + +
+ Sessions + {number(desktops?.session_count)} +
+
+ Hours + {hours(desktops?.hours)} +
+
+ Cost + {summaryCost(desktops?.cost, desktops?.session_count, desktops?.unpriced_sessions)} +
+ + {unpricedNote(desktops?.unpriced_sessions, desktops?.session_count, 'sessions')} +
+ + + ) + + const renderJobs = () => ( + + + +
+ Completed jobs + {number(jobs?.job_count)} +
+
+ Cost + {jobs?.cost_unavailable + ? 'Not available' + : summaryCost(jobs?.cost, jobs?.job_count, jobs?.unpriced_jobs)} +
+ + {unpricedNote(jobs?.unpriced_jobs, jobs?.job_count, 'jobs')} +
+
+
Most recent jobs} + items={jobs?.recent_jobs ?? []} + loading={loading} + loadingText="Retrieving jobs ..." + columnDefinitions={JOB_COLUMNS} + empty={empty('No completed jobs', `${who} not completed a job in the last 30 days.`)} + /> + + + ) + + return ( + + {showAiSection(summary, loading) && renderAi()} + {renderDesktops()} + {renderJobs()} + + ) +} + +export default CostSections diff --git a/source/idea/idea-cluster-manager/webapp/src/navigation/side-nav-items.tsx b/source/idea/idea-cluster-manager/webapp/src/navigation/side-nav-items.tsx index c9c1b06a..d861b073 100644 --- a/source/idea/idea-cluster-manager/webapp/src/navigation/side-nav-items.tsx +++ b/source/idea/idea-cluster-manager/webapp/src/navigation/side-nav-items.tsx @@ -77,6 +77,13 @@ export const IdeaSideNavItems = (context: AppContext): SideNavigationProps.Item[ href: '#/home/ssh-access' }) } + + // Every user with cluster-manager access has costs to read, so this link is not gated further. + userNav.items.push({ + type: 'link', + text: 'My Costs', + href: '#/home/my-costs' + }) if (context.getClusterSettingsService().isCustomDashboardEnabled()) { userNav.items.push({ type: 'link', @@ -159,6 +166,11 @@ export const IdeaSideNavItems = (context: AppContext): SideNavigationProps.Item[ text: 'Licenses', href: '#/soca/licenses' }, + { + type: 'link', + text: 'Custom AMIs', + href: '#/soca/custom-amis' + }, { type: 'link', text: 'Active Jobs', @@ -190,6 +202,16 @@ export const IdeaSideNavItems = (context: AppContext): SideNavigationProps.Item[ text: 'Projects', href: '#/cluster/projects' }, + { + type: 'link', + text: 'AI Usage', + href: '#/cluster/ai-usage' + }, + { + type: 'link', + text: 'User Costs', + href: '#/cluster/user-costs' + }, { type: 'link', text: 'Users', diff --git a/source/idea/idea-cluster-manager/webapp/src/pages/auth/auth-layout.tsx b/source/idea/idea-cluster-manager/webapp/src/pages/auth/auth-layout.tsx index efb70028..4e49190e 100644 --- a/source/idea/idea-cluster-manager/webapp/src/pages/auth/auth-layout.tsx +++ b/source/idea/idea-cluster-manager/webapp/src/pages/auth/auth-layout.tsx @@ -12,11 +12,12 @@ */ import React, {Component} from "react"; -import {AppLayout, Grid} from "@cloudscape-design/components"; +import {AppLayout, Flashbar, Grid} from "@cloudscape-design/components"; import {AppContext} from "../../common"; import './auth.scss' import Utils from "../../common/utils"; import AppLogger from "../../common/app-logger"; +import {maintenanceFlashbarItems} from "../../service/cluster-settings-service"; export interface AuthLayoutProps { loading?: boolean @@ -72,11 +73,19 @@ class AuthLayout extends Component { return this.props.loading; } + // Before sign in there is no token for the settings API, so the banner can only come from the + // value served with the page. + buildMaintenanceNotifications() { + const items = maintenanceFlashbarItems(AppContext.get().getClusterSettingsService().getMaintenance()) + return (items.length > 0) ? : undefined + } + render() { this.logger.debug(`Rendering AuthLayout component. Ready state: ${this.state.ready}`); return { + render( + + {}} + onPageChange={() => {}} + sideNavHeader={{text: 'IDEA', href: '#/'}} + sideNavItems={[]} + onSideNavChange={() => {}} + onFlashbarChange={() => {}} + flashbarItems={[]} + /> + + ); +}; + +const withUsage = (context: AppContext, listing: any[]) => { + vi.spyOn(context.client().projects(), 'listBedrockUsage').mockResolvedValue({listing: listing}); +}; + +describe('ai usage page', () => { + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('lists every bedrock project with its tokens, requests, cost and top model', async () => { + const context = initTestAppContext(); + withUsage(context, USAGE_LISTING); + renderAiUsagePage(); + + expect(await screen.findByText('Research')).toBeInTheDocument(); + expect(await screen.findByText('1,500')).toBeInTheDocument(); + expect(await screen.findByText('$3.00')).toBeInTheDocument(); + expect(await screen.findByText('nova-pro')).toBeInTheDocument(); + // A Bedrock project nobody used is still listed, so it is visibly at zero. + expect(await screen.findByText('Physics')).toBeInTheDocument(); + expect(await screen.findByText('No usage recorded')).toBeInTheDocument(); + }); + + it('breaks the selected project down per model and per user, cost labeled estimated', async () => { + const context = initTestAppContext(); + withUsage(context, USAGE_LISTING); + renderAiUsagePage(); + await screen.findByText('Research'); + + await userEvent.click(screen.getAllByRole('radio')[0]); + + expect(await screen.findByText('Per model')).toBeInTheDocument(); + expect(await screen.findByText('Per user')).toBeInTheDocument(); + // The input and output split comes from the day rows, so both are counted numbers. + expect(await screen.findByText('900')).toBeInTheDocument(); + expect(await screen.findByText('alice')).toBeInTheDocument(); + expect(await screen.findByText('bob')).toBeInTheDocument(); + // A share of the project spend is never presented as a priced total. + expect((await screen.findAllByText('$2.00 (estimated)')).length).toBeGreaterThan(0); + expect(await screen.findByText(/Cost per model and per user is estimated/)).toBeInTheDocument(); + }); + + it('reports a project whose usage read failed instead of claiming no usage', async () => { + const context = initTestAppContext(); + withUsage(context, [{ + project_id: 'p-3', + name: 'unreadable', + title: 'Unreadable', + enabled: true, + bedrock: {enabled: true, model_ids: ['vendor.model-a']}, + bedrock_usage: {is_unavailable: true} + }]); + renderAiUsagePage(); + + expect(await screen.findByText('Usage unavailable')).toBeInTheDocument(); + expect(screen.queryByText('No usage recorded')).toBeNull(); + }); +}); diff --git a/source/idea/idea-cluster-manager/webapp/src/pages/cluster-admin/ai-usage.tsx b/source/idea/idea-cluster-manager/webapp/src/pages/cluster-admin/ai-usage.tsx new file mode 100644 index 00000000..10401ca3 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/pages/cluster-admin/ai-usage.tsx @@ -0,0 +1,294 @@ +import React, {Component, RefObject} from "react"; + +import {TableProps} from "@cloudscape-design/components/table/interfaces"; +import {BedrockModelUsage, BedrockUserUsage, Project, SocaAmount} from '../../client/data-model' +import IdeaListView from "../../components/list-view"; +import IdeaSplitPanel from "../../components/split-panel"; +import {ProjectsClient} from "../../client"; +import {AppContext} from "../../common"; +import {Box, SpaceBetween, StatusIndicator, Table} from "@cloudscape-design/components"; +import Utils from "../../common/utils"; +import {IdeaSideNavigationProps} from "../../components/side-navigation"; +import IdeaAppLayout, {IdeaAppLayoutProps} from "../../components/app-layout"; +import {withRouter} from "../../navigation/navigation-utils"; + +export interface AiUsageProps extends IdeaAppLayoutProps, IdeaSideNavigationProps { + +} + +export interface AiUsageState { + projectSelected: boolean + splitPanelOpen: boolean +} + +// Strips the geography prefix, the provider prefix and the version suffix, so that +// us.amazon.nova-pro-v1:0 reads as nova-pro. +const shortModelName = (modelId: string): string => modelId + .replace(/^(us|eu|apac|global)\./, '') + .replace(/^(anthropic|amazon|meta)\./, '') + .replace(/-v\d+:\d+$/, '') + +const tokens = (value?: number): string => Utils.asNumber(value, 0).toLocaleString() + +// A per model or per user figure is a share of the project spend rather than a priced total, so +// it is always marked estimated. No answer from AWS Cost Explorer stays blank instead of zero. +const shareOfSpend = (spend?: SocaAmount, isEstimated?: boolean): string => { + if (!spend) { + return '--' + } + return `${Utils.getFormattedAmount(spend)}${isEstimated ? ' (estimated)' : ''}` +} + +const projectSpend = (project: Project) => { + const usage = project.bedrock_usage + if (!usage || usage.is_unavailable) { + return -- + } + if (usage.spend_is_unavailable || !usage.spend) { + return cost unavailable + } + return Utils.getFormattedAmount(usage.spend) +} + +const MODEL_COLUMN_DEFINITIONS: TableProps.ColumnDefinition[] = [ + { + id: 'model_id', + header: 'Model', + cell: entry => shortModelName(entry.model_id ?? '') + }, + { + id: 'input_tokens', + header: 'Input tokens', + cell: entry => tokens(entry.input_tokens) + }, + { + id: 'output_tokens', + header: 'Output tokens', + cell: entry => tokens(entry.output_tokens) + }, + { + id: 'total_tokens', + header: 'Total tokens', + cell: entry => tokens(entry.total_tokens) + }, + { + id: 'invocations', + header: 'Requests', + cell: entry => tokens(entry.invocations) + }, + { + id: 'spend', + header: 'Cost', + cell: entry => shareOfSpend(entry.spend, entry.spend_is_estimated) + } +] + +const USER_COLUMN_DEFINITIONS: TableProps.ColumnDefinition[] = [ + { + id: 'username', + header: 'User', + cell: entry => entry.username + }, + { + id: 'total_tokens', + header: 'Tokens', + cell: entry => tokens(entry.total_tokens) + }, + { + id: 'invocations', + header: 'Requests', + cell: entry => tokens(entry.invocations) + }, + { + id: 'spend', + header: 'Cost', + cell: entry => shareOfSpend(entry.spend, entry.spend_is_estimated) + }, + { + id: 'top_model_id', + header: 'Top model', + cell: entry => Utils.isEmpty(entry.top_model_id) ? '--' : shortModelName(entry.top_model_id!) + } +] + +const PROJECT_COLUMN_DEFINITIONS: TableProps.ColumnDefinition[] = [ + { + id: 'title', + header: 'Project', + cell: project => project.title, + sortingField: 'title' + }, + { + id: 'name', + header: 'Project Code', + cell: project => project.name, + sortingField: 'name' + }, + { + id: 'total_tokens', + header: 'Tokens', + cell: project => { + const usage = project.bedrock_usage + if (usage?.is_unavailable) { + return Usage unavailable + } + if (!usage || Utils.asNumber(usage.total_tokens, 0) === 0) { + return No usage recorded + } + return tokens(usage.total_tokens) + }, + sortingComparator: (a, b) => Utils.asNumber(a.bedrock_usage?.total_tokens, 0) - Utils.asNumber(b.bedrock_usage?.total_tokens, 0) + }, + { + id: 'invocations', + header: 'Requests', + cell: project => tokens(project.bedrock_usage?.invocations) + }, + { + id: 'spend', + header: 'Cost', + cell: project => projectSpend(project) + }, + { + id: 'top_model', + header: 'Top model', + // The service sorts by_model by tokens, so the first entry is the busiest. + cell: project => { + const top = project.bedrock_usage?.by_model?.[0] + return top ? shortModelName(top.model_id ?? '') : -- + } + } +] + +class AiUsage extends Component { + + listing: RefObject + + constructor(props: AiUsageProps) { + super(props); + this.listing = React.createRef() + this.state = { + projectSelected: false, + splitPanelOpen: false + } + } + + projects(): ProjectsClient { + return AppContext.get().client().projects() + } + + getListing(): IdeaListView { + return this.listing.current! + } + + getSelected(): Project | null { + if (this.getListing() == null) { + return null + } + return this.getListing().getSelectedItem() + } + + isSelected(): boolean { + return this.state.projectSelected && this.getSelected() != null + } + + buildListing() { + return ( + { + this.setState({ + projectSelected: false, + splitPanelOpen: false + }, () => { + this.getListing().fetchRecords() + }) + }} + onSelectionChange={() => { + this.setState({ + projectSelected: true, + splitPanelOpen: true + }) + }} + onFetchRecords={() => { + return this.projects().listBedrockUsage({}) + }} + columnDefinitions={PROJECT_COLUMN_DEFINITIONS} + /> + ) + } + + buildSplitPanelContent() { + const usage = this.getSelected()?.bedrock_usage + return (this.isSelected() && + + +
Per model} + columnDefinitions={MODEL_COLUMN_DEFINITIONS} + items={usage?.by_model ?? []} + empty={No usage recorded in the last 30 days.} + /> +
Per user} + columnDefinitions={USER_COLUMN_DEFINITIONS} + items={usage?.by_user ?? []} + empty={No usage recorded in the last 30 days.} + /> + + ) + } + + render() { + return ( + + {this.buildListing()} + + } + splitPanelOpen={this.state.splitPanelOpen} + splitPanel={this.buildSplitPanelContent()} + onSplitPanelToggle={(event: any) => { + this.setState({ + splitPanelOpen: event.detail.open + }) + }} + /> + ) + } +} + +export default withRouter(AiUsage) diff --git a/source/idea/idea-cluster-manager/webapp/src/pages/cluster-admin/cluster-settings.tsx b/source/idea/idea-cluster-manager/webapp/src/pages/cluster-admin/cluster-settings.tsx index f02bc8cc..733eaadd 100644 --- a/source/idea/idea-cluster-manager/webapp/src/pages/cluster-admin/cluster-settings.tsx +++ b/source/idea/idea-cluster-manager/webapp/src/pages/cluster-admin/cluster-settings.tsx @@ -14,7 +14,8 @@ import React, {Component, RefObject} from "react"; import {IdeaSideNavigationProps} from "../../components/side-navigation"; import IdeaAppLayout, {IdeaAppLayoutProps} from "../../components/app-layout"; -import {Alert, Box, Button, ColumnLayout, Container, FormField, Header, Input, Link, SpaceBetween, Table, Tabs} from "@cloudscape-design/components"; +import {Alert, Box, Button, ColumnLayout, Container, FormField, Header, Input, Link, SpaceBetween, Table, Tabs, Textarea, Toggle} from "@cloudscape-design/components"; +import moment from "moment"; import {KeyValue, KeyValueGroup} from "../../components/key-value"; import {AppContext} from "../../common"; import dot from "dot-object"; @@ -54,6 +55,13 @@ export interface ClusterSettingsState { bedrockPassRoleReady: boolean bedrockUsageLoggingManaged: boolean bedrockModelIdPendingRemoval: string | null + + maintenanceEnabled: boolean + maintenanceMessage: string + maintenanceEndsAt: string + maintenanceUpdating: boolean + maintenanceError: string | null + maintenanceSaved: boolean } const DEFAULT_ACTIVE_TAB_ID = 'general' @@ -99,7 +107,14 @@ class ClusterSettings extends Component { + const message = this.state.maintenanceMessage.trim() + const endsAt = this.state.maintenanceEndsAt.trim() + + if (Utils.isNotEmpty(endsAt) && !moment(endsAt, moment.ISO_8601, true).isValid()) { + this.setState({ + maintenanceError: 'End time must be an ISO 8601 timestamp, for example 2026-09-15T18:00:00Z. Leave it empty for no end time.' + }) + return + } + + const clusterSettingsService = AppContext.get().getClusterSettingsService() + const moduleId = Utils.asString(clusterSettingsService.getModuleId(Constants.MODULE_CLUSTER_MANAGER), Constants.MODULE_CLUSTER_MANAGER) + this.setState({ + maintenanceUpdating: true, + maintenanceError: null, + maintenanceSaved: false + }) + AppContext.get().client().clusterSettings().updateModuleSettings({ + module_id: moduleId, + settings: { + maintenance: { + enabled: this.state.maintenanceEnabled, + message: message, + ends_at: endsAt + } + } + }).then(result => { + if (!Utils.asBoolean(result.success, false)) { + this.setState({ + maintenanceUpdating: false, + maintenanceError: 'Failed to update the maintenance settings.' + }) + return + } + this.setState({ + maintenanceUpdating: false, + maintenanceMessage: message, + maintenanceEndsAt: endsAt, + maintenanceSaved: true + }) + }).catch(error => { + this.setState({ + maintenanceUpdating: false, + maintenanceError: error?.message ?? `${error}` + }) + }) + } + + buildMaintenanceSettings() { + return ( + + Maintenance Window}> + + {this.state.maintenanceError && this.setState({maintenanceError: null})}>{this.state.maintenanceError}} + {this.state.maintenanceSaved && this.setState({maintenanceSaved: false})}> + Saved. Open portal pages pick the change up within a minute, and the scheduler within about half a minute. + } + this.setState({maintenanceEnabled: event.detail.checked})}> + Show the maintenance banner and refuse job submissions + + +