diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..cebedf1a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +**/node_modules +**/dist +**/cdk.out* +source/idea/ideactl/tools/parity/live +source/idea/ideactl/tools/parity/fixtures +source/idea/ideactl/docs/port +source/idea/ideactl/tools/e2e/reference +*.pem diff --git a/.github/workflows/build_push.yaml b/.github/workflows/build_push.yaml index 3de6e38a..4467c83f 100644 --- a/.github/workflows/build_push.yaml +++ b/.github/workflows/build_push.yaml @@ -17,7 +17,188 @@ on: required: false default: '' type: string + control_plane_image_name: + description: 'Repository name for the control-plane image. Leave empty for the released image. A non-main dispatch must set this and scheduler_image_name.' + required: false + default: '' + type: string + scheduler_image_name: + description: 'Repository name for the scheduler image. Leave empty for the released image. A non-main dispatch must set this and control_plane_image_name.' + required: false + default: '' + type: string jobs: + build_ideactl_artifacts: + name: Build ideactl macOS arm64 + runs-on: macos-15 + defaults: + run: + working-directory: source/idea/ideactl + steps: + - uses: actions/checkout@v4 + - name: Set up release runtime + uses: actions/setup-node@v4 + with: + node-version: '26.7.0' + cache: npm + cache-dependency-path: source/idea/ideactl/package-lock.json + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Check package version + run: | + PACKAGE_VERSION=$(node --input-type=module -e 'import packageJson from "./package.json" with { type: "json" }; process.stdout.write(packageJson.version)') + RELEASE_VERSION=$(tr -d '[:space:]' < ../../../IDEA_VERSION.txt) + if [ "$PACKAGE_VERSION" != "$RELEASE_VERSION" ]; then + echo "::error::ideactl package version ${PACKAGE_VERSION} does not match IDEA_VERSION.txt ${RELEASE_VERSION}." + exit 1 + fi + - name: Install dependencies + run: npm ci + - name: Build the macOS release file + run: npm run build:dist -- --target darwin-arm64 + - name: Test the macOS release file from an empty directory + run: | + set -euo pipefail + ARTIFACT="${PWD}/dist/release/darwin-arm64/ideactl" + SMOKE_ROOT=$(mktemp -d) + trap 'rm -rf "$SMOKE_ROOT"' EXIT + mkdir -p "$SMOKE_ROOT/config" "$SMOKE_ROOT/home" "$SMOKE_ROOT/tmp" "$SMOKE_ROOT/empty-path" "$SMOKE_ROOT/work" + cp test/shell-path/values.yml "$SMOKE_ROOT/values.yml" + ( + cd "$SMOKE_ROOT/work" + env -i \ + HOME="$SMOKE_ROOT/home" \ + IDEA_USER_HOME="$SMOKE_ROOT/home/.idea" \ + LANG=en_US.UTF-8 \ + NODE_PATH= \ + PATH="$SMOKE_ROOT/empty-path" \ + TMPDIR="$SMOKE_ROOT/tmp" \ + "$ARTIFACT" about + env -i \ + HOME="$SMOKE_ROOT/home" \ + IDEA_USER_HOME="$SMOKE_ROOT/home/.idea" \ + LANG=en_US.UTF-8 \ + NODE_PATH= \ + PATH="$SMOKE_ROOT/empty-path" \ + TMPDIR="$SMOKE_ROOT/tmp" \ + "$ARTIFACT" config generate \ + --values-file "$SMOKE_ROOT/values.yml" \ + --config-dir "$SMOKE_ROOT/config" \ + --force + ) + test -s "$SMOKE_ROOT/config/config/idea.yml" + - name: Upload the macOS release artifact + uses: actions/upload-artifact@v4 + with: + name: ideactl-darwin-arm64 + path: | + source/idea/ideactl/dist/release/darwin-arm64/ideactl + source/idea/ideactl/dist/release/ideactl-v*-darwin-arm64.tar.gz + source/idea/ideactl/dist/release/ideactl-v*-darwin-arm64.tar.gz.sha256 + if-no-files-found: error + retention-days: 7 + build_ideactl_linux_artifact: + name: Build ideactl Linux arm64 + runs-on: ubuntu-24.04-arm + defaults: + run: + working-directory: source/idea/ideactl + steps: + - uses: actions/checkout@v4 + - name: Set up release runtime + uses: actions/setup-node@v4 + with: + node-version: '26.7.0' + cache: npm + cache-dependency-path: source/idea/ideactl/package-lock.json + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Check package version + run: | + PACKAGE_VERSION=$(node --input-type=module -e 'import packageJson from "./package.json" with { type: "json" }; process.stdout.write(packageJson.version)') + RELEASE_VERSION=$(tr -d '[:space:]' < ../../../IDEA_VERSION.txt) + if [ "$PACKAGE_VERSION" != "$RELEASE_VERSION" ]; then + echo "::error::ideactl package version ${PACKAGE_VERSION} does not match IDEA_VERSION.txt ${RELEASE_VERSION}." + exit 1 + fi + - name: Install dependencies + run: npm ci + - name: Build the Linux release file + run: npm run build:dist -- --target linux-arm64 + - name: Test the Linux release file from an empty directory + run: | + set -euo pipefail + ARTIFACT="${PWD}/dist/release/linux-arm64/ideactl" + chmod +x "$ARTIFACT" + SMOKE_ROOT=$(mktemp -d) + trap 'rm -rf "$SMOKE_ROOT"' EXIT + mkdir -p "$SMOKE_ROOT/config" "$SMOKE_ROOT/home" "$SMOKE_ROOT/tmp" "$SMOKE_ROOT/empty-path" "$SMOKE_ROOT/work" + cp test/shell-path/values.yml "$SMOKE_ROOT/values.yml" + ( + cd "$SMOKE_ROOT/work" + env -i \ + HOME="$SMOKE_ROOT/home" \ + IDEA_USER_HOME="$SMOKE_ROOT/home/.idea" \ + LANG=en_US.UTF-8 \ + NODE_PATH= \ + PATH="$SMOKE_ROOT/empty-path" \ + TMPDIR="$SMOKE_ROOT/tmp" \ + "$ARTIFACT" about + env -i \ + HOME="$SMOKE_ROOT/home" \ + IDEA_USER_HOME="$SMOKE_ROOT/home/.idea" \ + LANG=en_US.UTF-8 \ + NODE_PATH= \ + PATH="$SMOKE_ROOT/empty-path" \ + TMPDIR="$SMOKE_ROOT/tmp" \ + "$ARTIFACT" config generate \ + --values-file "$SMOKE_ROOT/values.yml" \ + --config-dir "$SMOKE_ROOT/config" \ + --force + ) + test -s "$SMOKE_ROOT/config/config/idea.yml" + - name: Test the Linux release file in a container image + run: | + set -euo pipefail + IMAGE=ideactl-linux-smoke + SMOKE_ROOT=$(mktemp -d) + trap 'rm -rf "$SMOKE_ROOT"' EXIT + mkdir -p "$SMOKE_ROOT/config" + cp test/shell-path/values.yml "$SMOKE_ROOT/values.yml" + docker build \ + --tag "$IMAGE" \ + --file scripts/ideactl-linux.Dockerfile \ + dist/release/linux-arm64 + docker run --rm \ + --env PATH=/tmp/empty \ + --workdir /tmp/work \ + "$IMAGE" \ + about + docker run --rm \ + --env PATH=/tmp/empty \ + --workdir /tmp/work \ + --volume "$SMOKE_ROOT/values.yml:/tmp/values.yml:ro" \ + --volume "$SMOKE_ROOT/config:/tmp/config" \ + "$IMAGE" \ + config generate \ + --values-file /tmp/values.yml \ + --config-dir /tmp/config \ + --force + test -s "$SMOKE_ROOT/config/config/idea.yml" + - name: Upload the Linux release artifact + uses: actions/upload-artifact@v4 + with: + name: ideactl-linux-arm64 + path: | + source/idea/ideactl/dist/release/linux-arm64/ideactl + source/idea/ideactl/dist/release/ideactl-v*-linux-arm64.tar.gz + source/idea/ideactl/dist/release/ideactl-v*-linux-arm64.tar.gz.sha256 + if-no-files-found: error + retention-days: 7 build_push: runs-on: ubuntu-large permissions: @@ -57,3 +238,158 @@ jobs: run: | VERSION=$(cat IDEA_VERSION.txt) invoke docker.build-push-multi "$ECR_REGISTRY" "$VERSION" --image-name "$IMAGE_NAME" --gha-cache + build_push_ideactl: + runs-on: ubuntu-large + permissions: + id-token: write + contents: read + steps: + - name: Validate the image names + env: + CONTROL_PLANE_IMAGE_NAME: ${{ github.event.inputs.control_plane_image_name }} + SCHEDULER_IMAGE_NAME: ${{ github.event.inputs.scheduler_image_name }} + run: | + if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ] && [ "$GITHUB_REF_NAME" != "main" ] && { [ -z "$CONTROL_PLANE_IMAGE_NAME" ] || [ -z "$SCHEDULER_IMAGE_NAME" ]; }; then + echo "::error::A dispatch from $GITHUB_REF_NAME must set both control_plane_image_name and scheduler_image_name, or it would overwrite released images from an unmerged ref." + exit 1 + fi + - uses: actions/checkout@v4 + - name: Check version consistency + run: | + SH_REV=$(grep -m1 '^IDEA_REVISION=' idea-admin.sh | sed -E 's/.*:-"([^"]+)"\}.*/\1/') + VERSION=$(tr -d '[:space:]' < IDEA_VERSION.txt) + EXPECTED="v${VERSION}" + echo "idea-admin.sh revision: ${SH_REV}" + echo "IDEA_VERSION.txt: ${VERSION} (expected revision ${EXPECTED})" + if [ "${SH_REV}" != "${EXPECTED}" ]; then + echo "::error::Version mismatch: IDEA_VERSION.txt=${VERSION} (expected revision ${EXPECTED}), idea-admin.sh default=${SH_REV}" + exit 1 + fi + echo "Version consistency check passed (${EXPECTED})" + - name: Setup Development Environment + uses: ./.github/actions/setup_dev_environment + - name: Build Modules + uses: ./.github/actions/build_modules + - name: Configure AWS Credentials for ECR Push + uses: aws-actions/configure-aws-credentials@v4.2.1 + with: + audience: sts.amazonaws.com + aws-region: us-east-1 + role-to-assume: ${{ secrets.ECR_ROLE }} + - name: Stage DCV packages + run: | + set -euo pipefail + PACKAGE_DIR=/tmp/dcv-packages + DOWNLOAD_HOST=https://d1uj6qtbmh3dt5.cloudfront.net + mkdir -p "$PACKAGE_DIR" + curl --fail --location --silent --show-error -o "$PACKAGE_DIR/NICE-GPG-KEY" "$DOWNLOAD_HOST/NICE-GPG-KEY" + curl --fail --location --silent --show-error -o "$PACKAGE_DIR/nice-dcv-session-manager-broker-amzn2023.noarch.rpm" "$DOWNLOAD_HOST/nice-dcv-session-manager-broker-amzn2023.noarch.rpm" + curl --fail --location --silent --show-error -o "$PACKAGE_DIR/nice-dcv-connection-gateway-amzn2023.x86_64.rpm" "$DOWNLOAD_HOST/nice-dcv-connection-gateway-amzn2023.x86_64.rpm" + curl --fail --location --silent --show-error -o "$PACKAGE_DIR/nice-dcv-connection-gateway-amzn2023.aarch64.rpm" "$DOWNLOAD_HOST/nice-dcv-connection-gateway-amzn2023.aarch64.rpm" + curl --fail --location --silent --show-error -o "$PACKAGE_DIR/nice-dcv-amzn2023-x86_64.tgz" "$DOWNLOAD_HOST/nice-dcv-amzn2023-x86_64.tgz" + curl --fail --location --silent --show-error -o "$PACKAGE_DIR/nice-dcv-amzn2023-aarch64.tgz" "$DOWNLOAD_HOST/nice-dcv-amzn2023-aarch64.tgz" + ( + cd "$PACKAGE_DIR" + sha256sum \ + nice-dcv-session-manager-broker-amzn2023.noarch.rpm \ + nice-dcv-connection-gateway-amzn2023.x86_64.rpm \ + nice-dcv-connection-gateway-amzn2023.aarch64.rpm \ + nice-dcv-amzn2023-x86_64.tgz \ + nice-dcv-amzn2023-aarch64.tgz \ + > checksums.txt + ) + - name: Build and Push Scheduler and Control-Plane Images + env: + ECR_REGISTRY: ${{ github.event.inputs.ecr_repository }} + CONTROL_PLANE_IMAGE_NAME: ${{ github.event.inputs.control_plane_image_name || 'idea-control-plane' }} + SCHEDULER_IMAGE_NAME: ${{ github.event.inputs.scheduler_image_name || 'idea-scheduler-pbs' }} + run: | + set -euo pipefail + VERSION=$(tr -d '[:space:]' < IDEA_VERSION.txt) + ECR_REGISTRY="${ECR_REGISTRY:-$(sed -nE 's|^IDEA_DOCKER_REPO_DEFAULT="([^/]+/[^/]+)/.*"$|\1|p' idea-admin.sh)}" + if [ -z "${ECR_REGISTRY}" ]; then + echo "::error::Could not determine the public registry from idea-admin.sh." + exit 1 + fi + cp "dist/all-${VERSION}.tar.gz" "deployment/ecr/idea-scheduler-pbs/all-${VERSION}.tar.gz" + cp "dist/all-${VERSION}.tar.gz" "deployment/ecr/idea-control-plane/all-${VERSION}.tar.gz" + cp "dist/idea-dcv-connection-gateway-${VERSION}.tar.gz" "deployment/ecr/idea-control-plane/idea-dcv-connection-gateway-${VERSION}.tar.gz" + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "${ECR_REGISTRY}" + docker buildx create --use --platform linux/amd64,linux/arm64 --name idea-images-builder + docker buildx inspect --bootstrap + docker buildx build --push --platform linux/amd64,linux/arm64 \ + --build-arg IDEA_VERSION="${VERSION}" \ + --cache-from type=gha,scope=idea-scheduler-pbs \ + --cache-to type=gha,mode=max,scope=idea-scheduler-pbs \ + -t "${ECR_REGISTRY}/${SCHEDULER_IMAGE_NAME}:v${VERSION}" \ + deployment/ecr/idea-scheduler-pbs + docker buildx build --push --platform linux/amd64,linux/arm64 \ + --build-arg IDEA_VERSION="${VERSION}" \ + --build-arg PBS_IMAGE="${ECR_REGISTRY}/${SCHEDULER_IMAGE_NAME}:v${VERSION}" \ + --build-context dcv-packages=/tmp/dcv-packages \ + --cache-from type=gha,scope=idea-control-plane \ + --cache-to type=gha,mode=max,scope=idea-control-plane \ + -t "${ECR_REGISTRY}/${CONTROL_PLANE_IMAGE_NAME}:v${VERSION}" \ + -t "${ECR_REGISTRY}/${CONTROL_PLANE_IMAGE_NAME}:${VERSION}" \ + -t "${ECR_REGISTRY}/${CONTROL_PLANE_IMAGE_NAME}:latest" \ + -f deployment/ecr/idea-control-plane/Dockerfile . + SMOKE_ROOT=$(mktemp -d) + trap 'rm -rf "$SMOKE_ROOT"' EXIT + mkdir -p "$SMOKE_ROOT/config" + cp source/idea/ideactl/test/shell-path/values.yml "$SMOKE_ROOT/values.yml" + IMAGE="${ECR_REGISTRY}/${CONTROL_PLANE_IMAGE_NAME}:v${VERSION}" + docker run --rm --workdir /tmp/work "$IMAGE" ideactl about + docker run --rm \ + --workdir /tmp/work \ + --volume "$SMOKE_ROOT/values.yml:/tmp/values.yml:ro" \ + --volume "$SMOKE_ROOT/config:/tmp/config" \ + "$IMAGE" \ + ideactl config generate \ + --values-file /tmp/values.yml \ + --config-dir /tmp/config \ + --force + test -s "$SMOKE_ROOT/config/config/idea.yml" + publish_ideactl_artifacts: + if: github.event_name == 'push' && github.ref_name == 'main' + needs: + - build_ideactl_artifacts + - build_ideactl_linux_artifact + - build_push_ideactl + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + - name: Download release artifacts + uses: actions/download-artifact@v4 + with: + pattern: ideactl-* + path: release + merge-multiple: true + - name: Verify and combine checksums + working-directory: release + run: | + set -euo pipefail + cat ./*.tar.gz.sha256 | LC_ALL=C sort > SHA256SUMS + sha256sum --check SHA256SUMS + - name: Publish source release + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + VERSION=$(tr -d '[:space:]' < IDEA_VERSION.txt) + TAG="v${VERSION}" + if gh release view "$TAG" >/dev/null 2>&1; then + gh release upload "$TAG" --clobber \ + release/*.tar.gz \ + release/*.tar.gz.sha256 \ + release/SHA256SUMS + else + gh release create "$TAG" \ + release/*.tar.gz \ + release/*.tar.gz.sha256 \ + release/SHA256SUMS \ + --target "$GITHUB_SHA" \ + --title "IDEA ${VERSION}" \ + --notes "Self-contained ideactl executables and checksums for this release." + fi diff --git a/.github/workflows/ideactl_gates.yaml b/.github/workflows/ideactl_gates.yaml new file mode 100644 index 00000000..7c72b031 --- /dev/null +++ b/.github/workflows/ideactl_gates.yaml @@ -0,0 +1,61 @@ +name: ideactl gates +on: + pull_request: + paths-ignore: + - 'docs/**' + push: + branches: + - main + - 'release-*' + paths-ignore: + - 'docs/**' + workflow_dispatch: +permissions: + contents: read +jobs: + gates: + runs-on: ubuntu-large + defaults: + run: + working-directory: source/idea/ideactl + steps: + - uses: actions/checkout@v4 + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: "26.7.0" + cache: npm + cache-dependency-path: source/idea/ideactl/package-lock.json + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Check package version + run: | + PACKAGE_VERSION=$(node --input-type=module -e 'import packageJson from "./package.json" with { type: "json" }; process.stdout.write(packageJson.version)') + RELEASE_VERSION=$(tr -d '[:space:]' < ../../../IDEA_VERSION.txt) + if [ "$PACKAGE_VERSION" != "$RELEASE_VERSION" ]; then + echo "::error::ideactl package version ${PACKAGE_VERSION} does not match IDEA_VERSION.txt ${RELEASE_VERSION}." + exit 1 + fi + - name: Install dependencies + run: npm ci + - name: Build without changing tracked files + run: | + npm run build + git diff --exit-code -- . + - name: Run credential-free gates + id: automated + # A runner has no captured cluster inputs: tools/parity/live, tools/parity/fixtures and + # docs/port are untracked by design, because they hold real cluster data and this + # repository is public. The fixture helper is fail-closed, so without this variable the + # dependent files throw at setup and the suite cannot pass here at all. Declaring the + # checkout public makes them skip instead, and the skip-policy gate records each one with + # its reason so the skips stay visible. This job therefore proves the credential-free half + # only; it is not evidence about anything that needs a captured cluster. + env: + IDEACTL_PUBLIC_CHECKOUT: '1' + run: node scripts/ci-gates.mjs all + - name: Print credential-backed gates after an earlier failure + if: always() && steps.automated.outcome == 'skipped' + run: node scripts/ci-gates.mjs human diff --git a/.github/workflows/lint_build.yaml b/.github/workflows/lint_build.yaml index c3460d49..fd348f5e 100644 --- a/.github/workflows/lint_build.yaml +++ b/.github/workflows/lint_build.yaml @@ -18,14 +18,12 @@ jobs: - name: Check version consistency run: | SH_REV=$(grep -m1 '^IDEA_REVISION=' idea-admin.sh | sed -E 's/.*:-"([^"]+)"\}.*/\1/') - PS1_REV=$(grep -m1 '[$]IDEARevision = if' idea-admin-windows.ps1 | sed -E 's/.*else \{"([^"]+)"\}.*/\1/') VERSION=$(tr -d '[:space:]' < IDEA_VERSION.txt) EXPECTED="v${VERSION}" echo "idea-admin.sh revision: ${SH_REV}" - echo "idea-admin-windows.ps1 revision: ${PS1_REV}" echo "IDEA_VERSION.txt: ${VERSION} (expected revision ${EXPECTED})" - if [ "${SH_REV}" != "${EXPECTED}" ] || [ "${PS1_REV}" != "${EXPECTED}" ]; then - echo "::error::Version mismatch: IDEA_VERSION.txt=${VERSION} (expected revision ${EXPECTED}), idea-admin.sh default=${SH_REV}, idea-admin-windows.ps1 default=${PS1_REV}" + if [ "${SH_REV}" != "${EXPECTED}" ]; then + echo "::error::Version mismatch: IDEA_VERSION.txt=${VERSION} (expected revision ${EXPECTED}), idea-admin.sh default=${SH_REV}" exit 1 fi echo "Version consistency check passed (${EXPECTED})" diff --git a/.github/workflows/sync_docs_branch.yaml b/.github/workflows/sync_docs_branch.yaml index 499577aa..6ccaba54 100644 --- a/.github/workflows/sync_docs_branch.yaml +++ b/.github/workflows/sync_docs_branch.yaml @@ -1,10 +1,8 @@ name: Sync docs branch -# The GitBook space syncs with the gitbook branch, because main only accepts pull requests. -# On every merge to main, and once a day, fast-forward gitbook to main so docs edited in the -# repository reach GitBook. When gitbook is ahead because edits were merged in the GitBook -# editor, open the pull request that carries them to main. The built-in token is enough: -# a pull request it creates triggers no Actions workflows, and the test workflows skip -# docs-only pull requests anyway, while the GitBook preview comes from the GitBook app. +# The GitBook space syncs with the gitbook branch, because main accepts pull requests only. +# Fast-forward gitbook to main so repository edits reach GitBook, and when gitbook is ahead +# because edits were merged in the GitBook editor, open the pull request that carries them +# back. The built-in token is enough: a pull request it creates triggers no workflows. on: push: branches: diff --git a/.gitignore b/.gitignore index ffb61e2f..dce576d6 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ __pycache__/ # IDEA Specific deployment/ecr/idea-administrator/*.tar.gz +deployment/ecr/*/*.tar.gz open-source/ deployment/global-s3-assets/ deployment/regional-s3-assets/ diff --git a/deployment/ecr/idea-control-plane/Dockerfile b/deployment/ecr/idea-control-plane/Dockerfile new file mode 100644 index 00000000..ced6c84f --- /dev/null +++ b/deployment/ecr/idea-control-plane/Dockerfile @@ -0,0 +1,151 @@ +# syntax=docker/dockerfile:1.7 +# Build the control-plane image. IDEA_CONTAINER_ROLE selects the task process. +# Use the repository root as the build context: +# docker buildx build --platform linux/amd64,linux/arm64 \ +# --build-context dcv-packages=/path/to/dcv-packages \ +# -f deployment/ecr/idea-control-plane/Dockerfile --build-arg IDEA_VERSION=26.09.0 . +# Copy the release tarballs into this directory before the build. +# The dcv-packages context contains NICE-GPG-KEY, checksums.txt, the broker RPM, +# both gateway RPMs, and both DCV archives. checksums.txt covers each package +# file, one sha256sum entry per line. +# Each architecture-dependent step declares ARG TARGETARCH inside its stage and reads the +# value the builder supplies. Do not declare TARGETARCH before the first FROM: under the +# dockerfile:1.7 frontend a global declaration with no default shadows the builder's value, +# every stage then sees an empty string, and the build only works if a single architecture is +# passed by hand, which a two-architecture build cannot do. +# Set PBS_IMAGE_REPO and PBS_IMAGE_TAG for a registry copy, or PBS_IMAGE for a full reference. + +ARG IDEA_VERSION +ARG PBS_IMAGE_REPO=idea-scheduler-pbs +ARG PBS_IMAGE_TAG=v${IDEA_VERSION} +ARG PBS_IMAGE=${PBS_IMAGE_REPO}:${PBS_IMAGE_TAG} + +FROM public.ecr.aws/amazonlinux/amazonlinux:2023 AS dcv-runtime + +ARG TARGETARCH + +ENV IDEA_APP_DEPLOY_DIR=/opt/idea/app +ENV PYTHONUNBUFFERED=1 +ENV LC_ALL="en_US.UTF-8" \ + LC_CTYPE="en_US.UTF-8" \ + LANG="en_US.UTF-8" +ENV PATH="/opt/pbs/bin:/opt/pbs/sbin:${PATH}" + +# Install runtime packages for the module, scheduler, broker, and gateway roles. +RUN dnf install -y --setopt=install_weak_deps=False \ + python3.13 python3.13-pip python3.13-devel \ + gcc openldap-devel cyrus-sasl-devel \ + postgresql17-server postgresql17-contrib \ + expat libedit tcl tk libical hwloc-libs \ + adcli krb5-workstation sssd-ad realmd \ + java-11-amazon-corretto-headless \ + nginx openssl nmap-ncat \ + glibc-langpack-en shadow-utils procps-ng iproute hostname sudo util-linux jq tar gzip which awscli-2 curl-minimal \ + && dnf clean all + +# Install the session-manager broker, connection gateway, and web viewer from the +# supplied package context. This avoids a build-time dependency on a public host. +COPY --from=dcv-packages / /tmp/dcv-packages/ +RUN set -eux; \ + case "${TARGETARCH:-}" in \ + amd64) A=x86_64 ;; \ + arm64) A=aarch64 ;; \ + *) echo "TARGETARCH must be amd64 or arm64 (build with buildx); got '${TARGETARCH:-}'" >&2; exit 1 ;; \ + esac; \ + for F in NICE-GPG-KEY checksums.txt nice-dcv-session-manager-broker-amzn2023.noarch.rpm "nice-dcv-connection-gateway-amzn2023.${A}.rpm" "nice-dcv-amzn2023-${A}.tgz"; do \ + test -f "/tmp/dcv-packages/${F}" || { echo "dcv-packages is missing ${F}" >&2; exit 1; }; \ + done; \ + (cd /tmp/dcv-packages && sha256sum -c checksums.txt); \ + rpm --import /tmp/dcv-packages/NICE-GPG-KEY; \ + rpm -K /tmp/dcv-packages/nice-dcv-session-manager-broker-amzn2023.noarch.rpm; \ + dnf install -y /tmp/dcv-packages/nice-dcv-session-manager-broker-amzn2023.noarch.rpm; \ + rpm -K "/tmp/dcv-packages/nice-dcv-connection-gateway-amzn2023.${A}.rpm"; \ + dnf install -y "/tmp/dcv-packages/nice-dcv-connection-gateway-amzn2023.${A}.rpm"; \ + cp "/tmp/dcv-packages/nice-dcv-amzn2023-${A}.tgz" /tmp/dcv.tgz; \ + mkdir -p /tmp/dcv && tar -xzf /tmp/dcv.tgz -C /tmp/dcv --strip-components=1; \ + rpm -K /tmp/dcv/nice-dcv-web-viewer-*.rpm; dnf install -y /tmp/dcv/nice-dcv-web-viewer-*.rpm; \ + rm -rf /tmp/dcv-packages /tmp/dcv.tgz /tmp/dcv; dnf clean all; \ + sed -i 's|> /dev/null|2>\&1|' /usr/share/dcv-session-manager-broker/bin/dcv-session-manager-broker.sh + +FROM ${PBS_IMAGE} AS pbs + +FROM dcv-runtime + +ARG IDEA_VERSION + +COPY --from=pbs /opt/pbs /opt/pbs + +# Install all module packages from the release bundle. The selected role runs at startup. +# Keep the release bundle in /root/.idea/downloads for package commands. +ADD deployment/ecr/idea-control-plane/all-${IDEA_VERSION}.tar.gz /root/.idea/downloads/ +RUN set -eux; \ + for MODULE in idea-cluster-manager idea-virtual-desktop-controller idea-scheduler; do \ + APP_NAME="${MODULE#idea-}"; \ + PKG="/tmp/idea/${MODULE}"; \ + mkdir -p "${PKG}"; \ + tar -xf "/root/.idea/downloads/${MODULE}-${IDEA_VERSION}.tar.gz" -C "${PKG}"; \ + python3.13 -m pip install --no-cache-dir -r "${PKG}/requirements.txt"; \ + python3.13 -m pip install --no-cache-dir "${PKG}"/*-lib.tar.gz; \ + mkdir -p "${IDEA_APP_DEPLOY_DIR}/${APP_NAME}"; \ + cp -r "${PKG}/resources" "${IDEA_APP_DEPLOY_DIR}/${APP_NAME}/"; \ + if [ -d "${PKG}/webapp" ]; then cp -r "${PKG}/webapp" "${IDEA_APP_DEPLOY_DIR}/${APP_NAME}/"; fi; \ + done; \ + rm -rf /root/.cache/pip; \ + mkdir -p "${IDEA_APP_DEPLOY_DIR}/logs" + +# Overlay the gateway web resources. +ADD deployment/ecr/idea-control-plane/idea-dcv-connection-gateway-${IDEA_VERSION}.tar.gz /tmp/idea-gateway/ +RUN cp -a /tmp/idea-gateway/static_resources/. /usr/share/dcv/www/ && rm -rf /tmp/idea-gateway /tmp/idea + +# Build ideactl and include its pinned CDK CLI. +# The build uses the repository path expected by its resource-copy script. +ARG NODE_VERSION=22.23.2 +RUN set -eux; \ + case "${TARGETARCH:-}" in \ + amd64) N=x64 ;; \ + arm64) N=arm64 ;; \ + *) echo "TARGETARCH must be amd64 or arm64 (build with buildx); got '${TARGETARCH:-}'" >&2; exit 1 ;; \ + esac; \ + mkdir -p /usr/local/node; \ + curl -fsSL -o /tmp/node.tar.gz "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${N}.tar.gz"; \ + tar -xzf /tmp/node.tar.gz -C /usr/local/node --strip-components=1 --no-same-owner; \ + rm -f /tmp/node.tar.gz + +ENV PATH="/usr/local/node/bin:/opt/idea/ideactl/node_modules/.bin:${PATH}" +ENV CDK_DISABLE_CLI_TELEMETRY=true + +COPY IDEA_VERSION.txt /idea-build/IDEA_VERSION.txt +COPY source/idea/idea-administrator/resources /idea-build/source/idea/idea-administrator/resources +# scripts/copy-resources.mjs stages this tree into dist/resources/bootstrap. +COPY source/idea/idea-bootstrap /idea-build/source/idea/idea-bootstrap +COPY source/idea/ideactl/package.json source/idea/ideactl/package-lock.json source/idea/ideactl/tsconfig.json source/idea/ideactl/cdk.json /idea-build/source/idea/ideactl/ +COPY source/idea/ideactl/src /idea-build/source/idea/ideactl/src +COPY source/idea/ideactl/scripts /idea-build/source/idea/ideactl/scripts +# The container module's config templates live here until resource ownership moves into the +# package. copy-resources.mjs overlays them onto dist/resources/config. +COPY source/idea/ideactl/resources-ecs /idea-build/source/idea/ideactl/resources-ecs +RUN set -eux; \ + cd /idea-build/source/idea/ideactl; \ + npm ci; \ + npm run build; \ + scripts/build-lambda-zips.sh; \ + npm prune --omit=dev; \ + npm cache clean --force; \ + find node_modules -type f -name '*.d.ts' -delete; \ + mkdir -p /opt/idea; \ + mv /idea-build/source/idea/ideactl /opt/idea/ideactl; \ + rm -rf /idea-build; \ + cd /opt/idea/ideactl; \ + node /opt/idea/ideactl/dist/src/cli/main.js about + +COPY deployment/ecr/idea-control-plane/nginx.conf /etc/nginx/conf.d/default.conf +COPY deployment/ecr/idea-control-plane/entrypoint.sh /opt/idea/entrypoint.sh +COPY deployment/ecr/idea-control-plane/sync_users.py /opt/idea/sync_users.py +COPY deployment/ecr/idea-control-plane/roles /opt/idea/roles +RUN chmod +x /opt/idea/entrypoint.sh /opt/idea/roles/*.sh + +# module 8443; broker 8444/8445/8446 client/agent/gateway and 47100/47500 between brokers; +# gateway 8443 web and QUIC and 8989 health. +EXPOSE 8443 8444 8445 8446 8989 47100 47500 + +ENTRYPOINT ["/opt/idea/entrypoint.sh"] diff --git a/deployment/ecr/idea-control-plane/entrypoint.sh b/deployment/ecr/idea-control-plane/entrypoint.sh new file mode 100644 index 00000000..8ca511c0 --- /dev/null +++ b/deployment/ecr/idea-control-plane/entrypoint.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# +# Run the process selected by IDEA_CONTAINER_ROLE. +set -euo pipefail + +# A role passed as the first argument overrides IDEA_CONTAINER_ROLE. +case "${1:-}" in + ideactl|cluster-manager|vdc|scheduler|dcv-broker|dcv-gateway) ROLE="$1"; shift ;; + *) ROLE="${IDEA_CONTAINER_ROLE:?IDEA_CONTAINER_ROLE is required: ideactl|cluster-manager|vdc|scheduler|dcv-broker|dcv-gateway}" ;; +esac + +# Module packages share an ideaserver command, so run each module's main directly. +run_module() { + exec python3.13 -c "import sys; sys.argv=['ideaserver']; from ${1}.app.app_main import main; sys.exit(main())" +} + +setting() { + aws dynamodb get-item --region "${AWS_DEFAULT_REGION}" \ + --table-name "${IDEA_CLUSTER_NAME}.cluster-settings" \ + --key "{\"key\":{\"S\":\"$1\"}}" --query 'Item.value.S' --output text +} + +# The three module roles bind TLS against one certificate pair under the cluster home on the +# applications file system. This substrate owns creating it: no host bootstrap runs, so on a fresh +# install the pair does not exist and the module raises a file-not-found before it binds its port. +# The roles start together on the same shared volume, so the pair is published under a link that +# elects one writer rather than by two roles each writing half of it. +ensure_app_certs() { + local home certs key crt zone tmp + home="$(setting cluster.home_dir)" + case "${home}" in ''|None) echo "[entrypoint] cluster.home_dir is not set" >&2; return 1 ;; esac + certs="${home}/certs" + key="${certs}/idea.key" + crt="${certs}/idea.crt" + if [[ -s "${key}" && -s "${crt}" ]]; then + return 0 + fi + zone="$(setting cluster.route53.private_hosted_zone_name)" + case "${zone}" in ''|None) echo "[entrypoint] cluster.route53.private_hosted_zone_name is not set" >&2; return 1 ;; esac + install -d -m 700 "${certs}" + tmp="$(mktemp -d "${certs}/.new.XXXXXX")" + openssl req -new -newkey rsa:4096 -days 3650 -nodes -x509 \ + -subj "/C=US/ST=California/L=Sunnyvale/CN=*.${zone}" \ + -keyout "${tmp}/idea.key" -out "${tmp}/idea.crt" 2>/dev/null + chmod 600 "${tmp}/idea.key" + # A hard link fails when the name is taken, so the first role to get there publishes the pair it + # generated and the rest wait for that one. + if ln "${tmp}/idea.key" "${key}" 2>/dev/null; then + mv -f "${tmp}/idea.crt" "${crt}" + echo "[entrypoint] generated the application certificate pair in ${certs}" + else + for _ in $(seq 1 60); do + [[ -s "${crt}" ]] && break + sleep 1 + done + fi + rm -rf "${tmp}" + if [[ ! -s "${key}" || ! -s "${crt}" ]]; then + echo "[entrypoint] no application certificate pair in ${certs}" >&2 + return 1 + fi +} + +case "${ROLE}" in + cluster-manager|vdc|scheduler) ensure_app_certs ;; +esac + +case "${ROLE}" in + ideactl) exec node /opt/idea/ideactl/dist/src/cli/main.js "$@" ;; + cluster-manager) run_module ideaclustermanager ;; + vdc) run_module ideavirtualdesktopcontroller ;; + scheduler) exec /opt/idea/roles/scheduler.sh ;; + dcv-broker) exec /opt/idea/roles/broker.sh ;; + dcv-gateway) exec /opt/idea/roles/gateway.sh ;; + *) echo "[entrypoint] unknown IDEA_CONTAINER_ROLE=${ROLE}" >&2; exit 1 ;; +esac diff --git a/deployment/ecr/idea-control-plane/nginx.conf b/deployment/ecr/idea-control-plane/nginx.conf new file mode 100644 index 00000000..6aeb21dd --- /dev/null +++ b/deployment/ecr/idea-control-plane/nginx.conf @@ -0,0 +1,5 @@ +server { + listen 80; + listen [::]:80; + root /usr/share/dcv/www; +} diff --git a/deployment/ecr/idea-control-plane/roles/broker.sh b/deployment/ecr/idea-control-plane/roles/broker.sh new file mode 100644 index 00000000..6a61f4ed --- /dev/null +++ b/deployment/ecr/idea-control-plane/roles/broker.sh @@ -0,0 +1,101 @@ +#!/bin/bash +# +# Render broker settings, register the authorization server, and run the broker. +# +# Required: IDEA_CLUSTER_NAME, IDEA_MODULE_ID (the virtual desktop module id, for the +# DynamoDB table prefix), AWS_DEFAULT_REGION, and one of +# IDEA_SERVICE_DISCOVERY_NAME (a name resolving to every broker; the ECS +# module sets it), IDEA_BROKER_DISCOVERY_ADDRESSES (host:port list) or +# IDEA_BROKER_CLIENT_TARGET_GROUP_ARN (the target group the client port is +# registered in; finds EC2 instance targets only) +# Optional: IDEA_COGNITO_PROVIDER_URL (read from the cluster settings when unset), +# IDEA_BROKER_CONF_FILE (where the rendered broker properties are written) +# +# The broker runs on a Java virtual machine in a task network namespace with one address family. +# JAVA_TOOL_OPTIONS carries -Djava.net.preferIPv4Stack=true from the task definition; the virtual +# machine reads that variable itself, so this script passes the environment through untouched. + +set -euo pipefail + +log() { echo "[entrypoint] $*"; } + +: "${IDEA_CLUSTER_NAME:?}" "${IDEA_MODULE_ID:?}" "${AWS_DEFAULT_REGION:?}" + +# Use service discovery for tasks registered by IP. +IDEA_BROKER_DISCOVERY_ADDRESSES="${IDEA_BROKER_DISCOVERY_ADDRESSES:-${IDEA_SERVICE_DISCOVERY_NAME:+${IDEA_SERVICE_DISCOVERY_NAME}:47500}}" +if [ -n "${IDEA_BROKER_DISCOVERY_ADDRESSES}" ]; then + DISCOVERY="broker-to-broker-discovery-addresses = ${IDEA_BROKER_DISCOVERY_ADDRESSES}" + if [ -n "${IDEA_SERVICE_DISCOVERY_NAME:-}" ]; then + # Wait for service discovery before the broker starts. + for _ in $(seq 30); do getent hosts "${IDEA_SERVICE_DISCOVERY_NAME}" >/dev/null && break; sleep 2; done + log "${IDEA_SERVICE_DISCOVERY_NAME} resolves to: $(getent hosts "${IDEA_SERVICE_DISCOVERY_NAME}" | awk '{print $1}' | tr '\n' ' ')" + fi +else + : "${IDEA_BROKER_CLIENT_TARGET_GROUP_ARN:?set IDEA_BROKER_DISCOVERY_ADDRESSES or IDEA_BROKER_CLIENT_TARGET_GROUP_ARN}" + DISCOVERY="broker-to-broker-discovery-aws-region = ${AWS_DEFAULT_REGION} +broker-to-broker-discovery-aws-alb-target-group-arn = ${IDEA_BROKER_CLIENT_TARGET_GROUP_ARN}" +fi + +setting() { + aws dynamodb get-item --region "${AWS_DEFAULT_REGION}" \ + --table-name "${IDEA_CLUSTER_NAME}.cluster-settings" \ + --key "{\"key\":{\"S\":\"$1\"}}" \ + --query "Item.value.S || Item.value.N" --output text 2>/dev/null | grep -v '^None$' || echo "$2" +} + +CLIENT_PORT="$(setting virtual-desktop-controller.dcv_broker.client_communication_port 8444)" +AGENT_PORT="$(setting virtual-desktop-controller.dcv_broker.agent_communication_port 8445)" +GATEWAY_PORT="$(setting virtual-desktop-controller.dcv_broker.gateway_communication_port 8446)" +TOKEN_MINUTES="$(setting virtual-desktop-controller.dcv_broker.session_token_validity 1440)" +RCU="$(setting virtual-desktop-controller.dcv_broker.dynamodb_table.read_capacity.min_units 5)" +WCU="$(setting virtual-desktop-controller.dcv_broker.dynamodb_table.write_capacity.min_units 5)" +PROVIDER_URL="${IDEA_COGNITO_PROVIDER_URL:-$(setting identity-provider.cognito.provider_url "")}" +: "${PROVIDER_URL:?identity-provider.cognito.provider_url is not set}" + +CONF="${IDEA_BROKER_CONF_FILE:-/etc/dcv-session-manager-broker/session-manager-broker.properties}" +cat > "${CONF}" <) +# Optional: DCV_BROKER_GATEWAY_PORT (8446), DCV_GATEWAY_LOG_LEVEL (info) + +set -euo pipefail + +log() { echo "[entrypoint] $*"; } + +: "${DCV_GATEWAY_CERT_PEM:?}" "${DCV_GATEWAY_KEY_PEM:?}" "${IDEA_INTERNAL_ALB_ENDPOINT:?}" +BROKER_PORT="${DCV_BROKER_GATEWAY_PORT:-8446}" + +CERTS=/etc/dcv-connection-gateway/certs +install -d -m 700 -o dcvcgw -g dcvcgw "${CERTS}" +printf '%s\n' "${DCV_GATEWAY_CERT_PEM}" > "${CERTS}/default_cert.pem" +printf '%s\n' "${DCV_GATEWAY_KEY_PEM}" | openssl pkcs8 -topk8 -inform PEM -outform PEM -nocrypt -out "${CERTS}/default_key_pkcs8.pem" +chmod 600 "${CERTS}"/default_*.pem +chown dcvcgw:dcvcgw "${CERTS}"/default_*.pem +unset DCV_GATEWAY_CERT_PEM DCV_GATEWAY_KEY_PEM + +# Every listener binds IPv4 only. The task network namespace has one address family, so a bind on +# "::" fails and the health port the load balancer checks would never come up. +cat > /etc/dcv-connection-gateway/dcv-connection-gateway.conf <..local) +# Optional: IDEA_ROUTE53_ZONE_ID (when set, the record is pointed at this task's address) + +set -euo pipefail + +log() { echo "[entrypoint] $*"; } + +: "${IDEA_SCHEDULER_DNS_NAME:?IDEA_SCHEDULER_DNS_NAME is required}" +PBS_HOME="${PBS_HOME:-/var/spool/pbs}" +SERVER_NAME="${IDEA_SCHEDULER_DNS_NAME%%.*}" + +# Use task metadata when available and reject link-local addresses. +TASK_IP="" +if [[ -n "${ECS_CONTAINER_METADATA_URI_V4:-}" ]]; then + TASK_IP="$(curl -fsS --max-time 3 "${ECS_CONTAINER_METADATA_URI_V4}/task" \ + | python3 -c 'import json,sys; t=json.load(sys.stdin); print(t["Containers"][0]["Networks"][0]["IPv4Addresses"][0])' 2>/dev/null || true)" +fi +if [[ -z "${TASK_IP}" ]]; then + TASK_IP="$(ip -4 -o addr show scope global | awk '{print $4}' | cut -d/ -f1 | grep -v '^169\.254\.' | head -1)" +fi +: "${TASK_IP:?could not determine the task address}" +case "${TASK_IP}" in 169.254.*) echo "[entrypoint] refusing link-local task address ${TASK_IP}" >&2; exit 1;; esac +log "task ip ${TASK_IP}, pbs server name ${SERVER_NAME}" + +# Put the scheduler name first in /etc/hosts for reverse lookup. +if ! grep -q " ${SERVER_NAME}\$" /etc/hosts 2>/dev/null; then + { echo "${TASK_IP} ${IDEA_SCHEDULER_DNS_NAME} ${SERVER_NAME}"; cat /etc/hosts; } > /etc/hosts.new + cat /etc/hosts.new > /etc/hosts + rm -f /etc/hosts.new +fi + +# Set the datastore user for a new PBS_HOME. +PBS_DATA_SERVICE_USER="${PBS_DATA_SERVICE_USER:-postgres}" + +cat > /etc/pbs.conf < ${TASK_IP}" + cat > /tmp/rr.json </dev/null +fi + +# Create the datastore only when PBS_HOME is new. +if [[ ! -d "${PBS_HOME}/datastore" ]]; then + log "first start: pbs_habitat will create PBS_HOME at ${PBS_HOME}" + PBS_CREATE=1 +else + log "reusing existing PBS_HOME at ${PBS_HOME}" + PBS_CREATE=0 +fi + +# Create the PostgreSQL runtime directory. +install -d -m 755 -o postgres -g postgres /run/postgresql + +# Initialize PBS_HOME and its datastore. +/opt/pbs/libexec/pbs_habitat || true + +# Configure a new PBS server before its daemons start. +setting() { + aws dynamodb get-item --region "${AWS_DEFAULT_REGION}" \ + --table-name "${IDEA_CLUSTER_NAME}.cluster-settings" \ + --key "{\"key\":{\"S\":\"$1\"}}" \ + --query "Item.value.S || Item.value.N" --output text 2>/dev/null | grep -v '^None$' || echo "$2" +} +# Use a marker so incomplete configuration runs again. +MARKER="${PBS_HOME}/.idea-server-configured" +if [[ ! -f "${MARKER}" ]]; then + log "applying the scheduler's PBS server configuration" + cat > "${PBS_HOME}/server_priv/resourcedef" <<'RESOURCEDEF' +anonymous_metrics type=string +availability_zone type=string +availability_zone_id type=string +base_os type=string +compute_node type=string flag=h +efa_support type=string +error_message type=string +force_ri type=string +fsx_lustre type=string +fsx_lustre_deployment_type type=string +fsx_lustre_per_unit_throughput type=string +fsx_lustre_size type=string +ht_support type=string +instance_profile type=string +instance_ami type=string +instance_id type=string +instance_type type=string +instance_type_used type=string +keep_ebs type=string +placement_group type=string +root_size type=string +scratch_iops type=string +scratch_size type=string +security_groups type=string +spot_allocation_count type=string +spot_allocation_strategy type=string +spot_price type=string +stack_id type=string +subnet_id type=string +system_metrics type=string +queue_type type=string +job_id type=string +job_group type=string +job_uid type=string +provisioning_time type=string +dry_run type=string +cluster_name type=string +cluster_version type=string +scaling_mode type=string +lifecycle type=string +tenancy type=string +spot_fleet_request type=string +auto_scaling_group type=string +keep_forever type=string +terminate_when_idle type=string +launch_time type=string +capacity_added type=string +job_started_email_template type=string +job_completed_email_template type=string +RESOURCEDEF + grep -q "compute_node" "${PBS_HOME}/sched_priv/sched_config" || \ + sed -i 's/resources: "ncpus, mem, arch, host, vnode, aoe, eoe"/resources: "ncpus, mem, arch, host, vnode, aoe, eoe, compute_node"/' "${PBS_HOME}/sched_priv/sched_config" + printf 'PATH=/bin:/usr/bin\nIDEA_SCHEDULER_UNIX_SOCKET=/run/idea.sock\n' > "${PBS_HOME}/pbs_environment" +fi + +log "starting pbs daemons (create=${PBS_CREATE})" +/opt/pbs/libexec/pbs_init.d start + +# Fail if the PBS server does not start. +for _ in $(seq 1 30); do + if /opt/pbs/bin/qstat -B >/dev/null 2>&1; then + log "pbs server is answering" + break + fi + sleep 5 +done +if ! /opt/pbs/bin/qstat -B >/dev/null 2>&1; then + log "pbs server did not come up" + exit 1 +fi + +# Keep the scheduler host aligned with the stable name. +/opt/pbs/bin/qmgr -c "set sched default sched_host = ${IDEA_SCHEDULER_DNS_NAME}" + +# Allow execution hosts time to reconnect after a replacement. +/opt/pbs/bin/qmgr -c "set server node_fail_requeue = ${PBS_NODE_FAIL_REQUEUE:-600}" + +# A person, not this tool, closes admission before a migration and reopens it afterwards. Those +# are qmgr operations and qmgr needs the caller in the server's managers list, which is otherwise +# the software default of root on this server's own host -- and this server's host is now a task +# with no shell into it. The bastion host keeps its batch client and stays an instance, so putting +# it in the list gives those commands an authorised client over the batch protocol. +# +# It must be the EC2 private DNS name: the server matches the name it reverse-resolves the +# caller's address to, not the private-zone alias in bastion-host.hostname. An absent row means no +# bastion module, so no grant. `|| true` because re-adding an existing entry must not fail a start. +# +# Known ceiling: the entry names an address-derived host name, so after the bastion is replaced +# the grant is stale until this task next starts, and the superseded entry is left behind. Prune +# or re-resolve here if either becomes a problem. +BASTION_PRIVATE_DNS_NAME="$(setting bastion-host.private_dns_name '')" +if [[ -n "${BASTION_PRIVATE_DNS_NAME}" ]]; then + log "granting qmgr manager rights to root@${BASTION_PRIVATE_DNS_NAME}" + /opt/pbs/bin/qmgr -c "set server managers += root@${BASTION_PRIVATE_DNS_NAME}" || true +fi + +log "starting pbs_sched" +/opt/pbs/sbin/pbs_sched + +# Ask execution hosts to refresh the scheduler address. +if [[ -n "${IDEA_ROUTE53_ZONE_ID:-}" ]]; then + log "asking execution hosts to re-read their configuration for the new server address" + aws ssm send-command --region "${AWS_DEFAULT_REGION}" \ + --targets "Key=tag:idea:ClusterName,Values=${IDEA_CLUSTER_NAME}" "Key=tag:idea:NodeType,Values=compute-node" \ + --document-name AWS-RunShellScript \ + --parameters "commands=[\"resolvectl flush-caches 2>/dev/null || true\",\"for i in \$(seq 1 18); do getent hosts ${IDEA_SCHEDULER_DNS_NAME} | grep -q '^${TASK_IP} ' && break; sleep 5; done\",\"pkill -HUP -x pbs_mom || true\"]" \ + --comment "scheduler address changed to ${TASK_IP}" \ + --query Command.CommandId --output text 2>&1 | sed "s/^/[entrypoint] ssm command: /" || true +fi + +if [[ ! -f "${MARKER}" ]]; then + log "server attributes, default queue and hooks" + /opt/pbs/bin/qmgr -c "set server flatuid = $(setting scheduler.openpbs.server.flatuid true)" + /opt/pbs/bin/qmgr -c "set server job_history_enable = $(setting scheduler.openpbs.server.job_history_enable 1)" + /opt/pbs/bin/qmgr -c "set server job_history_duration = $(setting scheduler.openpbs.server.job_history_duration 72:00:00)" + /opt/pbs/bin/qmgr -c "set server scheduler_iteration = $(setting scheduler.openpbs.server.scheduler_iteration 30)" + /opt/pbs/bin/qmgr -c "set server max_concurrent_provision = $(setting scheduler.openpbs.server.max_concurrent_provision 5000)" + /opt/pbs/bin/qmgr -c "create queue normal" 2>/dev/null || true + /opt/pbs/bin/qmgr -c "set queue normal queue_type = Execution" + /opt/pbs/bin/qmgr -c "set queue normal started = True" + /opt/pbs/bin/qmgr -c "set queue normal enabled = True" + /opt/pbs/bin/qmgr -c "set server default_queue = normal" + # Register the scheduler event hooks. + HOOKS="${IDEA_APP_DEPLOY_DIR}/scheduler/resources/openpbs/hooks" + /opt/pbs/bin/qmgr -c "create hook validate_job event='queuejob,modifyjob,movejob'" 2>/dev/null || true + /opt/pbs/bin/qmgr -c "import hook validate_job application/x-python default ${HOOKS}/openpbs_hook_handler.py" + /opt/pbs/bin/qmgr -c "create hook job_status event='runjob,execjob_begin,execjob_end'" 2>/dev/null || true + /opt/pbs/bin/qmgr -c "import hook job_status application/x-python default ${HOOKS}/openpbs_hook_handler.py" + /opt/pbs/bin/qmgr -c "create hook calculate_ncpus event='queuejob'" 2>/dev/null || true + /opt/pbs/bin/qmgr -c "import hook calculate_ncpus application/x-python default ${HOOKS}/calculate_ncpus_hook.py" + /opt/pbs/bin/qmgr -c "set hook calculate_ncpus order=2" + date -u +%Y-%m-%dT%H:%M:%SZ > "${MARKER}" + log "PBS server configured" +fi + +shutdown() { + log "SIGTERM: qterm -t quick" + /opt/pbs/bin/qterm -t quick || true + exit 0 +} +trap shutdown SIGTERM SIGINT + +# Synchronize cluster users before starting the scheduler module. +log "syncing cluster users and groups into the resolver" +python3.13 /opt/idea/sync_users.py --once +python3.13 /opt/idea/sync_users.py & + +log "starting scheduler module" +python3.13 -c "import sys; sys.argv=['ideaserver']; from ideascheduler.app.app_main import main; sys.exit(main())" & +IDEA_PID=$! +wait "${IDEA_PID}" diff --git a/deployment/ecr/idea-control-plane/sync_users.py b/deployment/ecr/idea-control-plane/sync_users.py new file mode 100644 index 00000000..49dcc8cd --- /dev/null +++ b/deployment/ecr/idea-control-plane/sync_users.py @@ -0,0 +1,97 @@ +"""Synchronize cluster users and groups into /etc/passwd and /etc/group. + +Preserve system accounts below 1000 and rewrite higher identifiers from the tables. +""" + +import os +import sys +import time + +import boto3 + +CLUSTER = os.environ['IDEA_CLUSTER_NAME'] +REGION = os.environ.get('AWS_DEFAULT_REGION') or os.environ['AWS_REGION'] +INTERVAL = int(os.environ.get('IDEA_USER_SYNC_INTERVAL', '60')) +SYSTEM_ID_LIMIT = 1000 + +ddb = boto3.resource('dynamodb', region_name=REGION) + + +def scan(table): + t = ddb.Table(f'{CLUSTER}.{table}') + items, kwargs = [], {} + while True: + page = t.scan(**kwargs) + items.extend(page.get('Items', [])) + if 'LastEvaluatedKey' not in page: + return items + kwargs['ExclusiveStartKey'] = page['LastEvaluatedKey'] + + +def write_atomic(path, lines): + tmp = f'{path}.idea-tmp' + with open(tmp, 'w') as f: + f.write('\n'.join(lines) + '\n') + os.chmod(tmp, 0o644) + os.replace(tmp, path) + + +def system_lines(path, id_field): + keep = [] + with open(path) as f: + for line in f: + line = line.rstrip('\n') + parts = line.split(':') + if len(parts) < 3: + continue + try: + ident = int(parts[id_field]) + except ValueError: + continue + if ident < SYSTEM_ID_LIMIT or ident >= 65534: + keep.append(line) + return keep + + +def sync_once(): + users = [u for u in scan('accounts.users') if u.get('enabled', True) and u.get('uid') is not None] + groups = [g for g in scan('accounts.groups') if g.get('enabled', True) and g.get('gid') is not None] + members = {} + for m in scan('accounts.group-members'): + members.setdefault(m['group_name'], set()).add(m['username']) + for u in users: + for g in u.get('additional_groups') or []: + members.setdefault(g, set()).add(u['username']) + + passwd = system_lines('/etc/passwd', 2) + [ + f"{u['username']}:x:{int(u['uid'])}:{int(u['gid'])}:{u['username']}:{u.get('home_dir') or '/'}:{u.get('login_shell') or '/bin/bash'}" + for u in sorted(users, key=lambda u: int(u['uid'])) + if int(u['uid']) >= SYSTEM_ID_LIMIT + ] + group = system_lines('/etc/group', 2) + [ + f"{g['group_name']}:x:{int(g['gid'])}:{','.join(sorted(members.get(g['group_name'], ())))}" + for g in sorted(groups, key=lambda g: int(g['gid'])) + if int(g['gid']) >= SYSTEM_ID_LIMIT + ] + write_atomic('/etc/passwd', passwd) + write_atomic('/etc/group', group) + return len(users), len(groups) + + +def main(): + once = '--once' in sys.argv + while True: + try: + nu, ng = sync_once() + print(f'[sync_users] {nu} users, {ng} groups', flush=True) + except Exception as e: # Retry on the next sync interval. + print(f'[sync_users] failed: {e}', file=sys.stderr, flush=True) + if once: + sys.exit(1) + if once: + return + time.sleep(INTERVAL) + + +if __name__ == '__main__': + main() diff --git a/deployment/ecr/idea-scheduler-pbs/Dockerfile b/deployment/ecr/idea-scheduler-pbs/Dockerfile new file mode 100644 index 00000000..65c29615 --- /dev/null +++ b/deployment/ecr/idea-scheduler-pbs/Dockerfile @@ -0,0 +1,89 @@ +# The scheduler module plus the OpenPBS server, scheduler and comm daemons in one +# container. They belong together: the hooks run inside pbs_server and the module shells +# out to /opt/pbs/bin, so splitting them would put a network hop in the middle of both. +# +# The base is Amazon Linux 2023, the same as the module host, for two reasons that are +# not preferences: +# - OpenPBS 23.06 embeds CPython and needs eval.h, which CPython removed after 3.10. +# AL2023 carries python3-devel at 3.9 for the build and python3.13 for the module, +# so one image serves both. +# - The datastore is a PostgreSQL cluster. Its major version has to match the packages +# here or an existing PBS_HOME cannot be reused. Check datastore/PG_VERSION. +# +# PBS_HOME lives on shared storage, which is the configuration OpenPBS uses for failover; +# pbs_dblock guards it, so exactly one of these runs at a time. A replacement keeps the +# datastore and the server name, and running jobs stay on their execution hosts. + +ARG OPENPBS_VERSION=23.06.06 + +FROM public.ecr.aws/amazonlinux/amazonlinux:2023 AS pbs-builder + +ARG OPENPBS_VERSION +ARG OPENPBS_URL=https://github.com/openpbs/openpbs/archive/v23.06.06.tar.gz +ARG OPENPBS_SHA384=8a4d7f9c326fd1de5c103e700422bc4d49edc9d50f142c033e9e7de8d10d52f5c4f92e902e107b8e89d3e12c147ebef4 + +# python3-devel is 3.9 here, which is what supplies eval.h. +RUN dnf install -y --setopt=install_weak_deps=False \ + gcc gcc-c++ make libtool autoconf automake tar gzip \ + hwloc-devel libX11-devel libXt-devel libedit-devel libical-devel ncurses-devel \ + perl postgresql17-devel python3-devel tcl-devel tk-devel swig \ + expat-devel openssl-devel libXext-devel libXft-devel zlib-devel \ + && dnf clean all + +WORKDIR /build +RUN set -eux; \ + curl -fsSL -o openpbs.tar.gz "${OPENPBS_URL}"; \ + echo "${OPENPBS_SHA384} openpbs.tar.gz" | sha384sum -c -; \ + tar xzf openpbs.tar.gz; \ + cd "openpbs-${OPENPBS_VERSION}"; \ + ./autogen.sh; \ + ./configure PBS_VERSION="${OPENPBS_VERSION}" --prefix=/opt/pbs; \ + make -j"$(nproc)"; \ + make install; \ + /opt/pbs/libexec/pbs_postinstall; \ + chmod 4755 /opt/pbs/sbin/pbs_iff /opt/pbs/sbin/pbs_rcp + + +FROM public.ecr.aws/amazonlinux/amazonlinux:2023 + +ARG IDEA_VERSION +ARG MODULE=idea-scheduler + +ENV IDEA_APP_DEPLOY_DIR=/opt/idea/app +ENV PYTHONUNBUFFERED=1 +ENV LC_ALL="en_US.UTF-8" \ + LC_CTYPE="en_US.UTF-8" \ + LANG="en_US.UTF-8" +ENV PATH="/opt/pbs/bin:/opt/pbs/sbin:${PATH}" + +# postgresql17-server carries the datastore binaries and creates the postgres user the +# PBS database runs as. glibc-langpack-en provides en_US, which the module requires and +# a bare container image does not have. +RUN dnf install -y --setopt=install_weak_deps=False \ + python3.13 python3.13-pip python3.13-devel \ + gcc openldap-devel cyrus-sasl-devel \ + postgresql17-server postgresql17-contrib \ + expat libedit tcl tk libical hwloc-libs \ + glibc-langpack-en shadow-utils procps-ng iproute hostname sudo tar gzip which awscli-2 curl-minimal \ + && dnf clean all + +COPY --from=pbs-builder /opt/pbs /opt/pbs + +ADD all-${IDEA_VERSION}.tar.gz /tmp/idea/ + +RUN set -eux; \ + APP_NAME="${MODULE#idea-}"; \ + PACKAGE_DIR="/tmp/idea/${MODULE}"; \ + mkdir -p "${PACKAGE_DIR}"; \ + tar -xf "/tmp/idea/${MODULE}-${IDEA_VERSION}.tar.gz" -C "${PACKAGE_DIR}"; \ + python3.13 -m pip install --no-cache-dir -r "${PACKAGE_DIR}/requirements.txt"; \ + python3.13 -m pip install --no-cache-dir "${PACKAGE_DIR}"/*-lib.tar.gz; \ + mkdir -p "${IDEA_APP_DEPLOY_DIR}/${APP_NAME}" "${IDEA_APP_DEPLOY_DIR}/logs"; \ + cp -r "${PACKAGE_DIR}/resources" "${IDEA_APP_DEPLOY_DIR}/${APP_NAME}/"; \ + rm -rf /tmp/idea + +COPY entrypoint.sh /opt/idea/entrypoint.sh +COPY sync_users.py /opt/idea/sync_users.py +RUN chmod +x /opt/idea/entrypoint.sh + +ENTRYPOINT ["/opt/idea/entrypoint.sh"] diff --git a/deployment/ecr/idea-scheduler-pbs/entrypoint.sh b/deployment/ecr/idea-scheduler-pbs/entrypoint.sh new file mode 100755 index 00000000..388444e5 --- /dev/null +++ b/deployment/ecr/idea-scheduler-pbs/entrypoint.sh @@ -0,0 +1,282 @@ +#!/bin/bash +# +# Starts the PBS daemons and then runs the scheduler module in the foreground. +# +# PBS is started by its own init script rather than a process supervisor: the daemons +# expect to fork, and ECS is already a supervisor. The container health check runs +# qstat, so a server that dies takes the task with it and ECS replaces it. +# +# Required: IDEA_CLUSTER_NAME, IDEA_MODULE_ID, IDEA_MODULE_SET, AWS_DEFAULT_REGION, +# IDEA_SCHEDULER_DNS_NAME (the stable name every execution host is configured +# with, for example scheduler...local) +# Optional: IDEA_ROUTE53_ZONE_ID (when set, the record is pointed at this task's address) + +set -euo pipefail + +log() { echo "[entrypoint] $*"; } + +: "${IDEA_SCHEDULER_DNS_NAME:?IDEA_SCHEDULER_DNS_NAME is required}" +PBS_HOME="${PBS_HOME:-/var/spool/pbs}" +SERVER_NAME="${IDEA_SCHEDULER_DNS_NAME%%.*}" + +# The task address. On Fargate the task metadata endpoint is authoritative; the +# interface list is not, because the agent's 169.254.172.x link-local interface also +# reports global scope and sorts first. Fall back to the interfaces only off ECS, and +# never accept a link-local address. +TASK_IP="" +if [[ -n "${ECS_CONTAINER_METADATA_URI_V4:-}" ]]; then + TASK_IP="$(curl -fsS --max-time 3 "${ECS_CONTAINER_METADATA_URI_V4}/task" \ + | python3 -c 'import json,sys; t=json.load(sys.stdin); print(t["Containers"][0]["Networks"][0]["IPv4Addresses"][0])' 2>/dev/null || true)" +fi +if [[ -z "${TASK_IP}" ]]; then + TASK_IP="$(ip -4 -o addr show scope global | awk '{print $4}' | cut -d/ -f1 | grep -v '^169\.254\.' | head -1)" +fi +: "${TASK_IP:?could not determine the task address}" +case "${TASK_IP}" in 169.254.*) echo "[entrypoint] refusing link-local task address ${TASK_IP}" >&2; exit 1;; esac +log "task ip ${TASK_IP}, pbs server name ${SERVER_NAME}" + +# PBS resolves its own name and the execution hosts resolve the same one. Mapping it +# locally means the server agrees with what the execution hosts were told. +# First, not appended: the runtime already wrote a line mapping this address to the +# task's own hostname, and the sched identifies itself by reverse lookup of its source +# address. If that comes back as anything but the server name, pbs_server refuses the +# sched's registration (PBSE_BADHOST on RegisterSched) and nothing ever runs. +if ! grep -q " ${SERVER_NAME}\$" /etc/hosts 2>/dev/null; then + { echo "${TASK_IP} ${IDEA_SCHEDULER_DNS_NAME} ${SERVER_NAME}"; cat /etc/hosts; } > /etc/hosts.new + cat /etc/hosts.new > /etc/hosts + rm -f /etc/hosts.new +fi + +# PBS_DATA_SERVICE_USER has no default in pbs_db_utility: it is written into +# /etc/pbs.conf by pbs_postinstall, and a module host then replaces that file with one +# that omits it. That is harmless on a host whose datastore already exists, but a +# container starting from an empty PBS_HOME needs it or the datastore is never created. +PBS_DATA_SERVICE_USER="${PBS_DATA_SERVICE_USER:-postgres}" + +cat > /etc/pbs.conf < ${TASK_IP}" + cat > /tmp/rr.json </dev/null +fi + +# First start on an empty shared PBS_HOME lays down the directory tree and the +# datastore. Afterwards the datastore is what carries the running jobs across a +# replacement, so it is never recreated. +if [[ ! -d "${PBS_HOME}/datastore" ]]; then + log "first start: pbs_habitat will create PBS_HOME at ${PBS_HOME}" + PBS_CREATE=1 +else + log "reusing existing PBS_HOME at ${PBS_HOME}" + PBS_CREATE=0 +fi + +# The datastore's PostgreSQL puts its socket and lock file in /run/postgresql. On a +# host systemd-tmpfiles creates that directory at boot from the package's rule; nothing +# does in a container, and without it postgres exits right after pg_ctl reports +# "server starting", which PBS cannot distinguish from success. +install -d -m 755 -o postgres -g postgres /run/postgresql + +# pbs_habitat runs pbs_postinstall, which lays down the whole PBS_HOME tree (spool, +# server_priv/accounting, sched_priv/sched_config, pbs_environment, db_user) but only +# when PBS_HOME does not exist yet, then creates the datastore. Nothing here pre-creates +# any of it: a partial tree makes postinstall skip population and the server then fails +# chk_file_sec on the pieces that are missing. On an existing PBS_HOME it is a no-op. +/opt/pbs/libexec/pbs_habitat || true + +# First start only: the configuration a module host applies to a new PBS server in +# configure_openpbs_server.jinja2, minus the host-only parts (systemd, the login alias). +# The server reads resourcedef, sched_config and pbs_environment at startup, so these go +# in before the daemons; the qmgr settings need a running server and follow. All of it +# lives in PBS_HOME or the datastore, so a replacement finds it in place. +setting() { + aws dynamodb get-item --region "${AWS_DEFAULT_REGION}" \ + --table-name "${IDEA_CLUSTER_NAME}.cluster-settings" \ + --key "{\"key\":{\"S\":\"$1\"}}" \ + --query "Item.value.S || Item.value.N" --output text 2>/dev/null | grep -v '^None$' || echo "$2" +} +# Gated on a marker rather than on the datastore's absence: a PBS_HOME that exists but was +# never configured (or a task that died mid-way) is configured on the next start. +MARKER="${PBS_HOME}/.idea-server-configured" +if [[ ! -f "${MARKER}" ]]; then + log "applying the scheduler's PBS server configuration" + cat > "${PBS_HOME}/server_priv/resourcedef" <<'RESOURCEDEF' +anonymous_metrics type=string +availability_zone type=string +availability_zone_id type=string +base_os type=string +compute_node type=string flag=h +efa_support type=string +error_message type=string +force_ri type=string +fsx_lustre type=string +fsx_lustre_deployment_type type=string +fsx_lustre_per_unit_throughput type=string +fsx_lustre_size type=string +ht_support type=string +instance_profile type=string +instance_ami type=string +instance_id type=string +instance_type type=string +instance_type_used type=string +keep_ebs type=string +placement_group type=string +root_size type=string +scratch_iops type=string +scratch_size type=string +security_groups type=string +spot_allocation_count type=string +spot_allocation_strategy type=string +spot_price type=string +stack_id type=string +subnet_id type=string +system_metrics type=string +queue_type type=string +job_id type=string +job_group type=string +job_uid type=string +provisioning_time type=string +dry_run type=string +cluster_name type=string +cluster_version type=string +scaling_mode type=string +lifecycle type=string +tenancy type=string +spot_fleet_request type=string +auto_scaling_group type=string +keep_forever type=string +terminate_when_idle type=string +launch_time type=string +capacity_added type=string +job_started_email_template type=string +job_completed_email_template type=string +RESOURCEDEF + grep -q "compute_node" "${PBS_HOME}/sched_priv/sched_config" || \ + sed -i 's/resources: "ncpus, mem, arch, host, vnode, aoe, eoe"/resources: "ncpus, mem, arch, host, vnode, aoe, eoe, compute_node"/' "${PBS_HOME}/sched_priv/sched_config" + printf 'PATH=/bin:/usr/bin\nIDEA_SCHEDULER_UNIX_SOCKET=/run/idea.sock\n' > "${PBS_HOME}/pbs_environment" +fi + +log "starting pbs daemons (create=${PBS_CREATE})" +/opt/pbs/libexec/pbs_init.d start + +# A replacement must not out-race the datastore: fail fast rather than serve a server +# that never came up. +for _ in $(seq 1 30); do + if /opt/pbs/bin/qstat -B >/dev/null 2>&1; then + log "pbs server is answering" + break + fi + sleep 5 +done +if ! /opt/pbs/bin/qstat -B >/dev/null 2>&1; then + log "pbs server did not come up" + exit 1 +fi + +# Every start, not only the first: the sched object's host is persisted in the datastore +# and defaults to whatever host created it (sched_func.c sets it only when unset). The +# sched identifies itself by the stable name, and the server refuses its registration +# with PBSE_BADHOST unless this matches, so pin it to the stable name here. +/opt/pbs/bin/qmgr -c "set sched default sched_host = ${IDEA_SCHEDULER_DNS_NAME}" + +# How long the server tolerates an unreachable execution host before requeuing its jobs. +# The default, 310 s, is shorter than a replacement plus the hosts' re-read, so a slow +# heal would rerun every running job from the start. Ten minutes covers a replacement +# with margin; a host that is really gone just waits that long before its jobs move. +/opt/pbs/bin/qmgr -c "set server node_fail_requeue = ${PBS_NODE_FAIL_REQUEUE:-600}" + +# The sched registers two connections with the server and the server checks the +# sched's address against sched_host on each. Starting it only after that attribute is +# pinned means its first registration is clean; starting it alongside the server (the +# init script default) leaves the server holding a half-registered sched it then rejects +# with PBSE_IVALREQ on every retry. +log "starting pbs_sched" +/opt/pbs/sbin/pbs_sched + +# A mom resolves the server address once, at start, for both the address it connects to +# and the list it authorises server messages against. A replacement task has a new +# address, so every execution host re-reads its configuration or stays down under the +# new server while its jobs keep running. SIGHUP is the mom re-read; running jobs are +# untouched. One tag-targeted command reaches the fleet; on a first start it reaches nothing. +# The re-read resolves the server name through the node's resolver cache, which the mom's own +# retries keep warm with the old address, so the command flushes that cache and waits until +# the name resolves to this task before signalling; otherwise the mom re-authorises the old +# address and rejects the new server for a TTL. +if [[ -n "${IDEA_ROUTE53_ZONE_ID:-}" ]]; then + log "asking execution hosts to re-read their configuration for the new server address" + aws ssm send-command --region "${AWS_DEFAULT_REGION}" \ + --targets "Key=tag:idea:ClusterName,Values=${IDEA_CLUSTER_NAME}" "Key=tag:idea:NodeType,Values=compute-node" \ + --document-name AWS-RunShellScript \ + --parameters "commands=[\"resolvectl flush-caches 2>/dev/null || true\",\"for i in \$(seq 1 18); do getent hosts ${IDEA_SCHEDULER_DNS_NAME} | grep -q '^${TASK_IP} ' && break; sleep 5; done\",\"pkill -HUP -x pbs_mom || true\"]" \ + --comment "scheduler address changed to ${TASK_IP}" \ + --query Command.CommandId --output text 2>&1 | sed "s/^/[entrypoint] ssm command: /" || true +fi + +if [[ ! -f "${MARKER}" ]]; then + log "server attributes, default queue and hooks" + /opt/pbs/bin/qmgr -c "set server flatuid = $(setting scheduler.openpbs.server.flatuid true)" + /opt/pbs/bin/qmgr -c "set server job_history_enable = $(setting scheduler.openpbs.server.job_history_enable 1)" + /opt/pbs/bin/qmgr -c "set server job_history_duration = $(setting scheduler.openpbs.server.job_history_duration 72:00:00)" + /opt/pbs/bin/qmgr -c "set server scheduler_iteration = $(setting scheduler.openpbs.server.scheduler_iteration 30)" + /opt/pbs/bin/qmgr -c "set server max_concurrent_provision = $(setting scheduler.openpbs.server.max_concurrent_provision 5000)" + /opt/pbs/bin/qmgr -c "create queue normal" 2>/dev/null || true + /opt/pbs/bin/qmgr -c "set queue normal queue_type = Execution" + /opt/pbs/bin/qmgr -c "set queue normal started = True" + /opt/pbs/bin/qmgr -c "set queue normal enabled = True" + /opt/pbs/bin/qmgr -c "set server default_queue = normal" + # The hooks are how PBS hands job validation and run/finish events to the module. Same + # three as install_app.sh.jinja2 installs on a module host, from the same shipped files. + HOOKS="${IDEA_APP_DEPLOY_DIR}/scheduler/resources/openpbs/hooks" + /opt/pbs/bin/qmgr -c "create hook validate_job event='queuejob,modifyjob,movejob'" 2>/dev/null || true + /opt/pbs/bin/qmgr -c "import hook validate_job application/x-python default ${HOOKS}/openpbs_hook_handler.py" + /opt/pbs/bin/qmgr -c "create hook job_status event='runjob,execjob_begin,execjob_end'" 2>/dev/null || true + /opt/pbs/bin/qmgr -c "import hook job_status application/x-python default ${HOOKS}/openpbs_hook_handler.py" + /opt/pbs/bin/qmgr -c "create hook calculate_ncpus event='queuejob'" 2>/dev/null || true + /opt/pbs/bin/qmgr -c "import hook calculate_ncpus application/x-python default ${HOOKS}/calculate_ncpus_hook.py" + /opt/pbs/bin/qmgr -c "set hook calculate_ncpus order=2" + date -u +%Y-%m-%dT%H:%M:%SZ > "${MARKER}" + log "PBS server configured" +fi + +# Terminating the server leaves running jobs on their execution hosts; they are picked +# up again when the replacement connects. Anything harsher risks the datastore. +shutdown() { + log "SIGTERM: qterm -t quick" + /opt/pbs/bin/qterm -t quick || true + exit 0 +} +trap shutdown SIGTERM SIGINT + +# The module submits every job as its owner ("su -c qsub"), so cluster users +# must resolve here. There is no directory join in a task; the identities come from the +# cluster's own tables instead. Once before the module starts, then refreshed in the +# background so users created later resolve within a minute. +log "syncing cluster users and groups into the resolver" +python3.13 /opt/idea/sync_users.py --once +python3.13 /opt/idea/sync_users.py & + +log "starting scheduler module" +ideaserver & +IDEA_PID=$! +wait "${IDEA_PID}" diff --git a/deployment/ecr/idea-scheduler-pbs/sync_users.py b/deployment/ecr/idea-scheduler-pbs/sync_users.py new file mode 100644 index 00000000..07271fb5 --- /dev/null +++ b/deployment/ecr/idea-scheduler-pbs/sync_users.py @@ -0,0 +1,104 @@ +"""Keep /etc/passwd and /etc/group in step with the cluster's user and group tables. + +A module host resolves cluster users through the directory (sssd). A task has no +directory join, but the cluster already assigns every uid and gid itself and records +them in DynamoDB, so the same identities can be written straight into the files the +resolver reads. The scheduler needs this because it submits each job with +"su -c qsub", which fails as "no such user" until the owner resolves. + +System accounts (uid or gid below 1000) are left exactly as the image shipped them; +everything at or above that is rewritten from the tables on each pass. +""" + +import os +import sys +import time + +import boto3 + +CLUSTER = os.environ['IDEA_CLUSTER_NAME'] +REGION = os.environ.get('AWS_DEFAULT_REGION') or os.environ['AWS_REGION'] +INTERVAL = int(os.environ.get('IDEA_USER_SYNC_INTERVAL', '60')) +SYSTEM_ID_LIMIT = 1000 + +ddb = boto3.resource('dynamodb', region_name=REGION) + + +def scan(table): + t = ddb.Table(f'{CLUSTER}.{table}') + items, kwargs = [], {} + while True: + page = t.scan(**kwargs) + items.extend(page.get('Items', [])) + if 'LastEvaluatedKey' not in page: + return items + kwargs['ExclusiveStartKey'] = page['LastEvaluatedKey'] + + +def write_atomic(path, lines): + tmp = f'{path}.idea-tmp' + with open(tmp, 'w') as f: + f.write('\n'.join(lines) + '\n') + os.chmod(tmp, 0o644) + os.replace(tmp, path) + + +def system_lines(path, id_field): + keep = [] + with open(path) as f: + for line in f: + line = line.rstrip('\n') + parts = line.split(':') + if len(parts) < 3: + continue + try: + ident = int(parts[id_field]) + except ValueError: + continue + if ident < SYSTEM_ID_LIMIT or ident >= 65534: + keep.append(line) + return keep + + +def sync_once(): + users = [u for u in scan('accounts.users') if u.get('enabled', True) and u.get('uid') is not None] + groups = [g for g in scan('accounts.groups') if g.get('enabled', True) and g.get('gid') is not None] + members = {} + for m in scan('accounts.group-members'): + members.setdefault(m['group_name'], set()).add(m['username']) + for u in users: + for g in u.get('additional_groups') or []: + members.setdefault(g, set()).add(u['username']) + + passwd = system_lines('/etc/passwd', 2) + [ + f"{u['username']}:x:{int(u['uid'])}:{int(u['gid'])}:{u['username']}:{u.get('home_dir') or '/'}:{u.get('login_shell') or '/bin/bash'}" + for u in sorted(users, key=lambda u: int(u['uid'])) + if int(u['uid']) >= SYSTEM_ID_LIMIT + ] + group = system_lines('/etc/group', 2) + [ + f"{g['group_name']}:x:{int(g['gid'])}:{','.join(sorted(members.get(g['group_name'], ())))}" + for g in sorted(groups, key=lambda g: int(g['gid'])) + if int(g['gid']) >= SYSTEM_ID_LIMIT + ] + write_atomic('/etc/passwd', passwd) + write_atomic('/etc/group', group) + return len(users), len(groups) + + +def main(): + once = '--once' in sys.argv + while True: + try: + nu, ng = sync_once() + print(f'[sync_users] {nu} users, {ng} groups', flush=True) + except Exception as e: # keep the loop alive; the next pass may succeed + print(f'[sync_users] failed: {e}', file=sys.stderr, flush=True) + if once: + sys.exit(1) + if once: + return + time.sleep(INTERVAL) + + +if __name__ == '__main__': + main() diff --git a/docs/first-time-users/cluster-operations/uninstall-idea.md b/docs/first-time-users/cluster-operations/uninstall-idea.md index 94a0c5ae..a58a6910 100644 --- a/docs/first-time-users/cluster-operations/uninstall-idea.md +++ b/docs/first-time-users/cluster-operations/uninstall-idea.md @@ -111,9 +111,9 @@ stack: idea-beta-cluster-manager, status: DELETE_IN_PROGRESS stack: idea-beta-cluster, status: DELETE_COMPLETE disabling termination protection for stack: idea-beta-bootstrap terminating cloud formation stack: idea-beta-bootstrap -found cluster s3 bucket: idea-beta-cluster-us-east-2-549172027899 -deleting s3 bucket: idea-beta-cluster-us-east-2-549172027899 for cluster ... -bucket idea-beta-cluster-us-east-2-549172027899 deleted successfully +found cluster s3 bucket: idea-beta-cluster-us-east-2-123456789012 +deleting s3 bucket: idea-beta-cluster-us-east-2-123456789012 for cluster ... +bucket idea-beta-cluster-us-east-2-123456789012 deleted successfully +--------------------------------------------------------------------------+ | Table Name | +--------------------------------------------------------------------------+ diff --git a/docs/first-time-users/cluster-operations/update-idea-cluster/patch-idea-module.md b/docs/first-time-users/cluster-operations/update-idea-cluster/patch-idea-module.md index 432e8ba4..33681db6 100644 --- a/docs/first-time-users/cluster-operations/update-idea-cluster/patch-idea-module.md +++ b/docs/first-time-users/cluster-operations/update-idea-cluster/patch-idea-module.md @@ -71,7 +71,7 @@ searching for applicable ec2 instances ... | i-0f45cb<REDACTED> | idea-prerc-cluster-manager | ip-10-0-211-98.us-east-2.compute.internal | 10.0.211.98 | running | +---------------------+----------------------------+-------------------------------------------+-------------+---------+ ? Are you sure you want to patch the above running ec2 instances for module: cluster-manager? Yes -uploading package: /Users/mcrozes/Solution-for-scale-out-computing-on-aws/dist/idea-cluster-manager-3.0.0.tar.gz to +uploading package: /Users/sampleuser/Solution-for-scale-out-computing-on-aws/dist/idea-cluster-manager-3.0.0.tar.gz to s3://idea-prerc-cluster-us-east-2-<REDACTED>/idea/patches/idea-cluster-manager-3.0.0.tar.gz ... patch command: sudo /bin/bash /root/bootstrap/latest/cluster-manager/install_app.sh s3://idea-prerc-cluster-us-east-2-<REDACTED>/idea/patches/idea-cluster-manager-3.0.0.tar.gz >> /root/bootstrap/logs/patch.log diff --git a/docs/first-time-users/cluster-operations/update-idea-cluster/update-idea-backend-resource.md b/docs/first-time-users/cluster-operations/update-idea-cluster/update-idea-backend-resource.md index b786e124..74c9f292 100644 --- a/docs/first-time-users/cluster-operations/update-idea-cluster/update-idea-backend-resource.md +++ b/docs/first-time-users/cluster-operations/update-idea-cluster/update-idea-backend-resource.md @@ -60,38 +60,38 @@ re-use code assets for lambda: idea_solution_metrics ... -cluster: building assets... -[0%] start: Building 0a9c7e320d724b92457f9df9325d6f0014ba434d71d23d0c20c6afb1ca20dc79:549172027899-us-east-2 -[0%] start: Building e8af6af9ddccbfe39cb70b54a772900ec28e77a49433a8f480db9779ec2a71f1:549172027899-us-east-2 -[0%] start: Building 402db69e73fdd83283c5df7754892d7a47e9a026ad3d019a8e42e1dffd79946b:549172027899-us-east-2 -[0%] start: Building fcdcab8ae7b888ac267e0822f4bfc89d5b97c54892fe6e3359036b79ddefe031:549172027899-us-east-2 -[0%] start: Building 821317b1ea7eb21bcb9aafec5de6c305c1a076d6bea89231bef1e43f9ddc8b93:549172027899-us-east-2 -[0%] start: Building b25e3a038f6877199484a5531e47bfd984d7850dd69ff0d0e72cfc5c4f90c2ab:549172027899-us-east-2 -[0%] start: Building 7169cf61baf93fd8b0b1ab7a7b21bfb3097e58e1516fd70e1e4efb52873b58e9:549172027899-us-east-2 -[14%] success: Built 0a9c7e320d724b92457f9df9325d6f0014ba434d71d23d0c20c6afb1ca20dc79:549172027899-us-east-2 -[28%] success: Built e8af6af9ddccbfe39cb70b54a772900ec28e77a49433a8f480db9779ec2a71f1:549172027899-us-east-2 -[42%] success: Built 402db69e73fdd83283c5df7754892d7a47e9a026ad3d019a8e42e1dffd79946b:549172027899-us-east-2 -[57%] success: Built fcdcab8ae7b888ac267e0822f4bfc89d5b97c54892fe6e3359036b79ddefe031:549172027899-us-east-2 -[71%] success: Built 821317b1ea7eb21bcb9aafec5de6c305c1a076d6bea89231bef1e43f9ddc8b93:549172027899-us-east-2 -[85%] success: Built b25e3a038f6877199484a5531e47bfd984d7850dd69ff0d0e72cfc5c4f90c2ab:549172027899-us-east-2 -[100%] success: Built 7169cf61baf93fd8b0b1ab7a7b21bfb3097e58e1516fd70e1e4efb52873b58e9:549172027899-us-east-2 +[0%] start: Building 0a9c7e320d724b92457f9df9325d6f0014ba434d71d23d0c20c6afb1ca20dc79:123456789012-us-east-2 +[0%] start: Building e8af6af9ddccbfe39cb70b54a772900ec28e77a49433a8f480db9779ec2a71f1:123456789012-us-east-2 +[0%] start: Building 402db69e73fdd83283c5df7754892d7a47e9a026ad3d019a8e42e1dffd79946b:123456789012-us-east-2 +[0%] start: Building fcdcab8ae7b888ac267e0822f4bfc89d5b97c54892fe6e3359036b79ddefe031:123456789012-us-east-2 +[0%] start: Building 821317b1ea7eb21bcb9aafec5de6c305c1a076d6bea89231bef1e43f9ddc8b93:123456789012-us-east-2 +[0%] start: Building b25e3a038f6877199484a5531e47bfd984d7850dd69ff0d0e72cfc5c4f90c2ab:123456789012-us-east-2 +[0%] start: Building 7169cf61baf93fd8b0b1ab7a7b21bfb3097e58e1516fd70e1e4efb52873b58e9:123456789012-us-east-2 +[14%] success: Built 0a9c7e320d724b92457f9df9325d6f0014ba434d71d23d0c20c6afb1ca20dc79:123456789012-us-east-2 +[28%] success: Built e8af6af9ddccbfe39cb70b54a772900ec28e77a49433a8f480db9779ec2a71f1:123456789012-us-east-2 +[42%] success: Built 402db69e73fdd83283c5df7754892d7a47e9a026ad3d019a8e42e1dffd79946b:123456789012-us-east-2 +[57%] success: Built fcdcab8ae7b888ac267e0822f4bfc89d5b97c54892fe6e3359036b79ddefe031:123456789012-us-east-2 +[71%] success: Built 821317b1ea7eb21bcb9aafec5de6c305c1a076d6bea89231bef1e43f9ddc8b93:123456789012-us-east-2 +[85%] success: Built b25e3a038f6877199484a5531e47bfd984d7850dd69ff0d0e72cfc5c4f90c2ab:123456789012-us-east-2 +[100%] success: Built 7169cf61baf93fd8b0b1ab7a7b21bfb3097e58e1516fd70e1e4efb52873b58e9:123456789012-us-east-2 -cluster: assets built -cluster: deploying... -[0%] start: Publishing 0a9c7e320d724b92457f9df9325d6f0014ba434d71d23d0c20c6afb1ca20dc79:549172027899-us-east-2 -[0%] start: Publishing e8af6af9ddccbfe39cb70b54a772900ec28e77a49433a8f480db9779ec2a71f1:549172027899-us-east-2 -[0%] start: Publishing 402db69e73fdd83283c5df7754892d7a47e9a026ad3d019a8e42e1dffd79946b:549172027899-us-east-2 -[0%] start: Publishing fcdcab8ae7b888ac267e0822f4bfc89d5b97c54892fe6e3359036b79ddefe031:549172027899-us-east-2 -[0%] start: Publishing 821317b1ea7eb21bcb9aafec5de6c305c1a076d6bea89231bef1e43f9ddc8b93:549172027899-us-east-2 -[0%] start: Publishing b25e3a038f6877199484a5531e47bfd984d7850dd69ff0d0e72cfc5c4f90c2ab:549172027899-us-east-2 -[0%] start: Publishing 7169cf61baf93fd8b0b1ab7a7b21bfb3097e58e1516fd70e1e4efb52873b58e9:549172027899-us-east-2 -[14%] success: Published 402db69e73fdd83283c5df7754892d7a47e9a026ad3d019a8e42e1dffd79946b:549172027899-us-east-2 -[28%] success: Published fcdcab8ae7b888ac267e0822f4bfc89d5b97c54892fe6e3359036b79ddefe031:549172027899-us-east-2 -[42%] success: Published 0a9c7e320d724b92457f9df9325d6f0014ba434d71d23d0c20c6afb1ca20dc79:549172027899-us-east-2 -[57%] success: Published 821317b1ea7eb21bcb9aafec5de6c305c1a076d6bea89231bef1e43f9ddc8b93:549172027899-us-east-2 -[71%] success: Published e8af6af9ddccbfe39cb70b54a772900ec28e77a49433a8f480db9779ec2a71f1:549172027899-us-east-2 -[85%] success: Published b25e3a038f6877199484a5531e47bfd984d7850dd69ff0d0e72cfc5c4f90c2ab:549172027899-us-east-2 -[100%] success: Published 7169cf61baf93fd8b0b1ab7a7b21bfb3097e58e1516fd70e1e4efb52873b58e9:549172027899-us-east-2 +[0%] start: Publishing 0a9c7e320d724b92457f9df9325d6f0014ba434d71d23d0c20c6afb1ca20dc79:123456789012-us-east-2 +[0%] start: Publishing e8af6af9ddccbfe39cb70b54a772900ec28e77a49433a8f480db9779ec2a71f1:123456789012-us-east-2 +[0%] start: Publishing 402db69e73fdd83283c5df7754892d7a47e9a026ad3d019a8e42e1dffd79946b:123456789012-us-east-2 +[0%] start: Publishing fcdcab8ae7b888ac267e0822f4bfc89d5b97c54892fe6e3359036b79ddefe031:123456789012-us-east-2 +[0%] start: Publishing 821317b1ea7eb21bcb9aafec5de6c305c1a076d6bea89231bef1e43f9ddc8b93:123456789012-us-east-2 +[0%] start: Publishing b25e3a038f6877199484a5531e47bfd984d7850dd69ff0d0e72cfc5c4f90c2ab:123456789012-us-east-2 +[0%] start: Publishing 7169cf61baf93fd8b0b1ab7a7b21bfb3097e58e1516fd70e1e4efb52873b58e9:123456789012-us-east-2 +[14%] success: Published 402db69e73fdd83283c5df7754892d7a47e9a026ad3d019a8e42e1dffd79946b:123456789012-us-east-2 +[28%] success: Published fcdcab8ae7b888ac267e0822f4bfc89d5b97c54892fe6e3359036b79ddefe031:123456789012-us-east-2 +[42%] success: Published 0a9c7e320d724b92457f9df9325d6f0014ba434d71d23d0c20c6afb1ca20dc79:123456789012-us-east-2 +[57%] success: Published 821317b1ea7eb21bcb9aafec5de6c305c1a076d6bea89231bef1e43f9ddc8b93:123456789012-us-east-2 +[71%] success: Published e8af6af9ddccbfe39cb70b54a772900ec28e77a49433a8f480db9779ec2a71f1:123456789012-us-east-2 +[85%] success: Published b25e3a038f6877199484a5531e47bfd984d7850dd69ff0d0e72cfc5c4f90c2ab:123456789012-us-east-2 +[100%] success: Published 7169cf61baf93fd8b0b1ab7a7b21bfb3097e58e1516fd70e1e4efb52873b58e9:123456789012-us-east-2 -cluster: creating CloudFormation changeset... -cluster | 0/3 | 3:16:48 PM | UPDATE_IN_PROGRESS | AWS::CloudFormation::Stack | -cluster User Initiated -cluster | 0/3 | 3:16:55 PM | UPDATE_IN_PROGRESS | AWS::Lambda::Function | solution-metrics (solutionmetricsAE489078) diff --git a/docs/help-and-support/faq.md b/docs/help-and-support/faq.md index eac9e35a..763296f9 100644 --- a/docs/help-and-support/faq.md +++ b/docs/help-and-support/faq.md @@ -47,15 +47,15 @@ Alternatively, you can run the following `idea-admin.sh` command: Use `ideactl` If you cannot receive email from Cognito due to IT restriction. Login to the Cluster Manager EC2 instance and run `ideactl accounts create-user`
# Make sure to run this command as root on the CLUSTER Manager
-# ideactl accounts create-user --email "mcrozes@myemail.com" --password "Password123@" --username "mcrozes2" --sudo --email-verified
+# ideactl accounts create-user --email "sampleuser@example.com" --password "Password123@" --username "sampleuser2" --sudo --email-verified
 {
-  "username": "mcrozes2",
-  "email": "mcrozes@myemail.com",
+  "username": "sampleuser2",
+  "email": "sampleuser@example.com",
   "uid": 5068,
   "gid": 5077,
-  "group_name": "mcrozes2-user-group",
+  "group_name": "sampleuser2-user-group",
   "login_shell": "/bin/bash",
-  "home_dir": "/data/home/mcrozes2",
+  "home_dir": "/data/home/sampleuser2",
   "sudo": true,
   "status": "CONFIRMED",
   "enabled": true,
diff --git a/docs/modules/cluster-manager/data-sharing-between-users.md b/docs/modules/cluster-manager/data-sharing-between-users.md
index 23cf25da..30a3705b 100644
--- a/docs/modules/cluster-manager/data-sharing-between-users.md
+++ b/docs/modules/cluster-manager/data-sharing-between-users.md
@@ -22,7 +22,7 @@ description: How to share your result files on IDEA
 
 
-* Open SSH client > connect to HPC IDEA Cluster. Note: If you haven't set this up yet, follow the tutorial here: [https://cidea.cfsenergy.com/#/home/ssh-access](https://cidea.cfsenergy.com/#/home/ssh-access) +* Open SSH client > connect to HPC IDEA Cluster. Note: If you haven't set this up yet, follow the SSH access tutorial in your IDEA cluster's web console (Home > SSH Access).
diff --git a/docs/modules/cluster-manager/menu/users-management.md b/docs/modules/cluster-manager/menu/users-management.md index 8fb546f3..0589bb40 100644 --- a/docs/modules/cluster-manager/menu/users-management.md +++ b/docs/modules/cluster-manager/menu/users-management.md @@ -99,15 +99,15 @@ Commands: For example, here is how to create a new user, setting a temp password and giving this user admin permission ```bash -# ideactl accounts create-user --email "mcrozes@myemail.com" --password "Password123@" --username "mcrozes2" --sudo --email-verified +# ideactl accounts create-user --email "sampleuser@example.com" --password "Password123@" --username "sampleuser2" --sudo --email-verified { - "username": "mcrozes2", - "email": "mcrozes@myemail.com", + "username": "sampleuser2", + "email": "sampleuser@example.com", "uid": 5068, "gid": 5077, - "group_name": "mcrozes2-user-group", + "group_name": "sampleuser2-user-group", "login_shell": "/bin/bash", - "home_dir": "/data/home/mcrozes2", + "home_dir": "/data/home/sampleuser2", "sudo": true, "status": "CONFIRMED", "enabled": true, diff --git a/docs/modules/cluster-manager/users-management.md b/docs/modules/cluster-manager/users-management.md index b9651b89..df74ab25 100644 --- a/docs/modules/cluster-manager/users-management.md +++ b/docs/modules/cluster-manager/users-management.md @@ -99,15 +99,15 @@ Commands: For example, here is how to create a new user, setting a temp password and giving this user admin permission ```bash -# ideactl accounts create-user --email "mcrozes@myemail.com" --password "Password123@" --username "mcrozes2" --sudo --email-verified +# ideactl accounts create-user --email "sampleuser@example.com" --password "Password123@" --username "sampleuser2" --sudo --email-verified { - "username": "mcrozes2", - "email": "mcrozes@myemail.com", + "username": "sampleuser2", + "email": "sampleuser@example.com", "uid": 5068, "gid": 5077, - "group_name": "mcrozes2-user-group", + "group_name": "sampleuser2-user-group", "login_shell": "/bin/bash", - "home_dir": "/data/home/mcrozes2", + "home_dir": "/data/home/sampleuser2", "sudo": true, "status": "CONFIRMED", "enabled": true, diff --git a/idea-admin-windows.ps1 b/idea-admin-windows.ps1 deleted file mode 100755 index 3b98141f..00000000 --- a/idea-admin-windows.ps1 +++ /dev/null @@ -1,109 +0,0 @@ -###################################################################################################################### -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # -# # -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # -# with the License. A copy of the License is located at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES # -# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # -# and limitations under the License. # -###################################################################################################################### - -function Exit-Fail($message,$command) { - Write-Host "[MESSAGE]: ${message} `n[COMMAND EXECUTED]: ${command}`n[HELP]: Refer to ${DocumentationError} for troubleshooting." -foregroundcolor "red" - Read-Host -Prompt "Installation was not successful, press Enter to exit" - Exit 1 -} - -function Verify-Command($type,$message,$command) { - if ($type -eq "Get-Command") { - if($Error) { - Exit-Fail $message $command - } - } - elseif ($type -eq "Invoke-Expression") { - if($LASTEXITCODE -ne 0) { - Exit-Fail $message $command - } - } - else { - Write-Output "type must be either Get-Command or Invoke-Expression" - Read-Host -Prompt "Installation was not successful, press Enter to exit" - Exit 1 - } -} - -$IDEADevMode = if ($Env:IDEA_DEV_MODE) {$Env:IDEA_DEV_MODE} else {""} -$VirtualEnv = if ($Env:VIRTUAL_ENV) {$Env:VIRTUAL_ENV} else {""} -$ScriptDir = $PSScriptRoot -$IDEARevision = if ($Env:IDEA_REVISION) {$Env:IDEA_REVISION} else {"v26.09.0"} -$IDEADockerRepo = "public.ecr.aws/s5o2b4m0" -$DocumentationError = "https://docs.idea-hpc.com" -$AWSProfile = if ($Env:AWS_PROFILE) {$Env:AWS_PROFILE} else {"default"} -$AWSRegion= if ($Env:AWS_REGION) {$Env:AWS_REGION} else {"us-east-1"} -Set-Location -Path "${ScriptDir}" - - -if ($IDEADevMode -ne "") { - if (Test-Path -Path "${ScriptDir}/IDEA_VERSION.txt") { - $IDEADevMode="true" - } - else { - $IDEADevMode="false" - } -} - -if ($IDEADevMode -eq "true") { - Write-Host "Development Mode is only supported on Linux/Mac" - <# - if ($VirtualEnv -eq "") { - if (Test-Path -Path "$ScriptDir/venv") { - . "$ScriptDir/venv/bin/activate" - } - else { - Verify-Command "Get-Command" "Python Virtual Environment not detected. Install virtual environment to execute idea-admin.sh in dev mode." "source ${ScriptDir}/venv/bin/activate" - } - } - IDEA_SKIP_WEB_BUILD=${IDEA_SKIP_WEB_BUILD:-'0'} - $IDESkipWebBuild = if (IDEA_SKIP_WEB_BUILD) {$Env:IDEA_DEV_MODE} else {""} - TOKENS=$(echo $(printf ",\"%s\"" "${@}")) - TOKENS=${TOKENS:1} - ARGS=$(echo "[${TOKENS}]" | base64) - CMD="invoke cli.admin --args=${ARGS}" - IDEA_SKIP_WEB_BUILD=${IDEA_SKIP_WEB_BUILD} eval $CMD - exit $? #> -} - -$DockerBin=$(Get-Command docker).source 2>$null -Verify-Command "Get-Command" "Docker not detected. Download and install it from https://docs.docker.com/get-docker/. Read the Docker Subscription Service Agreement first (https://www.docker.com/legal/docker-subscription-service-agreement/)." "Get-Command docker" - -$AWSCliBin=$(Get-Command aws).source 2>$null -Verify-Command "Get-Command" "awscli not detected. Download and install it from https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html" "Get-Command aws" - -if (-not (Test-Path "$HOME/.idea/clusters") ) { - New-Item -ItemType "directory" -Path "$HOME/.idea/clusters" | Out-Null -} - -Invoke-Expression "& '$DockerBin' version" | Out-Null 2>$null -Verify-Command "Invoke-Expression" "Docker is installed on the system but it does not seems to be running. Start Docker first." "docker info" - -[System.Net.Dns]::GetHostEntry("public.ecr.aws") 2>$null | Out-Null -Verify-Command "Get-Command" "Unable to query ECR. Are you connected to internet?" "[System.Net.Dns]::GetHostEntry($IDEADockerRepo)" - -# Select-String -Quiet does not work properly if the number of Docker images is 0, so we go old school and verify if the variable is empty -$ImageExist = Invoke-Expression "& '$DockerBin' images" | Select-String "$IDEADockerRepo/idea-administrator" | Select-String "$IDEARevision" -if ($ImageExist -eq $null) { - Invoke-Expression "& '$DockerBin' pull $IDEADockerRepo/idea-administrator:$IDEARevision" - Verify-Command "Invoke-Expression" "Unable to download IDEA container image. Refer to the error above. If your token has expired, run: docker logout public.ecr.aws" "$DockerBin pull $IDEADockerRepo/idea-administrator:$IDEARevision" -} - -if ($args.count -eq 0) { - $args = "quick-setup" - Write-Host "No arguments detected, defaulting to quick-setup. Use -h to see all options." -} - -Invoke-Expression "& '$DockerBin' run --rm -it -v $HOME/.idea/clusters:/root/.idea/clusters -v $HOME/.aws:/root/.aws $IDEADockerRepo/idea-administrator:$IDEARevision idea-admin $args" - -Read-Host -Prompt "Press Enter to exit" diff --git a/idea-admin.sh b/idea-admin.sh index 7c3af898..c5e6bd68 100755 --- a/idea-admin.sh +++ b/idea-admin.sh @@ -1,19 +1,6 @@ #!/bin/bash -###################################################################################################################### -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # -# # -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # -# with the License. A copy of the License is located at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES # -# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # -# and limitations under the License. # -###################################################################################################################### - -# Integrated Digital Engineering on AWS - Installation Script +# Run the control-plane container. # # Usage: # ./idea-admin.sh --help @@ -32,7 +19,7 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) IDEA_REVISION=${IDEA_REVISION:-"v26.09.0"} -IDEA_DOCKER_REPO_DEFAULT="public.ecr.aws/s5o2b4m0/idea-administrator" +IDEA_DOCKER_REPO_DEFAULT="public.ecr.aws/s5o2b4m0/idea-control-plane" IDEA_DOCKER_REPO=${IDEA_DOCKER_REPO:-"${IDEA_DOCKER_REPO_DEFAULT}"} IDEA_ECR_CREDS_RESET=${IDEA_ECR_CREDS_RESET:-"true"} IDEA_ADMIN_AWS_CREDENTIAL_PROVIDER=${IDEA_ADMIN_AWS_CREDENTIAL_PROVIDER:=""} @@ -97,6 +84,17 @@ command -v aws > /dev/null verify_command "awscli not detected. Download and install it from https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html" echo -e "${GREEN}✓ AWS CLI detected${NC}" +# SSO profiles cannot authenticate inside the container. +if [[ -n "${AWS_PROFILE}" ]] && \ + { aws configure get sso_session --profile "${AWS_PROFILE}" > /dev/null 2>&1 || \ + aws configure get sso_start_url --profile "${AWS_PROFILE}" > /dev/null 2>&1; }; then + echo -e "${RED}[MESSAGE]: AWS_PROFILE=${AWS_PROFILE} is an SSO profile, which cannot be used inside the container." + echo -e "[HELP]: Export static credentials on the host and unset AWS_PROFILE, for example:" + echo -e " eval \$(aws configure export-credentials --profile ${AWS_PROFILE} --format env)" + echo -e " unset AWS_PROFILE${NC}" + exit 1 +fi + # Create folder hierarchy MKDIR_BIN=$(command -v mkdir) ${MKDIR_BIN} -p "${HOME}"/.idea/clusters @@ -108,22 +106,21 @@ ${DOCKER_BIN} info >> /dev/null 2>&1 verify_command "Docker is installed on the system but it does not seems to be running. Start Docker first." echo -e "${GREEN}✓ Docker is running${NC}" -# `invoke docker.build` tags the image with no registry prefix, so the published-image check -# below can't see it and would pull the released image over your build; prefer the local one. +# A local build is tagged with no registry prefix, so the pull check below cannot see it and +# would fetch the released image over it. Prefer the local one. if [[ "${IDEA_DOCKER_REPO}" == "${IDEA_DOCKER_REPO_DEFAULT}" ]] && \ - ${DOCKER_BIN} image inspect "idea-administrator:${IDEA_REVISION}" >> /dev/null 2>&1; then - IDEA_DOCKER_REPO="idea-administrator" + ${DOCKER_BIN} image inspect "idea-control-plane:${IDEA_REVISION}" >> /dev/null 2>&1; then + IDEA_DOCKER_REPO="idea-control-plane" fi # Reset ECR credentials if [[ "${IDEA_ECR_CREDS_RESET}" == "true" && "${IDEA_DOCKER_REPO}" == *"/"* ]]; then echo -e "${YELLOW}[INFO] Resetting ECR credentials...${NC}" - # Check if user is connected to internet an can ping ECR repo DIG_BIN=$(command -v dig) IDEA_DOCKER_REPO_HOSTNAME=$(echo "${IDEA_DOCKER_REPO}" | cut -d '/' -f 1) if [[ -z "${DIG_BIN}" ]]; then - # dig ships in bind-utils, which a stock Amazon Linux 2023 host does not have. The - # reset is a convenience, so skip it rather than fail the command that was asked for. + # dig ships in bind-utils, which a stock Amazon Linux 2023 host does not have. The reset is a + # convenience, so skip it rather than fail the command that was asked for. echo -e "${YELLOW}[INFO] dig not found: skipping ECR credentials reset. Install bind-utils, or set IDEA_ECR_CREDS_RESET=false to skip this step without the warning.${NC}" else ${DIG_BIN} +tries=1 +time=3 "${IDEA_DOCKER_REPO_HOSTNAME}" >> /dev/null 2>&1 @@ -148,10 +145,9 @@ else fi IDEA_IMAGE_CREATED=$(${DOCKER_BIN} image inspect --format "{{.Created}}" "${IDEA_DOCKER_REPO}:${IDEA_REVISION}" 2>/dev/null) -echo -e "${YELLOW}[INFO] Administrator image: ${IDEA_DOCKER_REPO}:${IDEA_REVISION} (created ${IDEA_IMAGE_CREATED:-unknown})${NC}" +echo -e "${YELLOW}[INFO] Control plane image: ${IDEA_DOCKER_REPO}:${IDEA_REVISION} (created ${IDEA_IMAGE_CREATED:-unknown})${NC}" echo -e "${YELLOW}[INFO] Launching IDEA administrator...${NC}" -# Keep -it when stdin is an interactive terminal; otherwise drop -t so docker does not -# try to attach a TTY to non-interactive stdin. +# Drop -t when stdin is not a terminal, so docker does not try to attach a TTY to a pipe. if [[ -t 0 && "${IDEA_ADMIN_NO_TTY}" != "true" ]]; then DOCKER_TTY_FLAGS="-it" else @@ -165,4 +161,4 @@ ${DOCKER_BIN} run --rm ${DOCKER_TTY_FLAGS} -v "${HOME}/.idea/clusters:/root/.ide -e IDEA_ADMIN_AWS_CREDENTIAL_PROVIDER="${IDEA_ADMIN_AWS_CREDENTIAL_PROVIDER}" \ -e IDEA_ADMIN_ENABLE_CDK_NAG_SCAN="${IDEA_ADMIN_ENABLE_CDK_NAG_SCAN}" \ -v ~/.aws:/root/.aws "${IDEA_DOCKER_REPO}:${IDEA_REVISION}" \ - idea-admin "${@}" + ideactl "${@}" diff --git a/source/idea/idea-administrator/resources/config/templates/metrics/settings.yml b/source/idea/idea-administrator/resources/config/templates/metrics/settings.yml index fcfcc8dd..78584a16 100644 --- a/source/idea/idea-administrator/resources/config/templates/metrics/settings.yml +++ b/source/idea/idea-administrator/resources/config/templates/metrics/settings.yml @@ -6,7 +6,7 @@ # graphana, visualizations and dashboards is out of IDEA scope. # you need to manually configure the applicable dashboards based on the metrics provider you've configured for the cluster. -# provider can be one of [cloudwatch, amazon_managed_prometheus, prometheus] +# provider can be one of [cloudwatch, amazon_managed_prometheus, prometheus, dogstatsd] provider: "{{metrics_provider}}" {% if metrics_provider == 'cloudwatch' %} @@ -23,6 +23,15 @@ cloudwatch: force_flush_interval: 60 {% endif %} +{% if metrics_provider == 'dogstatsd' %} +dogstatsd: + # Where the modules send metrics. A Datadog agent next to the module (a sidecar, or + # an agent on the host) receives them; nothing is provisioned in AWS for this provider. + # udp://host:port, or unix:///path/to/dsd.socket for an agent sharing a volume. + # DD_DOGSTATSD_URL in the module environment overrides this. + url: udp://127.0.0.1:8125 +{% endif %} + {% if metrics_provider == 'amazon_managed_prometheus' %} amazon_managed_prometheus: workspace_name: {{cluster_name}}-workspace diff --git a/source/idea/idea-administrator/resources/policies/scheduler.yml b/source/idea/idea-administrator/resources/policies/scheduler.yml index bc5c40ee..a50b2e51 100644 --- a/source/idea/idea-administrator/resources/policies/scheduler.yml +++ b/source/idea/idea-administrator/resources/policies/scheduler.yml @@ -209,6 +209,20 @@ Statement: - '{{ context.vars.spot_fleet_request_role_arn }}' Effect: Allow +{% if context.config.get_bool('ecs.enabled', default=False) %} + # A container scheduler resolves cluster users from the account tables the cluster manager owns, + # because it has no directory client of its own. The sync runs once before the module starts and + # scans these three tables, so without this the task exits before it binds its port. + - Action: + - dynamodb:Scan + Resource: + - '{{ context.arns.get_ddb_table_arn("accounts.users") }}' + - '{{ context.arns.get_ddb_table_arn("accounts.groups") }}' + - '{{ context.arns.get_ddb_table_arn("accounts.group-members") }}' + Effect: Allow + Sid: ClusterUserSync +{% endif %} + {% if context.config.get_bool(context.config.get_module_id('cluster-manager') + '.bedrock.enabled', False) and context.config.get_bool(context.module_id + '.bedrock.enabled', False) %} # scoped to roles under the per-project IAM path, passed only to ec2; enabling bedrock for jobs requires redeploying this module along with cluster-manager. - Condition: @@ -295,6 +309,16 @@ Statement: - '*' Sid: ComputeNodeAmiBuilderPermissions2 +{% if context.config.get_bool('scheduler.use_stable_server_name', default=False) or context.config.get_bool('ecs.enabled', default=False) %} + # With a stable server name the scheduler owns its own DNS record and points it at + # whichever host or task is running the server. The container path always has the stable name, + # and the stack writes the row itself, so the flag alone is enough to grant the permission. + - Effect: Allow + Action: route53:ChangeResourceRecordSets + Resource: '{{ context.arns.get_route53_hostedzone_arn() }}' + Sid: SchedulerDnsRecord +{% endif %} + {% include '_templates/aws-managed-ad.yml' %} {% include '_templates/openldap.yml' %} diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/metrics_stack.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/metrics_stack.py index 8b4cd6f9..750357ac 100644 --- a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/metrics_stack.py +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/metrics_stack.py @@ -62,6 +62,9 @@ def __init__( self.build_amazon_managed_prometheus() elif self.is_prometheus(): self.build_prometheus() + elif self.is_dogstatsd(): + # the agent the metrics go to is deployed with the modules, not by this stack + pass else: raise exceptions.general_exception( f'metrics provider: {self.get_metrics_provider()} not supported' @@ -84,6 +87,9 @@ def is_amazon_managed_prometheus(self) -> bool: def is_prometheus(self) -> bool: return self.get_metrics_provider() == constants.METRICS_PROVIDER_PROMETHEUS + def is_dogstatsd(self) -> bool: + return self.get_metrics_provider() == constants.METRICS_PROVIDER_DOGSTATSD + def build_cloudwatch(self): dashboard_name = self.context.config().get_string( 'metrics.cloudwatch.dashboard_name', required=True diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/scheduler_stack.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/scheduler_stack.py index 152de4e7..0acef575 100644 --- a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/scheduler_stack.py +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/scheduler_stack.py @@ -635,10 +635,26 @@ def build_endpoints(self): ) def build_cluster_settings(self): + # The scheduler and the execution hosts both derive the PBS server name from + # private_dns_name. Pointing it at the cluster DNS record rather than the + # instance lets a replaced scheduler keep its name, so execution hosts do not + # need reconfiguring and running jobs survive. Off by default: turning it on for + # an existing cluster renames its PBS server, and execution hosts already running + # jobs would not follow the change. + use_stable_server_name = self.context.config().get_bool( + 'scheduler.use_stable_server_name', default=False + ) + if use_stable_server_name: + private_dns_name = self.context.config().get_string( + 'scheduler.hostname', required=True + ) + else: + private_dns_name = self.ec2_instance.attr_private_dns_name + cluster_settings = { 'deployment_id': self.deployment_id, 'private_ip': self.ec2_instance.attr_private_ip, - 'private_dns_name': self.ec2_instance.attr_private_dns_name, + 'private_dns_name': private_dns_name, } is_public = self.context.config().get_bool('scheduler.public', default=False) diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/installer_params.py b/source/idea/idea-administrator/src/ideaadministrator/app/installer_params.py index 96dec2cd..ef02125e 100644 --- a/source/idea/idea-administrator/src/ideaadministrator/app/installer_params.py +++ b/source/idea/idea-administrator/src/ideaadministrator/app/installer_params.py @@ -998,6 +998,10 @@ def get_choices(self, refresh: bool = False) -> List[SocaUserInputChoice]: value=constants.METRICS_PROVIDER_AMAZON_MANAGED_PROMETHEUS, disabled='aps' not in available_services, ), + SocaUserInputChoice( + title='Datadog agent (DogStatsD)', + value=constants.METRICS_PROVIDER_DOGSTATSD, + ), SocaUserInputChoice( title='Custom Prometheus Server', value=constants.METRICS_PROVIDER_PROMETHEUS, diff --git a/source/idea/idea-bootstrap/_templates/linux/openpbs_client.jinja2 b/source/idea/idea-bootstrap/_templates/linux/openpbs_client.jinja2 index 5c82412c..252a57fd 100644 --- a/source/idea/idea-bootstrap/_templates/linux/openpbs_client.jinja2 +++ b/source/idea/idea-bootstrap/_templates/linux/openpbs_client.jinja2 @@ -1,7 +1,25 @@ # Begin: OpenPBS Client {% include '_templates/linux/openpbs.jinja2' %} -echo -e "PBS_SERVER={{ context.config.get_string('scheduler.private_dns_name', required=True).split('.')[0] }} +{# With a stable server name the client resolves the server through DNS by its full name; + the short name is not in the resolver search path. #} +{% set scheduler_dns_name = context.config.get_string('scheduler.private_dns_name', required=True) %} +{% set pbs_server_name = scheduler_dns_name if context.config.get_bool('scheduler.use_stable_server_name', default=False) else scheduler_dns_name.split('.')[0] %} + +{% if context.config.get_bool('scheduler.use_stable_server_name', default=False) %} +{# pbs_init.d checks the server name after cutting it at the first dot, so the short name + has to resolve on this host. Put the cluster's private zone in the resolver search path; + the default search path carries only the EC2 domain. #} +{% set private_zone = context.config.get_string('cluster.route53.private_hosted_zone_name', required=True) %} +RESOLVER_IF=$(ip -o -4 route show default | awk '{print $5}' | head -1) +if command -v resolvectl >/dev/null 2>&1 && systemctl is-active systemd-resolved >/dev/null 2>&1; then + resolvectl domain "${RESOLVER_IF}" {{ private_zone }} $(resolvectl domain "${RESOLVER_IF}" | sed 's/^[^:]*: *//') +else + grep -q "^search .*{{ private_zone }}" /etc/resolv.conf || sed -i 's/^search /search {{ private_zone }} /' /etc/resolv.conf +fi +{% endif %} + +echo -e "PBS_SERVER={{ pbs_server_name }} PBS_START_SERVER=0 PBS_START_SCHED=0 PBS_START_COMM=0 @@ -13,7 +31,7 @@ PBS_SCP=/usr/bin/scp " > /etc/pbs.conf echo -e " -\$clienthost {{ context.config.get_string('scheduler.private_dns_name').split('.')[0] }} +\$clienthost {{ pbs_server_name }} \$usecp *:/dev/null /dev/null \$usecp *:{{ context.config.get_string('shared-storage.data.mount_dir') }} {{ context.config.get_string('shared-storage.data.mount_dir') }} \$usecp *:{{ context.config.get_string('shared-storage.apps.mount_dir') }} {{ context.config.get_string('shared-storage.apps.mount_dir') }} diff --git a/source/idea/idea-bootstrap/compute-node/_templates/configure_openpbs_compute_node.jinja2 b/source/idea/idea-bootstrap/compute-node/_templates/configure_openpbs_compute_node.jinja2 index a617b3b5..3b498b10 100644 --- a/source/idea/idea-bootstrap/compute-node/_templates/configure_openpbs_compute_node.jinja2 +++ b/source/idea/idea-bootstrap/compute-node/_templates/configure_openpbs_compute_node.jinja2 @@ -16,13 +16,31 @@ fi # pbs_mom exits at startup if it can't resolve its own hostname or the scheduler's, and neither # is in DNS; keyed on hostname so an ami-baked line for a recycled ip can't suppress this one. grep -qE "(^|[[:space:]])${INSTANCE_HOSTNAME}([[:space:]]|$)" /etc/hosts || echo "${INSTANCE_NODE_NAME} ${INSTANCE_HOSTNAME} ${INSTANCE_SHORT_HOSTNAME}" >> /etc/hosts -{% set scheduler_private_ip = context.config.get_string('scheduler.private_ip') %} -{% if scheduler_private_ip %} +{% set stable_server_name = context.config.get_bool('scheduler.use_stable_server_name', default=False) %} {% set scheduler_dns_name = context.config.get_string('scheduler.private_dns_name', required=True) %} +{# With a stable server name the record is in DNS and follows the server wherever it runs, so + the address is never pinned here; a pinned address would outlive a server replacement. The + short name is not in the resolver search path, so the daemon is given the full name. #} +{% set pbs_server_name = scheduler_dns_name if stable_server_name else scheduler_dns_name.split('.')[0] %} +{% set scheduler_private_ip = context.config.get_string('scheduler.private_ip') %} +{% if scheduler_private_ip and not stable_server_name %} grep -q "^{{ scheduler_private_ip }} " /etc/hosts || echo "{{ scheduler_private_ip }} {{ scheduler_dns_name }} {{ scheduler_dns_name.split('.')[0] }}" >> /etc/hosts {% endif %} -echo -e "PBS_SERVER={{ context.config.get_string('scheduler.private_dns_name', required=True).split('.')[0] }} +{% if stable_server_name %} +{# pbs_init.d checks the server name after cutting it at the first dot, so the short name + has to resolve on this host. Put the cluster's private zone in the resolver search path; + the default search path carries only the EC2 domain. #} +{% set private_zone = context.config.get_string('cluster.route53.private_hosted_zone_name', required=True) %} +RESOLVER_IF=$(ip -o -4 route show default | awk '{print $5}' | head -1) +if command -v resolvectl >/dev/null 2>&1 && systemctl is-active systemd-resolved >/dev/null 2>&1; then + resolvectl domain "${RESOLVER_IF}" {{ private_zone }} $(resolvectl domain "${RESOLVER_IF}" | sed 's/^[^:]*: *//') +else + grep -q "^search .*{{ private_zone }}" /etc/resolv.conf || sed -i 's/^search /search {{ private_zone }} /' /etc/resolv.conf +fi +{% endif %} + +echo -e "PBS_SERVER={{ pbs_server_name }} PBS_START_SERVER=0 PBS_START_SCHED=0 PBS_START_COMM=0 @@ -36,7 +54,7 @@ PBS_SCP=/usr/bin/scp " > /etc/pbs.conf echo -e " -\$clienthost {{ context.config.get_string('scheduler.private_dns_name').split('.')[0] }} +\$clienthost {{ pbs_server_name }} \$usecp *:/dev/null /dev/null \$usecp *:{{ context.config.get_string('shared-storage.data.mount_dir') }} {{ context.config.get_string('shared-storage.data.mount_dir') }} " > /var/spool/pbs/mom_priv/config diff --git a/source/idea/idea-bootstrap/scheduler/_templates/configure_openpbs_server.jinja2 b/source/idea/idea-bootstrap/scheduler/_templates/configure_openpbs_server.jinja2 index 198f0c92..136687d4 100644 --- a/source/idea/idea-bootstrap/scheduler/_templates/configure_openpbs_server.jinja2 +++ b/source/idea/idea-bootstrap/scheduler/_templates/configure_openpbs_server.jinja2 @@ -2,9 +2,13 @@ {% include '_templates/linux/openpbs.jinja2' %} +{% set scheduler_dns_name = context.config.get_string('scheduler.private_dns_name', required=True) %} SCHEDULER_PRIVATE_IP=$(get_server_ip) -SCHEDULER_HOSTNAME=$(hostname) -SCHEDULER_HOSTNAME_ALT=$(hostname -s) +# The server takes its name from the same setting the execution hosts read, so the two +# cannot drift. A host that comes back under a different name is then only a DNS change, +# not a rebuild. +SCHEDULER_HOSTNAME={{ scheduler_dns_name }} +SCHEDULER_HOSTNAME_ALT={{ scheduler_dns_name.split('.')[0] }} log_info "configure: /etc/hosts" echo ${SCHEDULER_PRIVATE_IP} ${SCHEDULER_HOSTNAME} ${SCHEDULER_HOSTNAME_ALT} >> /etc/hosts diff --git a/source/idea/idea-data-model/src/ideadatamodel/constants.py b/source/idea/idea-data-model/src/ideadatamodel/constants.py index f057f60a..ac1b625f 100644 --- a/source/idea/idea-data-model/src/ideadatamodel/constants.py +++ b/source/idea/idea-data-model/src/ideadatamodel/constants.py @@ -396,6 +396,7 @@ METRICS_PROVIDER_CLOUDWATCH = 'cloudwatch' METRICS_PROVIDER_PROMETHEUS = 'prometheus' METRICS_PROVIDER_AMAZON_MANAGED_PROMETHEUS = 'amazon_managed_prometheus' +METRICS_PROVIDER_DOGSTATSD = 'dogstatsd' # services SERVICE_ID_LEADER_ELECTION = 'leader-election' diff --git a/source/idea/idea-scheduler/src/ideascheduler/app/metrics/job_completion_metrics.py b/source/idea/idea-scheduler/src/ideascheduler/app/metrics/job_completion_metrics.py new file mode 100644 index 00000000..0b33d354 --- /dev/null +++ b/source/idea/idea-scheduler/src/ideascheduler/app/metrics/job_completion_metrics.py @@ -0,0 +1,136 @@ +from ideadatamodel import SocaJob +from ideasdk.context import SocaContext +from ideasdk.metrics import BaseMetrics +from ideasdk.utils import Utils + +from typing import Optional + +# used CPU time over allocated CPU time can read slightly over one when hyperthreads +# are counted; past this it is an accounting error and is not reported at all. +CPU_EFFICIENCY_MAX = 1.05 + + +class JobCompletionMetrics(BaseMetrics): + """ + One set of measurements per finished job, tagged the way spend gets sliced: what the + job was estimated to cost, how long it ran on the wall clock and how much of the CPU it + asked for it used. + + Published once from the finished-job processor. The duration comes from the job's own + start and end stamps; total_time_secs is what the price estimate used and can differ. + """ + + def __init__(self, context: SocaContext, job: SocaJob): + super().__init__(context, split_dimensions=False) + self.job = job + params = job.params + instance_type = self.instance_type(job) + self.with_dimension('project', self.tag(job.project)) + self.with_dimension('owner', self.tag(job.owner)) + self.with_dimension('queue', self.tag(job.queue)) + self.with_dimension('queue_type', self.tag(job.queue_type)) + self.with_dimension('instance_family', self.tag(self.instance_family(instance_type))) + self.with_dimension('capacity_type', self.capacity_type(job)) + self.with_dimension('base_os', self.tag(params.base_os if params else None)) + self.with_dimension('job_outcome', self.outcome(job)) + self.with_dimension('gpu', 'true' if params and (params.gpus or 0) > 0 else 'false') + + @staticmethod + def tag(value) -> str: + return str(value) if Utils.is_not_empty(value) else 'unknown' + + @staticmethod + def instance_type(job: SocaJob) -> Optional[str]: + for host in job.execution_hosts or []: + if Utils.is_not_empty(host.instance_type): + return host.instance_type + if job.params and job.params.instance_types: + return job.params.instance_types[0] + return None + + @staticmethod + def instance_family(instance_type: Optional[str]) -> Optional[str]: + if Utils.is_empty(instance_type): + return None + return instance_type.split('.', 1)[0] + + @staticmethod + def capacity_type(job: SocaJob) -> str: + for host in job.execution_hosts or []: + if host.capacity_type is not None: + return str(host.capacity_type.value) + if job.params and job.params.spot: + return 'spot' + return 'on-demand' + + @staticmethod + def outcome(job: SocaJob) -> str: + """ + PBS exit status: 0 ran to completion, negative never ran on the node (requeue, + pre-execution failure), above 128 was killed by a signal, otherwise the + application failed. + """ + if job.start_time is None: + return 'unprovisioned' + status = job.exit_status + if status is None: + return 'unknown' + if status == 0: + return 'success' + if status < 0: + return 'requeued' + if status > 128: + return 'killed' + return 'failure' + + @staticmethod + def wall_seconds(job: SocaJob) -> Optional[float]: + if job.start_time is None or job.end_time is None: + return None + seconds = (job.end_time - job.start_time).total_seconds() + return seconds if seconds > 0 else None + + @classmethod + def cpu_efficiency(cls, job: SocaJob) -> Optional[float]: + cpus = job.params.cpus if job.params else None + wall = cls.wall_seconds(job) + if not cpus or cpus <= 0 or wall is None: + return None + used = 0.0 + for host in job.execution_hosts or []: + runs = host.execution.runs if host.execution else None + for run in runs or []: + if run.resources_used and run.resources_used.cpu_time_secs: + used += run.resources_used.cpu_time_secs + if used <= 0: + return None + efficiency = used / (cpus * wall) + if efficiency > CPU_EFFICIENCY_MAX: + return None + return min(efficiency, 1.0) + + def publish(self): + job = self.job + self.count(MetricName='job.count', Value=1) + + wall = self.wall_seconds(job) + if wall is not None: + self.seconds(MetricName='job.duration_seconds', Value=wall) + + cost = job.estimated_bom_cost + if cost is not None: + if cost.total is not None and cost.total.amount is not None: + self.count(MetricName='job.cost', Value=cost.total.amount) + if cost.line_items_total is not None and cost.line_items_total.amount is not None: + self.count(MetricName='job.cost_ondemand', Value=cost.line_items_total.amount) + if cost.savings_total is not None and cost.savings_total.amount is not None: + self.count(MetricName='job.savings', Value=cost.savings_total.amount) + + efficiency = self.cpu_efficiency(job) + if efficiency is not None: + self._log( + MetricName='job.cpu_efficiency', + Value=efficiency, + MetricType='Summary', + Unit='None', + ) diff --git a/source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_monitor/finished_job_processor.py b/source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_monitor/finished_job_processor.py index db88998b..8f72175f 100644 --- a/source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_monitor/finished_job_processor.py +++ b/source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_monitor/finished_job_processor.py @@ -26,6 +26,7 @@ import arrow import logging +from ideascheduler.app.metrics.job_completion_metrics import JobCompletionMetrics from ideascheduler.app.provisioning.lifecycle_events import ( ProvisioningLifecycleEvents, ) @@ -190,6 +191,8 @@ def publish_job_metrics(self): queue_type=self.job.queue_type, duration_secs=int(total_duration.total_seconds()), ) + + JobCompletionMetrics(context=self._context, job=self.job).publish() except Exception as e: self._logger.exception( f'{self.job.log_tag} failed to publish job metrics: {e}' @@ -264,6 +267,10 @@ def invoke_unprovisioned(self) -> Optional[SocaJob]: self.log_job_complete() + # counted with its outcome, so the jobs that never got capacity sit in the + # same series as the ones that ran + self.publish_job_metrics() + self.publish_lifecycle_event(disposition=disposition) self.publish_to_job_export_log() diff --git a/source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_monitor/job_cache.py b/source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_monitor/job_cache.py index d1f67dd9..41f5f6c3 100644 --- a/source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_monitor/job_cache.py +++ b/source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_monitor/job_cache.py @@ -95,11 +95,30 @@ def __init__(self, context: ideascheduler.AppContext): self._db_lock = RLock() # Keep the RLock for database operations self.init_db() + # SQLite serialises writers; with the default rollback journal a writer also blocks + # every reader, and the API's job listings share this file with the job monitor's + # writes. WAL lets readers run beside the writer, the busy timeout keeps a writer + # waiting instead of failing, and the pool covers the API's worker threads plus the + # monitors. Measured: at 7 listings a second the defaults starved the pool the + # moment a job finished, and every listing then waited out the 30 s pool timeout. + ENGINE_KWARGS = { + 'pool_size': 20, + 'max_overflow': 20, + 'pool_timeout': 30, + 'connect_args': {'timeout': 30, 'check_same_thread': False}, + } + + def _connect(self): + db = dataset.connect(self.connection_string, engine_kwargs=self.ENGINE_KWARGS) + db.query('PRAGMA journal_mode=WAL') + db.query('PRAGMA synchronous=NORMAL') + return db + def init_db(self): self._logger.info(f'initializing job cache db file: {self.connection_string}') try: # Create the database connection - self.db = dataset.connect(self.connection_string) + self.db = self._connect() # First check if tables already exist existing_tables = self.db.tables @@ -166,7 +185,7 @@ def create_all_tables(self): conn.close() # Refresh database object after raw connection usage - self.db = dataset.connect(self.connection_string) + self.db = self._connect() def init_tables(self): # This method is kept for backward compatibility @@ -205,7 +224,7 @@ def drop_unique_index(self, table: str, index_name: str): if dropped: # Refresh database object after raw connection usage - self.db = dataset.connect(self.connection_string) + self.db = self._connect() def init_indices(self): """Initialize all indices outside of any transaction""" diff --git a/source/idea/idea-scheduler/src/ideascheduler/app/scheduler/openpbs/openpbs_scheduler.py b/source/idea/idea-scheduler/src/ideascheduler/app/scheduler/openpbs/openpbs_scheduler.py index 725978e4..99577744 100644 --- a/source/idea/idea-scheduler/src/ideascheduler/app/scheduler/openpbs/openpbs_scheduler.py +++ b/source/idea/idea-scheduler/src/ideascheduler/app/scheduler/openpbs/openpbs_scheduler.py @@ -58,7 +58,10 @@ def __init__(self, context: ideascheduler.AppContext): self._converter = OpenPBSConverter(context=self._context, logger=self._logger) def is_ready(self) -> bool: - result = self._shell.invoke('systemctl status pbs', shell=True) + # Ask the server rather than the init system. systemd reports the pbs unit active + # while pbs_comm and pbs_sched are up even after pbs_server has exited, and a + # container has no systemd at all. qstat -B answers only when the server does. + result = self._shell.invoke('/opt/pbs/bin/qstat -B', shell=True) return result.returncode == 0 def list_nodes(self, host: Optional[str] = None, **kwargs) -> List[SocaComputeNode]: @@ -336,7 +339,9 @@ def list_queues(self, queue: Optional[str] = None) -> List[SocaQueue]: json_response = Utils.from_json(result.stdout) queues = Utils.get_value_as_dict('Queue', json_response) - if len(queues.keys()) == 0: + # A server with no queues yet, which is every first start, comes back as None + # rather than an empty mapping. + if queues is None or len(queues.keys()) == 0: return [] response = [] diff --git a/source/idea/idea-scheduler/tests/test_job_completion_metrics.py b/source/idea/idea-scheduler/tests/test_job_completion_metrics.py new file mode 100644 index 00000000..35ccbc21 --- /dev/null +++ b/source/idea/idea-scheduler/tests/test_job_completion_metrics.py @@ -0,0 +1,87 @@ +""" +job completion metrics: the outcome, duration and CPU efficiency derived from a finished +job, and the tags they carry. +""" + +from ideadatamodel.scheduler.scheduler_model import ( + SocaJob, + SocaJobParams, + SocaJobExecutionHost, + SocaJobExecution, + SocaJobExecutionRun, + SocaJobExecutionResourcesUsed, + SocaCapacityType, +) +from ideascheduler.app.metrics.job_completion_metrics import JobCompletionMetrics + +import arrow + + +def _job(exit_status=0, cpus=4, wall_secs=3600, cpu_time_secs=None, started=True, **kwargs): + start = arrow.get('2026-09-09T10:00:00+00:00').datetime + runs = [] + if cpu_time_secs is not None: + runs.append( + SocaJobExecutionRun( + run_id='1', + resources_used=SocaJobExecutionResourcesUsed(cpu_time_secs=cpu_time_secs), + ) + ) + return SocaJob( + job_id='42', + project='fusion', + owner='alice', + queue='normal', + queue_type='compute', + exit_status=exit_status, + start_time=start if started else None, + end_time=arrow.get(start).shift(seconds=wall_secs).datetime if started else None, + params=SocaJobParams(cpus=cpus, gpus=0, base_os='rhel9', instance_types=['c7g.2xlarge']), + execution_hosts=[ + SocaJobExecutionHost( + instance_type='c7g.2xlarge', + capacity_type=SocaCapacityType.SPOT, + execution=SocaJobExecution(run_count=len(runs), runs=runs), + ) + ], + **kwargs, + ) + + +def test_outcome_follows_the_pbs_exit_status(): + assert JobCompletionMetrics.outcome(_job(exit_status=0)) == 'success' + assert JobCompletionMetrics.outcome(_job(exit_status=1)) == 'failure' + assert JobCompletionMetrics.outcome(_job(exit_status=-3)) == 'requeued' + assert JobCompletionMetrics.outcome(_job(exit_status=137)) == 'killed' + assert JobCompletionMetrics.outcome(_job(exit_status=None)) == 'unknown' + assert JobCompletionMetrics.outcome(_job(started=False)) == 'unprovisioned' + + +def test_duration_is_wall_clock_from_the_job_stamps(): + assert JobCompletionMetrics.wall_seconds(_job(wall_secs=86514)) == 86514 + assert JobCompletionMetrics.wall_seconds(_job(started=False)) is None + + +def test_cpu_efficiency_is_used_over_allocated_and_drops_nonsense(): + # 4 cpus for an hour, 3 cpu-hours used + assert JobCompletionMetrics.cpu_efficiency(_job(cpus=4, wall_secs=3600, cpu_time_secs=10800)) == 0.75 + # hyperthread accounting can read a little over; capped, not dropped + assert JobCompletionMetrics.cpu_efficiency(_job(cpus=4, wall_secs=3600, cpu_time_secs=14500)) == 1.0 + # ratios in the hundreds are an accounting error, so they are not reported at all + assert JobCompletionMetrics.cpu_efficiency(_job(cpus=4, wall_secs=3600, cpu_time_secs=5_000_000)) is None + assert JobCompletionMetrics.cpu_efficiency(_job(cpus=4, wall_secs=3600)) is None + + +def test_dimensions_are_the_cost_dashboard_slices(context): + metrics = JobCompletionMetrics(context=context, job=_job(exit_status=0)) + assert {d['Name']: d['Value'] for d in metrics.dimensions} == { + 'project': 'fusion', + 'owner': 'alice', + 'queue': 'normal', + 'queue_type': 'compute', + 'instance_family': 'c7g', + 'capacity_type': 'spot', + 'base_os': 'rhel9', + 'job_outcome': 'success', + 'gpu': 'false', + } diff --git a/source/idea/idea-sdk/src/ideasdk/metrics/dogstatsd/__init__.py b/source/idea/idea-sdk/src/ideasdk/metrics/dogstatsd/__init__.py new file mode 100644 index 00000000..d94efca8 --- /dev/null +++ b/source/idea/idea-sdk/src/ideasdk/metrics/dogstatsd/__init__.py @@ -0,0 +1 @@ +from ideasdk.metrics.dogstatsd.dogstatsd_metrics import DogStatsdMetrics # noqa diff --git a/source/idea/idea-sdk/src/ideasdk/metrics/dogstatsd/dogstatsd_metrics.py b/source/idea/idea-sdk/src/ideasdk/metrics/dogstatsd/dogstatsd_metrics.py new file mode 100644 index 00000000..bd1205b4 --- /dev/null +++ b/source/idea/idea-sdk/src/ideasdk/metrics/dogstatsd/dogstatsd_metrics.py @@ -0,0 +1,159 @@ +from ideasdk.protocols import MetricsProviderProtocol, SocaContextProtocol +from ideasdk.utils import Utils + +from typing import Dict, List, Optional, Tuple +from threading import RLock +from urllib.parse import urlparse +import os +import re +import socket + +DEFAULT_URL = 'udp://127.0.0.1:8125' +METRIC_PREFIX = 'idea' +# The agent reads datagrams of up to 8 KiB; one datagram carries many newline-separated lines. +MAX_DATAGRAM_BYTES = 8192 +# Tag values may not carry the characters the wire format uses to delimit. +_TAG_UNSAFE = re.compile(r'[,|#\n\r\t ]+') + +# Counters become counts and Summaries distributions, so the agent aggregates across +# tasks; anything else is a gauge. +_METRIC_TYPES = {'Counter': 'c', 'Summary': 'd'} + + +class DogStatsdMetrics(MetricsProviderProtocol): + """ + Ship IDEA metrics to a Datadog agent over DogStatsD. + + The namespace (cluster/module[/component]) and the metric dimensions travel as tags. + BaseMetrics publishes one entry per dimension plus one carrying all of them; only the + complete entry is sent, since a tagged metric already slices by every tag on it. + + The target is metrics.dogstatsd.url, then DD_DOGSTATSD_URL, then UDP on localhost: + udp://host:port, or unix:///path/to/dsd.socket when the agent shares a volume with + the task. Sending never raises: a missing agent costs a warning, not the request. + """ + + def __init__(self, context: SocaContextProtocol, namespace: str): + self.context = context + self.logger = context.logger('dogstatsd-metrics') + self.namespace = namespace + + self.base_tags: List[str] = [] + for key, value in zip( + ('idea_cluster', 'idea_module', 'component'), namespace.split('/') + ): + if Utils.is_not_empty(value): + self.base_tags.append(f'{key}:{self._tag_value(value)}') + + url = context.config().get_string('metrics.dogstatsd.url') + if Utils.is_empty(url): + url = os.environ.get('DD_DOGSTATSD_URL', DEFAULT_URL) + self._family, self._address = self._parse_url(url) + + self._socket: Optional[socket.socket] = None + self._lock = RLock() + self._send_failures = 0 + + @staticmethod + def _parse_url(url: str) -> Tuple[int, object]: + parsed = urlparse(url) + if parsed.scheme == 'unix' and Utils.is_not_empty(parsed.path): + return socket.AF_UNIX, parsed.path + if parsed.scheme == 'udp' and Utils.is_not_empty(parsed.hostname): + return socket.AF_INET, (parsed.hostname, parsed.port or 8125) + raise ValueError( + f'metrics.dogstatsd.url must be udp://host:port or unix:///path, got: {url}' + ) + + @staticmethod + def _tag_value(value) -> str: + text = _TAG_UNSAFE.sub('_', str(value).strip()) + return text[:200] if Utils.is_not_empty(text) else 'unknown' + + @staticmethod + def _value(value) -> str: + # plain decimals: no exponent, no trailing zeros, so 86514.0 travels as 86514 + text = f'{float(value):.6f}'.rstrip('0').rstrip('.') + return text if text not in ('', '-0') else '0' + + @staticmethod + def complete_entries(metric_data: List[Dict]) -> List[Dict]: + """ + one entry per metric name: the one with the most dimensions. BaseMetrics emits + the per-dimension splits and the complete entry in one batch; sending all of them + would count the same event once per dimension. + """ + chosen: Dict[str, Dict] = {} + order: List[str] = [] + for entry in metric_data: + name = entry.get('MetricName') + width = len(Utils.get_value_as_list('Dimensions', entry, [])) + if name not in chosen: + order.append(name) + chosen[name] = entry + elif width > len(Utils.get_value_as_list('Dimensions', chosen[name], [])): + chosen[name] = entry + return [chosen[name] for name in order] + + def format_entry(self, entry: Dict) -> Optional[str]: + name = entry.get('MetricName') + if Utils.is_empty(name): + return None + value = entry.get('Value') + if value is None: + return None + metric_type = _METRIC_TYPES.get(entry.get('MetricType'), 'g') + tags = list(self.base_tags) + for dimension in Utils.get_value_as_list('Dimensions', entry, []): + key = _TAG_UNSAFE.sub('_', str(dimension.get('Name', '')).strip().lower()) + if Utils.is_empty(key): + continue + tags.append(f'{key}:{self._tag_value(dimension.get("Value"))}') + line = f'{METRIC_PREFIX}.{name}:{self._value(value)}|{metric_type}' + if len(tags) > 0: + line = f'{line}|#{",".join(tags)}' + return line + + def log(self, metric_data: List[Dict]): + lines = [] + for entry in self.complete_entries(metric_data): + line = self.format_entry(entry) + if line is not None: + lines.append(line) + if len(lines) == 0: + return + + datagram = b'' + for line in lines: + encoded = line.encode('utf-8') + if len(datagram) + len(encoded) + 1 > MAX_DATAGRAM_BYTES and len(datagram) > 0: + self._send(datagram) + datagram = b'' + datagram = encoded if len(datagram) == 0 else datagram + b'\n' + encoded + if len(datagram) > 0: + self._send(datagram) + + def _send(self, datagram: bytes): + with self._lock: + try: + if self._socket is None: + self._socket = socket.socket(self._family, socket.SOCK_DGRAM) + self._socket.setblocking(False) + self._socket.sendto(datagram, self._address) + self._send_failures = 0 + except OSError as e: + if self._socket is not None: + self._socket.close() + self._socket = None + self._send_failures += 1 + # the first failure and then one in a hundred: the agent being down is + # one fact, not one log line per metric. + if self._send_failures == 1 or self._send_failures % 100 == 0: + self.logger.warning( + f'dogstatsd send to {self._address} failed ' + f'({self._send_failures} in a row): {e}' + ) + + def flush(self): + # datagrams leave as they are built; nothing is held back. + pass diff --git a/source/idea/idea-sdk/src/ideasdk/metrics/metrics_provider_factory.py b/source/idea/idea-sdk/src/ideasdk/metrics/metrics_provider_factory.py index cfa4cfe3..7536e543 100644 --- a/source/idea/idea-sdk/src/ideasdk/metrics/metrics_provider_factory.py +++ b/source/idea/idea-sdk/src/ideasdk/metrics/metrics_provider_factory.py @@ -12,6 +12,7 @@ from ideadatamodel import constants, errorcodes, exceptions from ideasdk.metrics.cloudwatch.cloudwatch_metrics import CloudWatchMetrics from ideasdk.metrics.prometheus.prometheus_metrics import PrometheusMetrics +from ideasdk.metrics.dogstatsd.dogstatsd_metrics import DogStatsdMetrics from ideasdk.metrics.null_metrics_provider import NullMetrics from ideasdk.protocols import ( MetricsProviderFactoryProtocol, @@ -52,6 +53,10 @@ def get_provider(self, namespace: str) -> MetricsProviderProtocol: metrics_provider = PrometheusMetrics( context=self.context, namespace=namespace ) + elif provider_name == constants.METRICS_PROVIDER_DOGSTATSD: + metrics_provider = DogStatsdMetrics( + context=self.context, namespace=namespace + ) if metrics_provider is None: raise exceptions.SocaException( diff --git a/source/idea/idea-sdk/src/ideasdk/metrics/prometheus/prometheus_metrics.py b/source/idea/idea-sdk/src/ideasdk/metrics/prometheus/prometheus_metrics.py index 1dac7079..7deee33d 100644 --- a/source/idea/idea-sdk/src/ideasdk/metrics/prometheus/prometheus_metrics.py +++ b/source/idea/idea-sdk/src/ideasdk/metrics/prometheus/prometheus_metrics.py @@ -15,6 +15,7 @@ from typing import List, Dict from prometheus_client import Counter, Summary from threading import RLock +import re class PrometheusMetrics(MetricsProviderProtocol): @@ -85,7 +86,9 @@ def _get_or_create_metric(self, entry: Dict): if unit == 'count': unit = 'total' - prometheus_metric_name = f'{metric_name}_{unit}' + prometheus_metric_name = re.sub(r'[^a-zA-Z0-9_:]', '_', metric_name) + if unit not in ('', 'none'): + prometheus_metric_name = f'{prometheus_metric_name}_{unit}' if metric_type == 'Counter': metric = Counter( diff --git a/source/idea/idea-sdk/tests/test_dogstatsd_metrics.py b/source/idea/idea-sdk/tests/test_dogstatsd_metrics.py new file mode 100644 index 00000000..381c3989 --- /dev/null +++ b/source/idea/idea-sdk/tests/test_dogstatsd_metrics.py @@ -0,0 +1,80 @@ +""" +the dogstatsd provider: one line per metric with the namespace and dimensions as tags, +the split entries BaseMetrics adds dropped, delivered over UDP or a socket path. +""" + +from ideasdk.metrics.dogstatsd.dogstatsd_metrics import DogStatsdMetrics + +import pytest +import socket + + +def _entry(name, value, dimensions, metric_type='Counter'): + return { + 'MetricType': metric_type, + 'MetricName': name, + 'Dimensions': [{'Name': k, 'Value': v} for k, v in dimensions], + 'Value': value, + 'Unit': 'Count', + } + + +def test_only_the_complete_entry_is_sent(context): + provider = DogStatsdMetrics(context=context, namespace='idea-mock/mock') + # what BaseMetrics publishes for two dimensions: one entry per dimension, then all + batch = [ + _entry('jobs_finished', 1, [('queue_type', 'compute')]), + _entry('jobs_finished', 1, [('project', 'p1')]), + _entry('jobs_finished', 1, [('queue_type', 'compute'), ('project', 'p1')]), + ] + chosen = provider.complete_entries(batch) + assert len(chosen) == 1 + assert provider.format_entry(chosen[0]) == ( + 'idea.jobs_finished:1|c|#idea_cluster:idea-mock,idea_module:mock,queue_type:compute,project:p1' + ) + + +def test_summary_is_a_distribution_and_tags_are_sanitized(context): + provider = DogStatsdMetrics(context=context, namespace='idea-mock/mock/api') + line = provider.format_entry( + _entry('api_invocations', 12.5, [('Api', 'Jobs.List, all|now')], 'Summary') + ) + assert line == ( + 'idea.api_invocations:12.5|d|#idea_cluster:idea-mock,idea_module:mock,component:api,api:Jobs.List_all_now' + ) + + +def test_values_are_plain_decimals(context): + provider = DogStatsdMetrics(context=context, namespace='idea-mock/mock') + assert provider.format_entry(_entry('job.duration_seconds', 86514.0, [])).startswith( + 'idea.job.duration_seconds:86514|' + ) + assert provider.format_entry(_entry('job.cost', 0.000123, [])).startswith( + 'idea.job.cost:0.000123|' + ) + + +def test_send_reaches_a_udp_listener(context, monkeypatch): + listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + listener.bind(('127.0.0.1', 0)) + listener.settimeout(2) + monkeypatch.setenv('DD_DOGSTATSD_URL', f'udp://127.0.0.1:{listener.getsockname()[1]}') + + provider = DogStatsdMetrics(context=context, namespace='idea-mock/mock') + provider.log([_entry('job.count', 1, [])]) + + assert listener.recv(8192) == b'idea.job.count:1|c|#idea_cluster:idea-mock,idea_module:mock' + listener.close() + + +def test_missing_agent_never_raises(context, monkeypatch): + monkeypatch.setenv('DD_DOGSTATSD_URL', 'unix:///nonexistent/dsd.socket') + provider = DogStatsdMetrics(context=context, namespace='idea-mock/mock') + provider.log([_entry('job.count', 1, [])]) + assert provider._send_failures == 1 + + +def test_unsupported_url_is_rejected(context, monkeypatch): + monkeypatch.setenv('DD_DOGSTATSD_URL', 'http://127.0.0.1:8125') + with pytest.raises(ValueError): + DogStatsdMetrics(context=context, namespace='idea-mock/mock') diff --git a/source/idea/ideactl/.gitignore b/source/idea/ideactl/.gitignore new file mode 100644 index 00000000..310e70c6 --- /dev/null +++ b/source/idea/ideactl/.gitignore @@ -0,0 +1,11 @@ +# working artifacts that carry account ids / cluster names; never commit +docs/port/ +tools/parity/live/ +tools/parity/fixtures/ +node_modules/ +cdk.out*/ +dist/ +tools/e2e/reference/ + +# written by the local database emulator the cluster-config tests start +dynamodb-local-metadata.json diff --git a/source/idea/ideactl/README.md b/source/idea/ideactl/README.md new file mode 100644 index 00000000..fd814da8 --- /dev/null +++ b/source/idea/ideactl/README.md @@ -0,0 +1,145 @@ +# ideactl + +`ideactl` is the TypeScript CDK administrator and cluster CLI for IDEA. It is +shipped with the control-plane CLI in one image, not as a separately deployed +service. The port's primary requirement is strict behavioural and +CloudFormation-template parity with the existing Python administrator: preserve +the construct tree, logical IDs, ordering, types, and deliberate quirks rather +than improving them. + +## Layout + +| Path | Purpose | +| --- | --- | +| `src/cdk/` | CDK app, base stack, stack implementations, constructs, policy rendering, code assets, and replayable synth reads. | +| `src/config/` | Cluster-settings access, values/config generation, ARN construction, and AMI selection. | +| `src/cli/` | The `ideactl` command tree and CDK invocation helpers. | +| `src/lambda/` | Custom-resource and event handler ports, with shared CloudFormation response support in `commons/`. | +| `src/util/` | Compatibility helpers such as identifiers, names, hashes, and YAML handling. | +| `resources-ecs/` | Container-module config templates, overlaid onto `dist/resources/config` during a build. The rest of the resource tree still lives in the administrator package. | +| `tools/parity/` | Offline template comparison, fixture capture, fixture-driven synth, and config flattening. | +| `tools/e2e/` | Opt-in tools for exercising a deployed control plane through its public endpoints. | +| `test/` | Focused `node:test` suites, one directory per area under test. | + +## Compatibility pins + +`package.json` is a compatibility surface, not a dependency wishlist. Exact +pins keep CDK-generated logical IDs, singleton resources, asset conventions, +and bootstrap behaviour reproducible. Keep the CDK library, CDK CLI, +constructs, and compliance package aligned with the recorded templates; do not +refresh one independently. The renderer, YAML parser, CLI parser, and SDK +clients have compatibility version constraints so their parsing and request +behaviour cannot drift. + +Use the installed Node runtime's native TypeScript stripping for focused tests; +do not compile first: + +```sh +node --test 'test/w4/*.test.ts' +``` + +The project is ESM. Relative imports use `.ts` extensions, type-only imports +use `import type`, and TypeScript must remain erasable: no enums, namespaces, +or parameter properties. Use Node built-ins before adding a dependency. + +## Offline parity + +The parity harness compares a synthesized template with the recorded Python +template strictly for deployed state. It intentionally masks only documented +volatile values. It compares security-linter suppression metadata and +`aws:cdk:path` exactly. Other metadata is excluded. + +- `parity.ts` accepts `diff [--ignore-version] ` for a + comparison and `paths ` to report logical IDs and construct + paths. Its result is `PARITY` or `MISMATCH`, followed by resource and property + counts. +- `synth.ts` requires `--cluster` and `--stack`. It supports `--against synth` + for a Python-synth reference, repeatable `--context key=value`, plus + `--ignore-version`, `--deployment-id`, `--app-override`, and `--keep`. It creates a temporary working directory + containing the package CDK configuration and captured context, then runs the + bundled CDK CLI without lookups. +- `capture.ts --from-raw` converts saved command output into the table dumps + and synth-read replay file. `--from-local` copies values, CDK context, and + Python outputs from a local administrator directory. `--live` also captures + live tables, reads, and deployed templates; it requires credentials and must + not be used for fixture-only development. +- `flatten.py` is the independent Layer-A config oracle. It reads a generated + config directory, emits sorted flat JSON, and supports `-o` and + `--key-prefix`. + +Run the harness checks and inspect the flattener interface from this package +directory: + +```sh +node --test 'test/w4/*.test.ts' +python3 tools/parity/flatten.py --help +``` + +Fixture material is gitignored. Use these shapes, never committed live values: + +```text +tools/parity/live/-.json +tools/parity/fixtures//raw/ +tools/parity/fixtures//{cluster-settings.json,modules.json,synth-reads.json} +tools/parity/fixtures//{values.yml,flat.json,cdk.context.json} +tools/parity/fixtures//python/_cdk/cdk.out./ +``` + +Tests using these local fixtures must skip when they are absent. Committed test +data must be synthetic. + +## Lambda handlers + +Each ported handler lives in `src/lambda//index.ts`. Keep the +exported production `handler` thin and put injectable collaborators behind a +factory when calls or responses need observation. The shared +`src/lambda/commons/cfn-response.ts` module defines the CloudFormation event +shape and sends success or failure responses with the same response semantics +as the Python helper. + +A handler test provides a synthetic event and context, injects SDK and response +recorders, then asserts the request payload, response status, physical ID, and +failure behaviour. It must cover every relevant CloudFormation request type +when Python treats them differently. For example: + +```sh +node --test 'test/w24-create_tags/*.test.ts' +``` + +## End-to-end tools + +The E2E tools are deliberately separate from parity. They contact an already +deployed control plane over its load-balancer or gateway endpoint, accept +credentials only by a password-file path, cache tokens locally per user, and +verify TLS unless `--insecure` is explicitly supplied. + +- `api.ts` makes one namespace request and routes it to the appropriate API. +- `load-api.ts` drives the portal-like API request mix and reports latency and + error percentiles. +- `load-gateway.ts` opens, holds, and reports TLS gateway connections. +- `load-jobs.ts` submits a bounded burst of short scheduler jobs and polls for + completion. + +Inspect their accepted flags before running a load against a deployment: + +```sh +node tools/e2e/api.ts --help +node tools/e2e/load-api.ts --help +node tools/e2e/load-gateway.ts --help +node tools/e2e/load-jobs.ts --help +``` + +## Hygiene and completion + +Do not make credentialed calls while developing the port. Real fixture +directories are ignored; do not place account identifiers, resource +identifiers, deployment hostnames, production cluster names, personal data, or +external-assistant names in tracked source, tests, tools, scripts, or images. +Use synthetic identifiers and `example.invalid` hosts in committed tests. + +A stack is declared done only after its offline fixture run prints a `PARITY` +line with zero missing resources, zero extra resources, and zero hard property +differences (asset notices are allowed), and a credentialed live +`ideactl cdk diff` reports `There were no differences`. Exercise the additional +synth-shaped fixtures for branches the recorded deployment does not cover +before calling the port complete. diff --git a/source/idea/ideactl/cdk.context.json b/source/idea/ideactl/cdk.context.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/source/idea/ideactl/cdk.context.json @@ -0,0 +1 @@ +{} diff --git a/source/idea/ideactl/cdk.json b/source/idea/ideactl/cdk.json new file mode 100644 index 00000000..dd6aa74f --- /dev/null +++ b/source/idea/ideactl/cdk.json @@ -0,0 +1,12 @@ +{ + "app": "node dist/src/cdk/app.js", + "context": { + "cli-telemetry": false, + "aws-cdk:enableDiffNoFail": "true", + "@aws-cdk/core:stackRelativeExports": "true", + "@aws-cdk/aws-ecr-assets:dockerIgnoreSupport": true, + "@aws-cdk/aws-kms:defaultKeyPolicies": true, + "@aws-cdk/aws-ecs-patterns:removeDefaultDesiredCount": true, + "@aws-cdk/customresources:installLatestAwsSdkDefault": false + } +} diff --git a/source/idea/ideactl/docs/COMMANDS.md b/source/idea/ideactl/docs/COMMANDS.md new file mode 100644 index 00000000..a1967fcd --- /dev/null +++ b/source/idea/ideactl/docs/COMMANDS.md @@ -0,0 +1,886 @@ +# ideactl command reference + +This page describes what the `ideactl` binary does today. Option spellings in the tables match `ideactl --help`. + +Every command also accepts `-h, --help` (`display help for command`). That flag is omitted from the tables below. With no arguments, `ideactl` prints the root help and exits 0. + +`--aws-profile` is optional wherever it appears. When it is set, that action uses the named profile. `--cluster-name` and `--aws-region` are required by the parser wherever the table says yes, even when `--help` does not print the word required. + +Local cluster files live under `$IDEA_USER_HOME/clusters///` (`IDEA_USER_HOME` defaults to `~/.idea`): `values.yml`, `config/`, `_cdk/`, `deployments/`, `logs/`, `support/`. + +## Exit codes + +These apply to every command unless a command section names a more specific code. + +| Code | When | +| --- | --- | +| 0 | Success, `--help`, or the operator declined a confirmation prompt. | +| 1 | Missing required flags or arguments, invalid values, cluster configuration errors, a refused change set, a missing AWS profile, configuration tables that do not exist, or `ExitWithCode(1)`. | +| other | A spawned `cdk` process returned that code. | +| 1 plus a traceback | Any other thrown error, including a failed integration-test run and an aborted cluster deletion. The process prints `Command failed with error: ...` and then the stack. | + +A configuration table that has not been created prints one red line ending in `Is the cluster configuration synced?` and exits 1. + +## `ideactl` + +IDEA cluster administration. + +**Usage:** `ideactl [options] [command]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `-V, --version` | no | none | no | + +**Reads:** `IDEA_VERSION.txt` for `-V`. **Changes:** nothing. **Example:** `ideactl -h` + +## `about` + +Print the release version. + +**Usage:** `ideactl about [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--no-banner` | no | none | no | + +**Reads:** `IDEA_VERSION.txt` (or `IDEA_VERSION`). **Changes:** nothing. **Exit codes:** 0. **Example:** `ideactl about` + +`--no-banner` is accepted and ignored. The command always prints `ideactl ` and never prints a banner. + +## `quick-setup-help` + +Print the packaged `values.yml` template. + +**Usage:** `ideactl quick-setup-help [options]` + +No command-specific options. + +**Reads:** the packaged `resources/config/values.yml`. **Changes:** nothing. **Example:** `ideactl quick-setup-help` + +## `quick-setup` + +Install a new cluster: generate and update configuration, bootstrap CDK, deploy modules in priority order, then wait for health checks. + +**Usage:** `ideactl quick-setup [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--values-file ` | yes | none | no | +| `--existing-resources` | no | none | no | +| `--termination-protection ` | yes | `"true"` | no | +| `--deployment-id ` | yes | none | no | +| `--optimize-deployment` | no | none | no | +| `--force` | no | none | no | +| `--skip-config` | no | none | no | +| `--rollback` | no | `true` | no | +| `--no-rollback` | no | none | no | +| `--module-set ` | yes | `"default"` | no | +| `--allow-replacement ` | yes, repeatable | none | no | + +**Reads:** `--values-file` or an interactive installer; DynamoDB cluster tables after update. **Changes:** local `values.yml` and `config/`, DynamoDB modules and settings tables, the `-bootstrap` stack, module CloudFormation stacks, bootstrap packages in the cluster bucket. **Exit codes:** 1 if `--skip-config` is set without `--values-file`; 0 if the operator aborts the deploy prompt. + +**Example:** `ideactl quick-setup --values-file ./values.yml --force --module-set default` + +`--termination-protection` is a string. Values `true`, `yes`, `y`, `1`, and `on` (any case) count as true. `--skip-config` skips generate and update and still prints the settings table, modules table, bootstrap, and deploy. + +## `config` + +Configuration options. A group; it prints its subcommand list. + +**Usage:** `ideactl config [options] [command]` + +No command-specific options. + +**Example:** `ideactl config -h` + +## `config generate` + +Render `values.yml` and the local `config/` tree from templates. + +**Usage:** `ideactl config generate [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--values-file ` | yes | none | no | +| `--config-dir ` | yes | none | no | +| `--force` | no | none | no | +| `--existing-resources` | no | none | no | +| `--regenerate` | no | none | no | + +**Reads:** `--values-file` if given, otherwise an interactive installer (STS for the account id). **Changes:** writes `values.yml` and `config/` under `--config-dir` or `~/.idea/clusters///`. **Exit codes:** 1 if `--config-dir` does not exist, or cluster name or region is missing; 0 if the overwrite prompt is declined. + +**Example:** `ideactl config generate --values-file ./values.yml --force` + +With `--values-file`, `--existing-resources` and `--regenerate` are not used. Without `--force`, a non-empty target directory is prompted and then cleared. With `--force`, that cleanup is skipped. + +## `config update` + +Push the local `config/` tree into the cluster settings and modules tables. + +**Usage:** `ideactl config update [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--aws-region ` | yes | none | yes | +| `--force` | no | none | no | +| `--overwrite` | no | none | no | +| `--key-prefix ` | yes | none | no | +| `--config-dir ` | yes | none | no | +| `--module-set ` | yes | `"default"` | no | + +**Reads:** `/config` or `~/.idea/clusters///config`. **Changes:** creates the DynamoDB tables if needed, then syncs modules and settings. Existing keys are skipped unless `--overwrite` is set. **Exit codes:** 1 if the config directory is missing or the local cluster name or region does not match; 0 if the operator chooses Exit. + +**Example:** `ideactl config update --cluster-name sample-cluster --aws-region us-east-2 --force` + +Without `--force`, the prompt is `Yes` / `Reload Changes` / `Exit`. + +## `config set` + +Write one or more typed settings keys. + +**Usage:** `ideactl config set [options] ` + +| Argument | Required | Help text | +| --- | --- | --- | +| `entries` | yes, variadic | `Key=KEY_NAME,Type=[str|int|float|bool|list|list|list|list],Value=VALUE` | + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--aws-region ` | yes | none | yes | +| `--force` | no | none | no | + +**Reads:** nothing local. **Changes:** DynamoDB settings rows via `setConfigEntry`. **Exit codes:** 1 on a malformed entry; 0 if the confirm prompt is declined. + +**Example:** `ideactl config set --cluster-name sample-cluster --aws-region us-east-2 --force 'Key=cluster.locale,Type=str,Value=en_US'` + +`list` is accepted and stored as a list of strings. Keys may not contain `,` or `:`. + +## `config show` + +Print cluster settings. The help summary says yaml. The default output is a table. + +**Usage:** `ideactl config show [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--aws-region ` | yes | none | yes | +| `-q, --query ` | yes | none | no | +| `--format ` | yes | none (omitted means table; choices `table`, `yaml`, `raw`) | no | + +**Reads:** DynamoDB `.cluster-settings`. `--aws-region` is required by the parser and is not used to name the table. **Changes:** nothing. + +**Example:** `ideactl config show --cluster-name sample-cluster --aws-region us-east-2 --format yaml` + +`--query` is a regular expression matched from the start of the key, not a search anywhere in the key. + +## `config export` + +Write the DynamoDB settings back to a `config/` tree. Refuses a non-empty target directory. + +**Usage:** `ideactl config export [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--aws-region ` | yes | none | yes | +| `--export-dir ` | yes | cluster `config/` directory | no | + +**Reads:** `.cluster-settings` and `.modules`. **Changes:** writes `idea.yml` and `/settings.yml` under the export directory. + +**Example:** `ideactl config export --cluster-name sample-cluster --aws-region us-east-2 --export-dir /tmp/sample-cluster-config` + +## `config delete` + +Delete every settings key under each given prefix. There is no confirmation prompt. + +**Usage:** `ideactl config delete [options] ` + +| Argument | Required | Help text | +| --- | --- | --- | +| `config-key-prefixes` | yes, variadic | config key prefixes | + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--aws-region ` | yes | none | yes | + +**Reads:** nothing local. **Changes:** DynamoDB settings rows matching each prefix. **Example:** `ideactl config delete --cluster-name sample-cluster --aws-region us-east-2 global-settings.custom_tags` + +## `config diff` + +Compare local `config/` files to the settings table as MODIFIED, DELETED, or ADDED. + +**Usage:** `ideactl config diff [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--aws-region ` | yes | none | yes | +| `--config-dir ` | yes | cluster `config/` directory | no | + +**Reads:** local `config/` and DynamoDB `.cluster-settings`. **Changes:** nothing. **Example:** `ideactl config diff --cluster-name sample-cluster --aws-region us-east-2` + +## `config preview-upgrade` + +Print the configuration drift an upgrade would apply, without writing it. + +**Usage:** `ideactl config preview-upgrade [options] [modules...]` + +| Argument | Required | Help text | +| --- | --- | --- | +| `modules` | no, variadic | module ids | + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--aws-region ` | yes | none | yes | +| `--base-os ` | yes | none | no | +| `--values-file ` | yes | cluster `values.yml` | no | +| `--skip-global-settings-update` | no | none | no | + +**Reads:** cluster tables, local or supplied `values.yml`, AMI maps. **Changes:** nothing. **Example:** `ideactl config preview-upgrade --cluster-name sample-cluster --aws-region us-east-2 --base-os amazonlinux2023` + +## `config save-values` + +Upload `values.yml` to the cluster bucket at `values/values.yml`. + +**Usage:** `ideactl config save-values [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--aws-region ` | yes | none | yes | +| `--values-file ` | yes | cluster `values.yml` | no | + +**Reads:** the values file; DynamoDB `cluster.cluster_s3_bucket` or STS plus the conventional bucket name. **Changes:** S3 object `values/values.yml`. **Example:** `ideactl config save-values --cluster-name sample-cluster --aws-region us-east-2` + +## `config download-values` + +Download `values/values.yml` from the cluster bucket. + +**Usage:** `ideactl config download-values [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--aws-region ` | yes | none | yes | +| `--values-dir ` | yes | none | no | + +**Reads:** S3 `values/values.yml`. **Changes:** writes `values.yml` to `--values-dir` or the cluster directory. Reloads the body through YAML so the file is a dump of the parsed document, not the raw object bytes. **Exit codes:** 1 if the object is missing. + +**Example:** `ideactl config download-values --cluster-name sample-cluster --aws-region us-east-2 --values-dir /tmp/sample-cluster` + +## `cdk` + +CDK options. A group. + +**Usage:** `ideactl cdk [options] [command]` + +No command-specific options. **Example:** `ideactl cdk -h` + +## `cdk synth` + +Synthesize the CloudFormation template for one module. + +**Usage:** `ideactl cdk synth [options] ` + +| Argument | Required | Help text | +| --- | --- | --- | +| `module` | yes | module id | + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--deployment-id ` | yes | none | no | +| `--module-set ` | yes | `"default"` | no | + +**Reads:** DynamoDB cluster config. **Changes:** writes under the cluster `_cdk/` directory and runs `cdk synth`. **Example:** `ideactl cdk synth --cluster-name sample-cluster --aws-region us-east-2 metrics` + +## `cdk diff` + +Compare one module template to the deployed stack. + +**Usage:** `ideactl cdk diff [options] ` + +| Argument | Required | Help text | +| --- | --- | --- | +| `module` | yes | module id | + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--deployment-id ` | yes | none | no | +| `--module-set ` | yes | `"default"` | no | + +**Reads:** DynamoDB cluster config and the live stack. **Changes:** none besides local CDK output. **Example:** `ideactl cdk diff --cluster-name sample-cluster --aws-region us-east-2 metrics` + +## `cdk cdk-app` + +Build exactly one stack and synthesize it. This is the `--app` re-entry the CDK CLI runs, not an operator command. + +**Usage:** `ideactl cdk cdk-app [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--aws-region ` | yes | none | yes | +| `--module-name ` | yes | none | yes | +| `--module-id ` | yes | none | yes | +| `--deployment-id ` | yes | none | no | +| `--termination-protection ` | yes | `"true"` | no | +| `--config-file ` | yes | none | no | +| `--synth-reads ` | yes | none | no | + +**Reads:** DynamoDB, or `--config-file` and `--synth-reads` replay files. **Changes:** writes a CDK assembly under the process working directory. **Example:** `ideactl cdk cdk-app --cluster-name sample-cluster --aws-region us-east-2 --module-name metrics --module-id metrics` + +## `bootstrap` + +Render the CDK toolkit template and run `cdk bootstrap` for the cluster. + +**Usage:** `ideactl bootstrap [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--aws-region ` | yes | none | yes | +| `--termination-protection ` | yes | `"true"` | no | +| `--custom-permissions-boundary ` | yes | `""` | no | +| `--cloudformation-execution-policies ` | yes | `""` | no | +| `--public-access-block-configuration ` | yes | `"true"` | no | +| `--module-set ` | yes | `"default"` | no | + +**Reads:** DynamoDB `cluster.cluster_s3_bucket` and custom tags. **Changes:** writes `_cdk/cdk_toolkit_stack.yml`, then creates or updates `-bootstrap` and uses the cluster bucket as the CDK staging bucket. Empty permissions-boundary and execution-policy strings are omitted from the CDK argv. **Exit codes:** the `cdk` process code. + +**Example:** `ideactl bootstrap --cluster-name sample-cluster --aws-region us-east-2` + +## `deploy` + +Deploy module stacks. `all` may be the only module id and means every undeployed module (or every module with `--upgrade`). + +**Usage:** `ideactl deploy [options] ` + +| Argument | Required | Help text | +| --- | --- | --- | +| `modules` | yes, variadic | module ids, or `all` | + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--termination-protection ` | yes | `"true"` | no | +| `--deployment-id ` | yes | none | no | +| `--upgrade` | no | none | no | +| `--force-build-bootstrap` | no | none | no | +| `--rollback` | no | `true` | no | +| `--no-rollback` | no | none | no | +| `--optimize-deployment` | no | none | no | +| `--module-set ` | yes | `"default"` | no | +| `--allow-replacement ` | yes, repeatable | none | no | + +**Reads:** DynamoDB modules and settings. If the deployment includes `ecs`, also reads the account `awsvpcTrunking` setting. **Changes:** bootstrap packages in the cluster bucket, CloudFormation stacks via a change set that is inspected before execute, `deployments//-outputs.json`. Using `all` with any other module id exits 1 (`fatal error - use of "all" deployment must be the only requested module`). If `awsvpcTrunking` is not enabled, the command prints the exact `aws ecs put-account-setting-default` command and exits 1 without deploying. + +**Example:** `ideactl deploy --cluster-name sample-cluster --aws-region us-east-2 metrics` + +A change set that would replace or remove a stateful resource is refused unless that logical id is passed to `--allow-replacement`. + +## `replace` + +Replace one stateful component, deliberately. An upgrade and a migration never replace one: the change-set guard refuses it, and the synthesized templates carry `UpdateReplacePolicy: Retain` on the resources an upgrade has never replaced, so an accidental replacement stops rather than proceeding. This is the path for the case where it is intended. + +The jump host and the scheduler host are listed here but are outside that protection: an ordinary upgrade replaces both, so they carry no retain policy and this command is simply the deliberate way to do on purpose what an upgrade does on its own. + +**Usage:** `ideactl replace [options] ` + +| Argument | Required | Help text | +| --- | --- | --- | +| `component` | yes | one of: jump-host, search-domain, directory, user-pool, scheduler-host, shared-file-system, backup-vault | + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--module-set ` | yes | `"default"` | no | +| `--deployment-id ` | yes | none | no | +| `--confirm ` | yes | none | no | + +The command always prints what is lost before anything else happens, then stops unless `--confirm` repeats the component name exactly. No other value proceeds, including `yes` or `true`. `shared-file-system` and `backup-vault` print their consequence and refuse: moving to a new file system is a copy while both exist, and a replacement backup vault is empty with no way to move the old recovery points into it. + +It does not create a replacement by itself. CloudFormation replaces a resource when a property that cannot be changed in place changes, so the order is: change the setting, then run this. The printed warning names the properties that force it for that component. If the change set holds no replacement of that component, the deploy runs and nothing is replaced. + +**Reads:** DynamoDB modules and settings. **Changes:** the one module stack that owns the component, through the same inspected change set as `deploy`, with the replacement permitted for that one resource type only. A removal is never permitted by this command. + +**Example:** `ideactl replace search-domain --cluster-name sample-cluster --aws-region us-east-2 --confirm search-domain` + +## `check-cluster-status` + +GET each app module `/healthcheck` and the analytics dashboards URL. TLS is not verified. + +**Usage:** `ideactl check-cluster-status [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--wait` | no | none | no | +| `--wait-timeout ` | yes | `900` | no | +| `--debug` | no | none | no | +| `--module-set ` | yes | `"default"` | no | + +**Reads:** DynamoDB cluster config, then HTTPS GET. **Changes:** nothing. **Exit codes:** 1 if any endpoint is not HTTP 200 after the last pass. + +**Example:** `ideactl check-cluster-status --cluster-name sample-cluster --aws-region us-east-2 --wait --wait-timeout 120` + +Without `--wait`, `--wait-timeout` does not change the single pass. With `--wait`, the loop sleeps 60 seconds between passes until every endpoint succeeds or the timeout is reached. + +## `list-modules` + +Print Title, Name, Module ID, Type, Stack Name, Version, and Status. + +**Usage:** `ideactl list-modules [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | + +**Reads:** DynamoDB `.modules`. **Changes:** nothing. **Example:** `ideactl list-modules --cluster-name sample-cluster --aws-region us-east-2` + +## `show-connection-info` + +Print portal, bastion SSH, Session Manager, and analytics URLs for deployed modules. + +**Usage:** `ideactl show-connection-info [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--module-set ` | yes | `"default"` | no | + +**Reads:** DynamoDB cluster config. **Changes:** nothing. **Example:** `ideactl show-connection-info --cluster-name sample-cluster --aws-region us-east-2` + +If nothing is deployed, the command prints an error to stderr and still exits 0. + +## `upgrade-cluster` + +Upgrade an existing cluster: refuse EOL base OS that is still referenced, preview drift, then run phases 1 to 4 (values base OS, global settings backup and rewrite, optional full config sync, AMI and instance-type keys, then module deploy). Empty `modules` means every module. + +**Usage:** `ideactl upgrade-cluster [options] [modules...]` + +| Argument | Required | Help text | +| --- | --- | --- | +| `modules` | no, variadic | module ids | + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--termination-protection ` | yes | `"true"` | no | +| `--deployment-id ` | yes | none | no | +| `--base-os ` | yes | none | no | +| `--force-build-bootstrap` | no | none | no | +| `--rollback` | no | `true` | no | +| `--no-rollback` | no | none | no | +| `--optimize-deployment` | no | none | no | +| `--module-set ` | yes | `"default"` | no | +| `--force` | no | none | no | +| `--accept-config-drift` | no | none | no | +| `--skip-global-settings-update` | no | none | no | +| `--disable-eol-stacks-in-use` | no | none | no | + +**Reads:** cluster tables, `values.yml`, AMI maps, EC2 images and instance types, OpenSearch instance types, eVDI software-stack tables. **Changes:** `values.yml`, a `config.golden./` copy, DynamoDB settings, instance termination protection (cleared then restored), module stacks, and an upload of `values.yml` to the cluster bucket. The scheduler should be drained before this run. **Exit codes:** 1 on EOL refusal, missing AMI, unsupported instance type, or configuration rows the run would overwrite whose value differs from generated configuration without `--accept-config-drift`; 0 if a confirmation is declined. + +**Example:** `ideactl upgrade-cluster --cluster-name sample-cluster --aws-region us-east-2 --base-os amazonlinux2023 --force` + +`awsvpcTrunking` is checked whenever the run reaches the `ecs` module, which includes an all-module upgrade of a cluster that has it. `--disable-eol-stacks-in-use` disables in-use EOL eVDI stacks instead of refusing. The drift preview stops the run only where a row it overwrites differs from generated configuration, names those rows, and asks; `--force` skips the other confirmations but does not accept those rows, which is what `--accept-config-drift` is for. + +## `migrate` + +Run or resume the one-phase control-plane migration: the fixed serial order of 22 durable boundaries that moves an existing cluster to the container control plane in one operation. Progress lives in a conditionally written record in the cluster bucket, so an interrupted run resumes at the boundary it stopped on instead of starting again. + +**Usage:** `ideactl migrate [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--state-bucket ` | yes | none | yes | +| `--target-base-os ` | yes | none | no | +| `--image-digest ` | yes | none | no | +| `--module-set ` | yes | `"default"` | no | +| `--selected-module ` | yes, repeatable | none | no | +| `--deployment-id ` | yes | none | no | +| `--resume ` | yes | none | no | +| `--accept-template-comparison ` | yes | none | no | +| `--accept-drift ` | yes | none | no | + +**Reads:** the cluster tables, every deployed module-stack template and status, the tagged instance, load balancer, listener, target group and private DNS inventory, the effective account setting for task network interface trunking, the stored `values.yml`, and the external endpoint's health. **Changes:** the operation record and the before-state capture, both objects in the cluster bucket. A step that changes the cluster refuses before its started marker is written when this release cannot execute or verify it, so a run that cannot finish changes nothing. **Exit codes:** 1 on any refusal. + +**Example:** `ideactl migrate --cluster-name sample-cluster --aws-region us-east-2 --state-bucket sample-cluster-cluster-bucket --target-base-os amazonlinux2023 --image-digest registry.example.invalid/control-plane@sha256: --selected-module cluster --selected-module cluster-manager` + +A new run needs `--target-base-os`, `--image-digest` and at least one `--selected-module`; `--resume ` takes those from the record instead and cannot be combined with `--deployment-id`. The image reference must be immutable, ending in `@sha256:` and 64 hexadecimal characters. Pre-flight is a refusal, not a repair: it checks task network interface trunking, that the target-template comparison was accepted for exactly the templates deployed now, and that no configuration row the run overwrites holds a value the generator would not produce. The last two are accepted by fingerprint, which the refusal prints, so an acceptance cannot outlive the report it was given for. Pre-flight also prints what it could not check, and the migration steps this release cannot yet execute along with the missing piece each one waits on. + +## `delete-cluster` + +Delete a cluster. Bootstrap, databases, backups, and log groups stay unless their flags (or `--delete-all`) are set. The bootstrap bucket is retained unless `--delete-bootstrap` or `--delete-all` is set. + +**Usage:** `ideactl delete-cluster [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--delete-bootstrap` | no | none | no | +| `--delete-databases` | no | none | no | +| `--delete-backups` | no | none | no | +| `--delete-cloudwatch-logs` | no | none | no | +| `--delete-all` | no | none | no | +| `--force` | no | none | no | + +**Reads:** tagged EC2 instances and CloudFormation stacks, Cognito user pools, backup vault, DynamoDB table names, log groups. **Changes:** terminates instances, deletes stacks (identity-provider and cluster last), optionally recovery points, tables, log groups, the bootstrap stack, and the cluster bucket. **Exit codes:** unhandled abort errors exit 1 with a traceback. Declining the first prompt returns 0. + +**Example:** `ideactl delete-cluster --cluster-name sample-cluster --aws-region us-east-2 --force` + +## `delete-backups` + +Delete completed or expired recovery points in `-cluster-backup-vault`. + +**Usage:** `ideactl delete-backups [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--force` | no | none | no | + +**Reads:** the backup vault. **Changes:** recovery points. Declining the prompt returns without deleting. **Example:** `ideactl delete-backups --cluster-name sample-cluster --aws-region us-east-2 --force` + +## `sso` + +Single sign-on configuration. A group. + +**Usage:** `ideactl sso [options] [command]` + +No command-specific options. **Example:** `ideactl sso -h` + +## `sso show-idp-info` + +Print the Cognito redirect URL, and for SAML the entity id. + +**Usage:** `ideactl sso show-idp-info [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--provider-type ` | yes | none | yes | + +**Reads:** DynamoDB identity-provider keys. **Changes:** nothing. **Example:** `ideactl sso show-idp-info --cluster-name sample-cluster --aws-region us-east-2 --provider-type OIDC` + +`--provider-type` must be `SAML` or `OIDC` (case is folded for the check, then compared as given for the URL path). Help has no description text for this command or its flags. + +## `sso configure` + +Create or update the Cognito identity provider and app client, store the client secret, link existing users, then set `cognito.sso_enabled` to true. + +**Usage:** `ideactl sso configure [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--provider-name ` | yes | none | yes | +| `--provider-type ` | yes | none | yes | +| `--provider-email-attribute ` | yes | none | yes | +| `--refresh-token-validity-hours ` | yes | none (code uses 12 when omitted or `<= 0`) | no | +| `--oidc-client-id ` | yes | none | no | +| `--oidc-client-secret ` | yes | none | no | +| `--oidc-issuer ` | yes | none | no | +| `--oidc-attributes-request-method ` | yes | none (code uses `GET`) | no | +| `--oidc-authorize-scopes ` | yes | none (code uses `openid`) | no | +| `--oidc-authorize-url ` | yes | none | no | +| `--oidc-token-url ` | yes | none | no | +| `--oidc-attributes-url ` | yes | none | no | +| `--oidc-jwks-uri ` | yes | none | no | +| `--saml-metadata-url ` | yes | none | no | +| `--saml-metadata-file ` | yes | none | no | + +**Reads:** cluster config, optional SAML metadata file. **Changes:** Cognito IdP and user pool client, Secrets Manager secret `-sso-client-secret`, identity-provider settings keys. OIDC requires client id, secret, and issuer. SAML requires metadata URL or file. Invalid SSO input is printed to stdout and the command still exits 0. + +**Example:** `ideactl sso configure --cluster-name sample-cluster --aws-region us-east-2 --provider-name ExampleIdp --provider-type OIDC --provider-email-attribute email --oidc-client-id example-client --oidc-client-secret example-secret --oidc-issuer https://idp.example.invalid` + +## `directoryservice` + +Directory service commands. A group. + +**Usage:** `ideactl directoryservice [options] [command]` + +No command-specific options. **Example:** `ideactl directoryservice -h` + +## `directoryservice create-service-account-secrets` + +Create username and password secrets. This command does not read cluster configuration. + +**Usage:** `ideactl directoryservice create-service-account-secrets [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--username ` | yes | none | no | +| `--password ` | yes | none | no | +| `--kms-key-id ` | yes | none | no | +| `--purpose ` | yes | none | no | + +**Reads:** nothing from the cluster tables. Prompts for username and password when either is missing. **Changes:** Secrets Manager secrets named `-directoryservice--username` and `-directoryservice--password`. If credentials are supplied and `--purpose` is omitted, the name uses the literal `None`. **Example:** `ideactl directoryservice create-service-account-secrets --cluster-name sample-cluster --aws-region us-east-2 --username svc --password "example-pass" --purpose service-account` + +## `shared-storage` + +Shared storage commands. A group. + +**Usage:** `ideactl shared-storage [options] [command]` + +No command-specific options. **Example:** `ideactl shared-storage -h` + +## `shared-storage add-file-system` + +Interactive questionnaire to create a new file system in cluster settings, then optionally deploy the shared-storage module. + +**Usage:** `ideactl shared-storage add-file-system [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | no | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--kms-key-id ` | yes | none | no | + +**Reads:** prompts; optional EFS/FSx describe calls. The live adapter still requires a cluster name even though the parser does not. **Changes:** shared-storage settings keys, and a module deploy if the operator picks that next step. **Example:** `ideactl shared-storage add-file-system --cluster-name sample-cluster --aws-region us-east-2` + +## `shared-storage attach-file-system` + +Same questionnaire as add-file-system, for an existing file system (`use_existing_fs: true`). + +**Usage:** `ideactl shared-storage attach-file-system [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | no | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--kms-key-id ` | yes | none | no | + +**Reads:** prompts plus EFS/FSx describe. **Changes:** shared-storage settings. Does not offer the deploy next step. **Example:** `ideactl shared-storage attach-file-system --cluster-name sample-cluster --aws-region us-east-2` + +## `utils` + +Utility commands. A group. + +**Usage:** `ideactl utils [options] [command]` + +No command-specific options. **Example:** `ideactl utils -h` + +## `utils aws-services` + +Print the static required/optional AWS service matrix. No AWS calls. + +**Usage:** `ideactl utils aws-services [options]` + +No command-specific options. **Example:** `ideactl utils aws-services` + +## `utils check-aws-services` + +Print which of those services exist in each requested region (SSM global-infrastructure parameters). + +**Usage:** `ideactl utils check-aws-services [options] ` + +| Argument | Required | Help text | +| --- | --- | --- | +| `aws-regions` | yes, variadic | (none) | + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--aws-profile ` | yes | none | no | + +**Reads:** SSM `/aws/service/global-infrastructure/regions//services`. The first region is also used to build the AWS client. **Changes:** nothing. **Example:** `ideactl utils check-aws-services us-east-2 us-west-2` + +## `utils vpc-endpoints` + +VPC endpoint commands. A group. + +**Usage:** `ideactl utils vpc-endpoints [options] [command]` + +No command-specific options. **Example:** `ideactl utils vpc-endpoints -h` + +## `utils vpc-endpoints service-info` + +Print whether each IDEA gateway and interface endpoint is available in the region. + +**Usage:** `ideactl utils vpc-endpoints service-info [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | + +**Reads:** EC2 `DescribeVpcEndpointServices`. **Changes:** nothing. **Example:** `ideactl utils vpc-endpoints service-info --aws-region us-east-2` + +## `utils cluster-prefix-list` + +Cluster prefix list commands. A group. + +**Usage:** `ideactl utils cluster-prefix-list [options] [command]` + +No command-specific options. **Example:** `ideactl utils cluster-prefix-list -h` + +## `utils cluster-prefix-list show` + +Print CIDR entries from the cluster managed prefix list. + +**Usage:** `ideactl utils cluster-prefix-list show [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | + +**Reads:** DynamoDB `cluster.network.cluster_prefix_list_id`, then EC2 prefix-list entries. **Changes:** nothing. **Example:** `ideactl utils cluster-prefix-list show --cluster-name sample-cluster --aws-region us-east-2` + +## `utils cluster-prefix-list add-entry` + +Add a CIDR to the cluster prefix list. + +**Usage:** `ideactl utils cluster-prefix-list add-entry [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--cidr ` | yes | none | yes | +| `--description ` | yes | none | yes | + +**Reads:** the prefix list id and current version. **Changes:** EC2 managed prefix list. Refuses a CIDR that is already present. **Example:** `ideactl utils cluster-prefix-list add-entry --cluster-name sample-cluster --aws-region us-east-2 --cidr 192.0.2.0/24 --description office` + +## `utils cluster-prefix-list remove-entry` + +Remove a CIDR from the cluster prefix list. + +**Usage:** `ideactl utils cluster-prefix-list remove-entry [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--cidr ` | yes | none | yes | + +**Reads:** the prefix list id and current version. **Changes:** EC2 managed prefix list. **Example:** `ideactl utils cluster-prefix-list remove-entry --cluster-name sample-cluster --aws-region us-east-2 --cidr 192.0.2.0/24` + +## `backup-update-global-settings` + +Copy local `config/` to `config.golden./`, regenerate from `values.yml`, and replace only `global-settings` keys in DynamoDB. + +**Usage:** `ideactl backup-update-global-settings [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--force` | no | none | no | +| `--module-set ` | yes | none (help text says default; the parser has no default, ClusterConfig still uses `default`) | no | + +**Reads:** cluster config export, `values.yml`. **Changes:** local `config/`, golden backup directory, DynamoDB `global-settings.*`. Without `--force`, a confirmation prompt can abort with a message and exit 0. **Example:** `ideactl backup-update-global-settings --cluster-name sample-cluster --aws-region us-east-2 --force` + +Help has no command description. + +## `support` + +Support options. A group. + +**Usage:** `ideactl support [options] [command]` + +No command-specific options. **Example:** `ideactl support -h` + +## `support deployment` + +Build a deployment debug archive. + +**Usage:** `ideactl support deployment [options]` + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--module-set ` | yes | `"default"` | no | + +**Reads:** local `logs/`, `values.yml`, `config/`, and a DynamoDB dump when those package contents are selected. The default contents (when nothing is chosen) are deployment logs, `values.yml`, the database dump, and local config. CDK config and the deployments directory are included only if chosen. **Changes:** writes `support/idea-deployment-debug-pkg-/` and a `.tar.gz` next to it. **Example:** `ideactl support deployment --cluster-name sample-cluster --aws-region us-east-2` + +## `run-integration-tests` + +Run shipped module integration-test cases. + +**Usage:** `ideactl run-integration-tests [options] ` + +| Argument | Required | Help text | +| --- | --- | --- | +| `modules` | yes, variadic | (none) | + +| Flag | Value | Default | Required | +| --- | --- | --- | --- | +| `--cluster-name ` | yes | none | yes | +| `--aws-region ` | yes | none | yes | +| `--aws-profile ` | yes | none | no | +| `--admin-username ` | yes | none | yes | +| `--admin-password ` | yes | none | yes | +| `--test-case-id ` | yes | none | no | +| `--debug` | no | none | no | +| `-p, --param ` | yes, repeatable | none | no | +| `--module-set ` | yes | none (help text says default; the parser has no default, ClusterConfig still uses `default`) | no | + +**Reads:** cluster modules table; each test case may call cluster APIs. **Changes:** whatever the selected tests change. Duplicate module ids are dropped. `--test-case-id` is a comma-separated list. `-p` keeps the last value for a duplicate key and ignores tokens without `=`. A module that is not deployed, or a failed case, ends the run. Failed cases throw after printing `[FAIL]`; that error is not mapped to a quiet exit, so the process prints a traceback. + +**Example:** `ideactl run-integration-tests --cluster-name sample-cluster --aws-region us-east-2 --admin-username clusteradmin --admin-password "example-pass" cluster-manager` + +Help has no command description. + +## `help` + +Commander built-in. Prints help for the program or for a named command. + +**Usage:** `ideactl help [command]` + +**Example:** `ideactl help deploy` diff --git a/source/idea/ideactl/resources-ecs/config/templates/ecs/settings.yml b/source/idea/ideactl/resources-ecs/config/templates/ecs/settings.yml new file mode 100644 index 00000000..d7b2a9ad --- /dev/null +++ b/source/idea/ideactl/resources-ecs/config/templates/ecs/settings.yml @@ -0,0 +1,48 @@ +# ECS Settings + +# The ECS stack writes service and target group settings after deployment. +enabled: {{ enable_ecs | default(false) | lower }} +# The release image for this partition, at the release version tag. It must stay equal to the +# matching image_repositories entry below. The stack reads a plain registry reference, so a deploy +# may replace this with a digest-qualified reference to the same manifest. A partition with no +# repository leaves this null, and the stack then refuses rather than pulling nothing. +image: {% if aws_partition == 'aws' %}public.ecr.aws/s5o2b4m0/idea-control-plane:{{ idea_release_version }}{% else %}~{% endif %} + +# Deploy selects this repository using cluster.aws.partition, then stores the +# tagged or digest-qualified result in ecs.image. +image_repositories: + aws: public.ecr.aws/s5o2b4m0/idea-control-plane + aws-us-gov: ~ + +hosts: + instance_type: m7g.large + min: 3 + max: 4 + volume_size: 60 + +tasks: + cluster-manager: + cpu: 256 + memory: 1024 + desired: 2 + vdc: + cpu: 256 + memory: 1024 + desired: 2 + scheduler: + cpu: 512 + memory: 2048 + desired: 1 + dcv-broker: + cpu: 512 + memory: 4096 + desired: 2 + dcv-gateway: + cpu: 256 + memory: 512 + desired: 2 + +datadog: + enabled: false + api_key_secret_arn: ~ + image: public.ecr.aws/datadog/agent:7.83.1 diff --git a/source/idea/ideactl/resources-ecs/observability.yml b/source/idea/ideactl/resources-ecs/observability.yml new file mode 100644 index 00000000..40bfd152 --- /dev/null +++ b/source/idea/ideactl/resources-ecs/observability.yml @@ -0,0 +1,181 @@ +# Observability continuity for the container cutover. +# Placeholders: {cluster}, {cm-id}, {scheduler-id}, {vdc-id}. +# Retention for adopted agent groups is cluster.cloudwatch_logs.retention_in_days (default 90). + +retention_in_days: 90 + +preserved_log_groups: + - name: "/{cluster}/{cm-id}" + stream_prefix: application + verdict: fix + fix: adopt the agent group, keep 90-day retention, awslogs plus a file-tail sidecar for application.log + - name: "/{cluster}/{scheduler-id}" + stream_prefix: application + verdict: fix + fix: same adopt, retention, awslogs, and application.log sidecar + - name: "/{cluster}/{scheduler-id}/openpbs" + stream_prefix: openpbs + verdict: fix + fix: sidecar tails PBS_HOME server_logs, sched_logs, and server_priv/accounting + - name: "/{cluster}/{vdc-id}/controller" + stream_prefix: application + verdict: fix + fix: same adopt, retention, awslogs, and application.log sidecar + - name: "/{cluster}/{vdc-id}/dcv-broker" + stream_prefix: dcv-session-manager-broker + verdict: fix + fix: adopt the agent group, awslogs on stdout, sidecar tails /var/log/dcv-session-manager-broker + - name: "/{cluster}/{vdc-id}/dcv-connection-gateway" + stream_prefix: dcv-connection-gateway + verdict: fix + fix: adopt the agent group, awslogs on stdout, sidecar tails /var/log/dcv-connection-gateway + +preserved_metric_namespaces: + - name: "{cluster}/{cm-id}" + verdict: fix + fix: IDEA_MODULE_ID is the cluster-manager module id so PutMetricData keeps this namespace + - name: "{cluster}/{scheduler-id}" + verdict: fix + fix: IDEA_MODULE_ID is the scheduler module id + - name: "{cluster}/{vdc-id}/controller" + verdict: fix + fix: IDEA_MODULE_ID is the VDC module id, and the controller process appends /controller + - name: "{cluster}" + metric: DCV broker fleet dimension metrics-fleet-name-dimension + verdict: fix + fix: broker properties keep metrics-fleet-name-dimension equal to the cluster name + +preserved_application_metric_names: + - api_invocations_count + - api_invocations_duration + - count + - jobs_pending + - jobs_provisioned + - jobs_running + - jobs_finished + - jobs_pending_duration + - jobs_provisioning_duration + - jobs_running_duration + - jobs_total_duration + - nodes_ready_duration + - node_housekeeping_duration + - node_housekeeping_failed + - job_cache_sync_failed + - instance_cache_sync_failed + - job.count + - job.duration_seconds + - job.cost + - job.cost_ondemand + - job.savings + - job.cpu_efficiency + +planned_target_group_identifiers: + - cm-ecs-e + - cm-ecs-i + - cm-ecs-w + - vdc-ecs-e + - vdc-ecs-i + - sched-ecs-e + - sched-ecs-i + - brk-ecs-c + - brk-ecs-a + - brk-ecs-g + - gw-ecs-TN + - gw-ecs-TUN + +new_surfaces: + - name: "/{cluster}/ecs/datadog" + stream_prefix: datadog + verdict: new + note: used only when ecs.datadog.enabled is true, never the gateway group + - name: ECS/ContainerInsights + verdict: new + note: classic Container Insights on cluster {cluster}-ecs + - name: "/aws/ecs/containerinsights/{cluster}-ecs/performance" + verdict: new + - name: AWS/ECS + verdict: new + +dropped: + - id: host-metrics-per-module + surface: CloudWatch agent cpu, disk, diskio, mem, net, netstat, processes, swap in {cluster}/{module} + verdict: drop + reason: nothing publishes those series after the module instances are gone. Use ECS/ContainerInsights and AWS/ECS by service name. + - id: syslog-streams + surface: system_{ip} and syslog_{ip} streams on the five moved roles + verdict: drop + reason: awslogs captures container stdout, not host syslog. + - id: asg-ec2-metrics + surface: AWS/EC2 and AWS/AutoScaling on the five deleted module groups, including scheduler detailed monitoring + verdict: drop + reason: those resources are deleted. Retarget alarms before cutover. + - id: prometheus-node-exporter + surface: node_exporter and app_exporter on module hosts + verdict: drop + reason: only clusters whose metrics.provider is prometheus or amazon_managed_prometheus used it. Not re-homed on ECS hosts. + - id: session-manager-module-vm + surface: SSM onto a module VM to read /opt/idea/app/logs or PBS_HOME + verdict: drop + reason: there is no module VM. Logs are in the preserved CloudWatch groups. + - id: hostname-cardinality + surface: one hostname dimension per module VM + verdict: drop + reason: three shared ECS hosts are not five module-shaped series. + - id: ip-stream-names + surface: log stream suffix _{ip_address} + verdict: drop + reason: tasks have no stable IP in stream names. Query {prefix}/{container}/{task-id}. + - id: broker-instance-id-dimension + surface: DCV broker CloudWatch series that attached InstanceId from instance metadata + verdict: drop + reason: tasks block instance metadata. Fleet name {cluster} remains. + - id: asg-name-as-autoscaling-group + surface: settings keys asg_name and asg_arn treated as Auto Scaling groups + verdict: drop + reason: the keys stay, the values become the ECS service name and ARN. + - id: empty-dashboard + surface: CloudWatch dashboard {cluster}_{region} with body {"widgets":[]} + verdict: drop + reason: it is empty today, so leaving it empty is not a cutover regression. + - id: lambda-log-retention + surface: /aws/lambda/{cluster}-* retention + verdict: drop + reason: already unset, unrelated to ECS. + +regressions: + - id: log-group-create-collision + verdict: fix + note: do not emit AWS::Logs::LogGroup for agent names. Adopt via create-or-ignore, never delete. + - id: log-group-names + verdict: fix + note: groups use module ids, not container role strings. + - id: log-retention-90 + verdict: fix + note: adopted groups keep cluster.cloudwatch_logs.retention_in_days, default 90, not the CDK two-year default. + - id: application-file-logs + verdict: fix + note: production profile still writes files. A sidecar tails them into the preserved group. + - id: openpbs-logs + verdict: fix + note: OpenPBS files stay in /{cluster}/{scheduler-id}/openpbs. + - id: gateway-broker-file-logs + verdict: fix + note: file-tail sidecars keep the agent file destinations in the preserved groups. + - id: stream-prefix + verdict: fix + note: awslogs-stream-prefix is pinned per group. + - id: idea-module-id + verdict: fix + note: IDEA_MODULE_ID is the cluster-settings module id so metric namespaces do not rename. + - id: put-metric-data-iam + verdict: request + note: cluster-manager and scheduler policies still condition PutMetricData on IDEA/*. Keep the CloudWatch agent managed policy on those roles until the condition matches {cluster}/*. + - id: module-asg-alarms + verdict: request + note: alarms on deleted ASG, instance, or instance-target-group dimensions must be retargeted or disabled before cutover. + - id: broker-put-metric-data-without-imds + verdict: request + note: confirm broker PutMetricData with instance metadata blocked. cloud-watch-region is already set. InstanceId dimensions are dropped. + - id: production-console-handler + verdict: request + note: add the console handler when IDEA_CONTAINER_ROLE is set so stdout carries application lines without a sidecar. diff --git a/source/idea/ideactl/scripts/build-lambda-zips.sh b/source/idea/ideactl/scripts/build-lambda-zips.sh new file mode 100755 index 00000000..2bac6036 --- /dev/null +++ b/source/idea/ideactl/scripts/build-lambda-zips.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# +# Builds the Lambda code assets once, at image build time, so synth is a path lookup. +# +# Builds the matching Lambda asset layout and installs runtime dependencies. +# +# Usage: build-lambda-zips.sh [] [] +set -euo pipefail + +PKG_ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) +SRC_DIR="${1:-${PKG_ROOT}/dist/resources/lambda_functions}" +OUT_DIR="${2:-${PKG_ROOT}/dist/resources/lambda_assets}" +PYTHON="${PYTHON:-python3.13}" +COMMONS="idea_lambda_commons" + +[[ -d "${SRC_DIR}/${COMMONS}" ]] || { echo "no ${COMMONS} under ${SRC_DIR}: run npm run build first" >&2; exit 1; } + +rm -rf "${OUT_DIR}" +mkdir -p "${OUT_DIR}" + +for dir in "${SRC_DIR}"/*/; do + pkg=$(basename "${dir}") + [[ "${pkg}" == "${COMMONS}" ]] && continue + + build="${OUT_DIR}/${pkg}" + mkdir -p "${build}" + cp -r "${SRC_DIR}/${COMMONS}" "${build}/${COMMONS}" + cp -r "${dir%/}" "${build}/${pkg}" + + if [[ -f "${build}/${pkg}/requirements.txt" ]]; then + mv "${build}/${pkg}/requirements.txt" "${build}/requirements.txt" + (cd "${build}" && "${PYTHON}" -m pip install -r requirements.txt \ + --platform manylinux2014_x86_64 --only-binary=:all: --target . --upgrade) + fi + echo "lambda asset: ${build}" +done diff --git a/source/idea/ideactl/scripts/build-sea.mjs b/source/idea/ideactl/scripts/build-sea.mjs new file mode 100644 index 00000000..38c76d40 --- /dev/null +++ b/source/idea/ideactl/scripts/build-sea.mjs @@ -0,0 +1,783 @@ +#!/usr/bin/env node +/** + * Builds the two same-architecture release files: macOS for this machine, + * and Linux for a small virtual machine of the same processor. + * + * `npm run build:dist` produces both. Each file is the official Node runtime + * for that operating system, with the application, deployment CLI, and + * resource tree compressed inside. First use extracts the support tree into a + * per-user temporary directory. + */ + +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + chmodSync, + copyFileSync, + cpSync, + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, join, relative, resolve, sep } from "node:path"; +import { fileURLToPath } from "node:url"; +import { gzipSync } from "node:zlib"; + +const PACKAGE_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const PACKAGE_JSON = JSON.parse(readFileSync(join(PACKAGE_ROOT, "package.json"), "utf8")); +const VERSION = requireNonemptyString(PACKAGE_JSON.version, "package version"); +const DEFAULT_OUTPUT_DIRECTORY = join(PACKAGE_ROOT, "dist", "release"); +const INTERNAL_CDK_ARGUMENT = "__ideactl_internal_cdk__"; +const TAR_BLOCK_SIZE = 512; + +/** + * Validates an untrusted JSON field before it enters a path or artifact name. + * + * @param {unknown} value parsed JSON value + * @param {string} name field name used in an error + * @returns {string} + */ +function requireNonemptyString(value, name) { + if (typeof value !== "string" || value.trim() === "") { + throw new Error(`${name} must be a non-empty string`); + } + return value; +} + +/** + * Converts runtime platform names to release artifact conventions. + * + * @param {NodeJS.Platform} platform runtime platform + * @param {string} architecture runtime architecture + * @returns {string} + */ +function releaseTarget(platform, architecture) { + if (platform !== "darwin" && platform !== "linux") { + throw new Error(`unsupported release operating system: ${platform}`); + } + const releaseArchitecture = + architecture === "x64" ? "amd64" : architecture === "arm64" ? "arm64" : undefined; + if (releaseArchitecture === undefined) { + throw new Error(`unsupported release processor architecture: ${architecture}`); + } + return `${platform}-${releaseArchitecture}`; +} + +/** + * Returns the two release targets for the builder's processor architecture. + * + * @returns {[string, string]} + */ +function releaseTargets() { + return [ + releaseTarget("darwin", process.arch), + releaseTarget("linux", process.arch), + ]; +} + +/** + * Parses build controls and rejects a target that does not match the builder. + * + * @param {string[]} argv command-line arguments + * @returns {{ + * outputDirectory: string; + * target: string; + * executable: string | undefined; + * builder: string | undefined; + * shellArtifact: string | undefined; + * }} + */ +function parseArguments(argv) { + let outputDirectory = DEFAULT_OUTPUT_DIRECTORY; + let target = releaseTarget(process.platform, process.arch); + let executable; + let builder; + let shellArtifact; + + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + if ( + argument !== "--output-directory" && + argument !== "--target" && + argument !== "--executable" && + argument !== "--builder" && + argument !== "--shell-artifact" + ) { + throw new Error(`unknown argument: ${argument}`); + } + const value = argv[index + 1]; + if (value === undefined || value === "") { + throw new Error(`${argument} requires a value`); + } + if (argument === "--output-directory") outputDirectory = resolve(value); + if (argument === "--target") target = value; + if (argument === "--executable") executable = resolve(value); + if (argument === "--builder") builder = resolve(value); + if (argument === "--shell-artifact") shellArtifact = resolve(value); + index += 1; + } + + if (!releaseTargets().includes(target)) { + throw new Error( + `unsupported release target for ${process.arch}: ${target}; expected ${releaseTargets().join(" or ")}`, + ); + } + const hostTarget = releaseTarget(process.platform, process.arch); + if (target.startsWith("darwin-") && process.platform !== "darwin") { + throw new Error(`target ${target} must be built and signed on macOS`); + } + if ( + executable === undefined && + target !== hostTarget && + !(process.platform === "darwin" && target.startsWith("linux-")) + ) { + throw new Error( + `target ${target} requires --executable for that target, current runtime is ${hostTarget}`, + ); + } + if (executable !== undefined && !existsSync(executable)) { + throw new Error(`target runtime executable not found: ${executable}`); + } + if (builder !== undefined && !existsSync(builder)) { + throw new Error(`builder runtime executable not found: ${builder}`); + } + if (shellArtifact !== undefined && !existsSync(shellArtifact)) { + throw new Error(`bundled application artifact not found: ${shellArtifact}`); + } + return { outputDirectory, target, executable, builder, shellArtifact }; +} + +/** + * Runs one required build command and includes its output in failures. + * + * @param {string} command executable name or path + * @param {string[]} args command arguments + * @param {string} cwd working directory + */ +function run(command, args, cwd = PACKAGE_ROOT) { + const result = spawnSync(command, args, { + cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + if (result.stdout !== "") process.stdout.write(result.stdout); + if (result.stderr !== "") process.stderr.write(result.stderr); + if (result.error !== undefined) throw result.error; + if (result.status !== 0) { + throw new Error(`${command} exited with status ${String(result.status)}`); + } +} + +/** + * Returns whether a runtime was compiled with direct SEA generation enabled. + * + * @param {string} executable runtime executable + * @param {string} probeConfig deliberately absent configuration path + * @returns {boolean} + */ +function canBuildSea(executable, probeConfig) { + const result = spawnSync(executable, ["--build-sea", probeConfig], { + cwd: PACKAGE_ROOT, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + if (result.error !== undefined) throw result.error; + return !`${result.stdout}${result.stderr}`.includes("Single executable application is disabled"); +} + +/** + * Downloads and verifies the official runtime when the installed runtime has + * SEA generation disabled. The cache stays under node_modules and is not a + * release input until its published checksum has been verified. + * + * @param {string} temporaryRoot temporary build root + * @returns {string} + */ +function officialRuntime(target, temporaryRoot) { + const separator = target.lastIndexOf("-"); + const platform = target.slice(0, separator); + const releaseArchitecture = target.slice(separator + 1) === "amd64" ? "x64" : "arm64"; + const directoryName = `node-${process.version}-${platform}-${releaseArchitecture}`; + const archiveName = `${directoryName}.tar.gz`; + const downloadRoot = `https://nodejs.org/dist/${process.version}`; + const cacheRoot = join(PACKAGE_ROOT, "node_modules", ".cache", "ideactl-sea", directoryName); + const cachedExecutable = join(cacheRoot, "bin", "node"); + if (existsSync(cachedExecutable)) return cachedExecutable; + + const downloadDirectory = join(temporaryRoot, "node-download"); + const archive = join(downloadDirectory, archiveName); + const checksums = join(downloadDirectory, "SHASUMS256.txt"); + const extraction = join(downloadDirectory, "extract"); + mkdirSync(downloadDirectory, { recursive: true }); + mkdirSync(extraction, { recursive: true }); + run("curl", ["--fail", "--location", "--silent", "--show-error", "-o", checksums, `${downloadRoot}/SHASUMS256.txt`]); + run("curl", ["--fail", "--location", "--silent", "--show-error", "-o", archive, `${downloadRoot}/${archiveName}`]); + + const checksumLine = readFileSync(checksums, "utf8") + .split(/\r?\n/) + .find((line) => line.endsWith(` ${archiveName}`)); + if (checksumLine === undefined) { + throw new Error(`published checksum not found for ${archiveName}`); + } + const expected = checksumLine.split(/\s+/)[0]; + const actual = createHash("sha256").update(readFileSync(archive)).digest("hex"); + if (actual !== expected) { + throw new Error(`runtime checksum mismatch for ${archiveName}: expected ${expected}, got ${actual}`); + } + + run("tar", ["-xzf", archive, "-C", extraction]); + const extractedExecutable = join(extraction, directoryName, "bin", "node"); + if (!existsSync(extractedExecutable)) { + throw new Error(`runtime archive did not contain ${directoryName}/bin/node`); + } + mkdirSync(dirname(cacheRoot), { recursive: true }); + rmSync(cacheRoot, { recursive: true, force: true }); + cpSync(join(extraction, directoryName), cacheRoot, { recursive: true }); + return cachedExecutable; +} + +/** + * Downloads the official runtime for the current host when SEA is disabled. + * + * @param {string} temporaryRoot temporary build root + * @returns {string} + */ +function officialSeaBuilder(temporaryRoot) { + return officialRuntime(releaseTarget(process.platform, process.arch), temporaryRoot); +} + +/** + * Selects a SEA-enabled builder and fails if an explicit builder is disabled. + * + * @param {string | undefined} requested explicit builder + * @param {string} temporaryRoot temporary build root + * @returns {string} + */ +function seaBuilder(requested, temporaryRoot) { + const probe = join(temporaryRoot, "missing-sea-config.json"); + const candidate = requested ?? process.execPath; + if (canBuildSea(candidate, probe)) return candidate; + if (requested !== undefined) { + throw new Error(`builder runtime has single-executable generation disabled: ${requested}`); + } + const official = officialSeaBuilder(temporaryRoot); + if (!canBuildSea(official, probe)) { + throw new Error(`official runtime has single-executable generation disabled: ${official}`); + } + return official; +} + +/** + * Copies one required path and reports the missing input directly. + * + * @param {string} source required source path + * @param {string} destination destination path + */ +function copyRequired(source, destination) { + if (!existsSync(source)) throw new Error(`required distribution input not found: ${source}`); + mkdirSync(dirname(destination), { recursive: true }); + if (statSync(source).isDirectory()) { + cpSync(source, destination, { recursive: true }); + } else { + copyFileSync(source, destination); + } +} + +/** + * Stages only files the executable needs after extraction. + * + * @param {string} shellArtifact direct-runtime distribution root + * @param {string} runtimeRoot temporary support-tree root + */ +function stageRuntime(shellArtifact, runtimeRoot) { + const shellDist = join(shellArtifact, "dist"); + copyRequired( + join(shellDist, "src", "cli"), + join(runtimeRoot, "dist", "src", "cli"), + ); + copyRequired( + join(shellDist, "src", "IDEA_VERSION.txt"), + join(runtimeRoot, "dist", "src", "IDEA_VERSION.txt"), + ); + copyRequired( + join(shellDist, "resources"), + join(runtimeRoot, "dist", "resources"), + ); + copyRequired( + join(shellDist, "node_modules", "aws-cdk"), + join(runtimeRoot, "dist", "node_modules", "aws-cdk"), + ); + copyRequired(join(shellDist, "cdk.json"), join(runtimeRoot, "dist", "cdk.json")); + + writeFileSync( + join(runtimeRoot, "launcher.cjs"), + `"use strict"; +module.exports = import("./dist/src/cli/main.js").then(({ run }) => run()); +`, + ); +} + +/** + * Lists a tree in stable archive order and rejects unsupported file types. + * + * @param {string} root tree root + * @returns {Array<{ archiveName: string; path: string; isDirectory: boolean }>} + */ +function tarEntries(root) { + const entries = []; + const visit = (directory) => { + const names = readdirSync(directory).sort(); + for (const name of names) { + const path = join(directory, name); + const stats = lstatSync(path); + const archiveName = relative(root, path).split(sep).join("/"); + if (stats.isSymbolicLink()) { + throw new Error(`symbolic links are not supported in the release artifact: ${archiveName}`); + } + if (stats.isDirectory()) { + entries.push({ archiveName, path, isDirectory: true }); + visit(path); + } else if (stats.isFile()) { + entries.push({ archiveName, path, isDirectory: false }); + } else { + throw new Error(`unsupported file type in release artifact: ${archiveName}`); + } + } + }; + visit(root); + return entries; +} + +/** + * Writes an ASCII field into a tar header. + * + * @param {Buffer} target destination header + * @param {string} value field value + * @param {number} offset byte offset + * @param {number} length field length + */ +function writeString(target, value, offset, length) { + const encoded = Buffer.from(value, "utf8"); + if (encoded.length > length) throw new Error(`tar field exceeds ${String(length)} bytes: ${value}`); + encoded.copy(target, offset); +} + +/** + * Writes a fixed-width octal tar field. + * + * @param {Buffer} target destination header + * @param {number} value non-negative integer + * @param {number} offset byte offset + * @param {number} length field length + */ +function writeOctal(target, value, offset, length) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error(`invalid tar numeric field: ${String(value)}`); + } + const encoded = value.toString(8).padStart(length - 1, "0"); + if (encoded.length >= length) throw new Error(`tar numeric field is too large: ${String(value)}`); + writeString(target, `${encoded}\0`, offset, length); +} + +/** + * Splits a path into the POSIX ustar name and prefix fields. + * + * @param {string} archiveName relative archive path + * @returns {{ name: string; prefix: string }} + */ +function splitTarPath(archiveName) { + if (Buffer.byteLength(archiveName, "utf8") <= 100) { + return { name: archiveName, prefix: "" }; + } + const separators = [...archiveName.matchAll(/\//g)].map((match) => match.index ?? -1).reverse(); + for (const separator of separators) { + const prefix = archiveName.slice(0, separator); + const name = archiveName.slice(separator + 1); + if (Buffer.byteLength(prefix, "utf8") <= 155 && Buffer.byteLength(name, "utf8") <= 100) { + return { name, prefix }; + } + } + throw new Error(`archive path exceeds the ustar path limit: ${archiveName}`); +} + +/** + * Builds one deterministic POSIX ustar header. + * + * @param {{ archiveName: string; path: string; isDirectory: boolean }} entry archive entry + * @returns {Buffer} + */ +function tarHeader(entry) { + const stats = lstatSync(entry.path); + const archiveName = entry.isDirectory ? `${entry.archiveName}/` : entry.archiveName; + const { name, prefix } = splitTarPath(archiveName); + const header = Buffer.alloc(TAR_BLOCK_SIZE); + writeString(header, name, 0, 100); + writeOctal(header, stats.mode & 0o777, 100, 8); + writeOctal(header, 0, 108, 8); + writeOctal(header, 0, 116, 8); + writeOctal(header, entry.isDirectory ? 0 : stats.size, 124, 12); + writeOctal(header, 0, 136, 12); + header.fill(0x20, 148, 156); + writeString(header, entry.isDirectory ? "5" : "0", 156, 1); + writeString(header, "ustar\0", 257, 6); + writeString(header, "00", 263, 2); + writeString(header, prefix, 345, 155); + const checksum = header.reduce((total, byte) => total + byte, 0); + writeString(header, `${checksum.toString(8).padStart(6, "0")}\0 `, 148, 8); + return header; +} + +/** + * Creates a deterministic gzip-compressed ustar archive in memory. + * + * @param {string} root source tree + * @returns {Buffer} + */ +function createTarGz(root) { + const blocks = []; + for (const entry of tarEntries(root)) { + blocks.push(tarHeader(entry)); + if (entry.isDirectory) continue; + const contents = readFileSync(entry.path); + blocks.push(contents); + const remainder = contents.length % TAR_BLOCK_SIZE; + if (remainder !== 0) blocks.push(Buffer.alloc(TAR_BLOCK_SIZE - remainder)); + } + blocks.push(Buffer.alloc(TAR_BLOCK_SIZE * 2)); + return gzipSync(Buffer.concat(blocks), { level: 9 }); +} + +/** + * Measures regular-file bytes in a tree. + * + * @param {string} root tree root + * @returns {number} + */ +function treeBytes(root) { + let bytes = 0; + const visit = (path) => { + const stats = lstatSync(path); + if (stats.isDirectory()) { + for (const name of readdirSync(path)) visit(join(path, name)); + } else if (stats.isFile()) { + bytes += stats.size; + } + }; + visit(root); + return bytes; +} + +/** + * Produces the injected CommonJS launcher. + * + * @param {string} runtimeHash support archive digest + * @returns {string} + */ +function seaLauncher(runtimeHash) { + return `"use strict"; +const { createRequire } = require("node:module"); +const { + chmodSync, + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} = require("node:fs"); +const { getAsset } = require("node:sea"); +const { tmpdir } = require("node:os"); +const { dirname, join, resolve, sep } = require("node:path"); +const { gunzipSync } = require("node:zlib"); + +const VERSION = ${JSON.stringify(VERSION)}; +const RUNTIME_HASH = ${JSON.stringify(runtimeHash)}; +const INTERNAL_CDK_ARGUMENT = ${JSON.stringify(INTERNAL_CDK_ARGUMENT)}; +const BLOCK_SIZE = 512; + +function field(buffer, offset, length) { + const end = buffer.indexOf(0, offset); + const limit = end >= offset && end < offset + length ? end : offset + length; + return buffer.toString("utf8", offset, limit); +} + +function octal(buffer, offset, length) { + const value = field(buffer, offset, length).trim(); + if (!/^[0-7]*$/.test(value)) throw new Error("invalid embedded archive numeric field"); + return value === "" ? 0 : Number.parseInt(value, 8); +} + +function extractRuntime(destination) { + const archive = gunzipSync(Buffer.from(getAsset("runtime.tar.gz"))); + let offset = 0; + while (offset + BLOCK_SIZE <= archive.length) { + const header = archive.subarray(offset, offset + BLOCK_SIZE); + offset += BLOCK_SIZE; + if (header.every((byte) => byte === 0)) break; + const name = field(header, 0, 100); + const prefix = field(header, 345, 155); + const archiveName = prefix === "" ? name : \`\${prefix}/\${name}\`; + const size = octal(header, 124, 12); + const mode = octal(header, 100, 8) & 0o777; + const type = field(header, 156, 1) || "0"; + const target = resolve(destination, archiveName); + const destinationPrefix = \`\${resolve(destination)}\${sep}\`; + if (!target.startsWith(destinationPrefix)) { + throw new Error(\`invalid embedded archive path: \${archiveName}\`); + } + if (type === "5") { + mkdirSync(target, { recursive: true, mode }); + chmodSync(target, mode); + } else if (type === "0") { + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, archive.subarray(offset, offset + size), { mode }); + chmodSync(target, mode); + } else { + throw new Error(\`unsupported embedded archive entry type: \${type}\`); + } + offset += Math.ceil(size / BLOCK_SIZE) * BLOCK_SIZE; + } +} + +function shellQuote(value) { + return \`'\${value.replaceAll("'", "'\\\\''")}'\`; +} + +function ensureRuntime() { + const uid = typeof process.getuid === "function" ? String(process.getuid()) : "user"; + const parent = join(tmpdir(), \`ideactl-\${uid}\`); + const destination = join(parent, \`\${VERSION}-\${RUNTIME_HASH.slice(0, 16)}\`); + const marker = join(destination, ".complete"); + mkdirSync(parent, { recursive: true, mode: 0o700 }); + chmodSync(parent, 0o700); + if (!existsSync(marker) || readFileSync(marker, "utf8") !== RUNTIME_HASH) { + if (existsSync(destination)) { + if (lstatSync(destination).isSymbolicLink()) { + throw new Error(\`refusing symbolic-link runtime directory: \${destination}\`); + } + rmSync(destination, { recursive: true, force: true }); + } + const staging = mkdtempSync(join(parent, ".extract-")); + try { + extractRuntime(staging); + writeFileSync(join(staging, ".complete"), RUNTIME_HASH, { mode: 0o600 }); + try { + renameSync(staging, destination); + } catch (error) { + if (!existsSync(marker) || readFileSync(marker, "utf8") !== RUNTIME_HASH) throw error; + rmSync(staging, { recursive: true, force: true }); + } + } catch (error) { + rmSync(staging, { recursive: true, force: true }); + throw error; + } + } + + const cdkShim = join(destination, "bin", "cdk"); + mkdirSync(dirname(cdkShim), { recursive: true }); + writeFileSync( + cdkShim, + \`#!/bin/sh\\nexec \${shellQuote(process.execPath)} ${INTERNAL_CDK_ARGUMENT} "$@"\\n\`, + { mode: 0o700 }, + ); + chmodSync(cdkShim, 0o700); + process.env.IDEA_CDK_BIN = cdkShim; + process.env.PATH = \`\${dirname(process.execPath)}:\${process.env.PATH ?? ""}\`; + return destination; +} + +async function main() { + const runtimeRoot = ensureRuntime(); + const requireFromRuntime = createRequire(join(runtimeRoot, "launcher.cjs")); + if (process.argv[2] === INTERNAL_CDK_ARGUMENT) { + process.argv.splice(2, 1); + requireFromRuntime(join(runtimeRoot, "dist", "node_modules", "aws-cdk", "bin", "cdk")); + return; + } + const code = await requireFromRuntime(join(runtimeRoot, "launcher.cjs")); + if (typeof code === "number" && code !== 0) process.exitCode = code; +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); +`; +} + +/** + * Writes the release tarball containing only the executable. + * + * @param {string} executable built executable path + * @param {string} archive output archive path + * @param {string} temporaryRoot temporary build root + */ +function writeReleaseArchive(executable, archive, temporaryRoot) { + const releaseRoot = join(temporaryRoot, "release-package"); + mkdirSync(releaseRoot); + copyFileSync(executable, join(releaseRoot, "ideactl")); + chmodSync(join(releaseRoot, "ideactl"), 0o755); + writeFileSync(archive, createTarGz(releaseRoot)); +} + +/** + * Builds, verifies, measures, and checksums the host release artifact. + */ +function main() { + const { outputDirectory, target, executable, builder, shellArtifact } = parseArguments( + process.argv.slice(2), + ); + const temporaryRoot = mkdtempSync(join(tmpdir(), "ideactl-sea-build-")); + const shellRoot = shellArtifact ?? join(temporaryRoot, "shell"); + const runtimeRoot = join(temporaryRoot, "runtime"); + const runtimeArchive = join(temporaryRoot, "runtime.tar.gz"); + const launcher = join(temporaryRoot, "sea-launcher.cjs"); + const seaConfig = join(temporaryRoot, "sea-config.json"); + const targetDirectory = join(outputDirectory, target); + const builtExecutable = join(targetDirectory, "ideactl"); + const artifactBase = `ideactl-v${VERSION}-${target}`; + const releaseArchive = join(outputDirectory, `${artifactBase}.tar.gz`); + const checksumFile = `${releaseArchive}.sha256`; + const metadataFile = join(outputDirectory, `${artifactBase}.json`); + + try { + const builderExecutable = seaBuilder(builder, temporaryRoot); + let targetExecutable = executable; + const hostTarget = releaseTarget(process.platform, process.arch); + if (targetExecutable === undefined && target !== hostTarget) { + targetExecutable = officialRuntime(target, temporaryRoot); + } + if (shellArtifact === undefined) { + run(process.execPath, [ + join(PACKAGE_ROOT, "scripts", "build-shell-bundle.mjs"), + "--output", + shellRoot, + "--no-archive", + ]); + } + + mkdirSync(runtimeRoot, { recursive: true }); + stageRuntime(shellRoot, runtimeRoot); + const extractedBytes = treeBytes(runtimeRoot); + const runtimeBytes = createTarGz(runtimeRoot); + writeFileSync(runtimeArchive, runtimeBytes); + const runtimeHash = createHash("sha256").update(runtimeBytes).digest("hex"); + writeFileSync(launcher, seaLauncher(runtimeHash)); + + rmSync(targetDirectory, { recursive: true, force: true }); + mkdirSync(targetDirectory, { recursive: true }); + const config = { + main: launcher, + mainFormat: "commonjs", + output: builtExecutable, + disableExperimentalSEAWarning: true, + useSnapshot: false, + useCodeCache: false, + execArgvExtension: "env", + assets: { "runtime.tar.gz": runtimeArchive }, + }; + if (targetExecutable !== undefined) config.executable = targetExecutable; + writeFileSync(seaConfig, `${JSON.stringify(config, null, 2)}\n`); + run(builderExecutable, ["--build-sea", seaConfig]); + if (target.startsWith("darwin-")) { + if (process.platform !== "darwin") { + throw new Error(`target ${target} must be signed on a macOS builder`); + } + run("codesign", ["--sign", "-", "--force", builtExecutable]); + } + chmodSync(builtExecutable, 0o755); + + if (target === releaseTarget(process.platform, process.arch)) { + run(builtExecutable, ["about"]); + } + + mkdirSync(outputDirectory, { recursive: true }); + writeReleaseArchive(builtExecutable, releaseArchive, temporaryRoot); + const releaseBytes = readFileSync(releaseArchive); + const releaseHash = createHash("sha256").update(releaseBytes).digest("hex"); + writeFileSync(checksumFile, `${releaseHash} ${basename(releaseArchive)}\n`); + + const metadata = { + schemaVersion: 1, + packageVersion: VERSION, + target, + buildRuntime: process.version, + seaBuilder: builderExecutable === process.execPath ? "installed runtime" : "verified official runtime", + selfContained: true, + runTimeRequirements: [], + resourceHandling: + "Resources and the pinned deployment CLI are embedded and extracted to a per-user temporary cache.", + sizes: { + executableBytes: statSync(builtExecutable).size, + releaseArchiveBytes: statSync(releaseArchive).size, + embeddedSupportArchiveBytes: runtimeBytes.length, + extractedSupportBytes: extractedBytes, + }, + sha256: releaseHash, + }; + writeFileSync(metadataFile, `${JSON.stringify(metadata, null, 2)}\n`); + + console.log(`target: ${target}`); + console.log(`executable bytes: ${String(metadata.sizes.executableBytes)}`); + console.log(`release archive bytes: ${String(metadata.sizes.releaseArchiveBytes)}`); + console.log(`embedded support bytes: ${String(metadata.sizes.embeddedSupportArchiveBytes)}`); + console.log(`extracted support bytes: ${String(metadata.sizes.extractedSupportBytes)}`); + console.log(`sha256: ${releaseHash}`); + console.log(`archive: ${releaseArchive}`); + } finally { + rmSync(temporaryRoot, { recursive: true, force: true }); + } +} + +/** + * Builds both same-architecture release files with one shared application bundle. + * + * @param {string[]} args command-line arguments without a target + */ +function buildReleasePair(args) { + if (process.platform !== "darwin") { + throw new Error("the two-target build must run on macOS; pass --target for a native build"); + } + if (args.includes("--executable") || args.includes("--builder") || args.includes("--shell-artifact")) { + throw new Error("--executable, --builder, and --shell-artifact require an explicit --target"); + } + + const temporaryRoot = mkdtempSync(join(tmpdir(), "ideactl-release-pair-")); + const shellRoot = join(temporaryRoot, "shell"); + try { + run(process.execPath, [ + join(PACKAGE_ROOT, "scripts", "build-shell-bundle.mjs"), + "--output", + shellRoot, + "--no-archive", + ]); + for (const target of releaseTargets()) { + run(process.execPath, [ + fileURLToPath(import.meta.url), + ...args, + "--target", + target, + "--shell-artifact", + shellRoot, + ]); + } + } finally { + rmSync(temporaryRoot, { recursive: true, force: true }); + } +} + +const buildArguments = process.argv.slice(2); +if (buildArguments.includes("--target")) { + main(); +} else { + buildReleasePair(buildArguments); +} diff --git a/source/idea/ideactl/scripts/build-shell-bundle.mjs b/source/idea/ideactl/scripts/build-shell-bundle.mjs new file mode 100644 index 00000000..da377e9e --- /dev/null +++ b/source/idea/ideactl/scripts/build-shell-bundle.mjs @@ -0,0 +1,490 @@ +#!/usr/bin/env node +/** + * Builds the direct-runtime release artifact. + * + * The artifact contains one bundled application file, the pinned deployment + * CLI, and the resource files read at run time. It does not contain a package + * installation tree and does not require a compiler or package manager. + */ + +import { spawnSync } from "node:child_process"; +import { builtinModules } from "node:module"; +import { + chmodSync, + copyFileSync, + cpSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const PACKAGE_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const PACKAGE_JSON = JSON.parse(readFileSync(join(PACKAGE_ROOT, "package.json"), "utf8")); +const BUNDLER_VERSION = "0.25.9"; +const DEFAULT_OUTPUT = join(PACKAGE_ROOT, "dist", "ideactl-shell"); +const DEFAULT_ARCHIVE = join(PACKAGE_ROOT, "dist", `ideactl-shell-${PACKAGE_JSON.version}.tar.gz`); + +/** + * Parses the two output controls accepted by the release task. + * + * @param {string[]} argv command-line arguments + * @returns {{ output: string, archive: string | undefined }} + */ +function parseArguments(argv) { + let output = DEFAULT_OUTPUT; + let archive = DEFAULT_ARCHIVE; + + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + if (argument === "--no-archive") { + archive = undefined; + continue; + } + if (argument !== "--output" && argument !== "--archive") { + throw new Error(`unknown argument: ${argument}`); + } + const value = argv[index + 1]; + if (value === undefined || value === "") { + throw new Error(`${argument} requires a path`); + } + if (argument === "--output") output = resolve(value); + if (argument === "--archive") archive = resolve(value); + index += 1; + } + + if (archive !== undefined && archive.startsWith(`${output}/`)) { + throw new Error("--archive must be outside --output"); + } + return { output, archive }; +} + +/** + * Runs a required build command and forwards its transcript. + * + * @param {string} command executable name or path + * @param {string[]} args command arguments + * @param {NodeJS.ProcessEnv} [env] optional environment + */ +function run(command, args, env = process.env) { + const result = spawnSync(command, args, { + cwd: PACKAGE_ROOT, + env, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + if (result.stdout !== "") process.stdout.write(result.stdout); + if (result.stderr !== "") process.stderr.write(result.stderr); + if (result.error !== undefined) throw result.error; + if (result.status !== 0) { + throw new Error(`${command} exited with status ${String(result.status)}`); + } +} + +/** + * Returns the first existing directory or file from a candidate list. + * + * @param {string[]} candidates ordered paths + * @param {string} description value named in an error + * @returns {string} + */ +function firstExisting(candidates, description) { + const found = candidates.find((candidate) => existsSync(candidate)); + if (found === undefined) { + throw new Error(`${description} not found: ${candidates.join(", ")}`); + } + return found; +} + +/** + * Replaces the run-time stack lookup with static imports in a temporary copy. + * Static imports let the bundler include every stack in the single application + * file without changing the source module owned by the stack port. + * + * @param {string} sourceRoot copied source tree + */ +function makeStackImportsStatic(sourceRoot) { + const appFile = join(sourceRoot, "cdk", "app.ts"); + let source = readFileSync(appFile, "utf8"); + const localImport = + "import { liveSynthReads, replaySynthReads, type SynthReads } from './synth-reads.ts';"; + const staticImports = [ + "import { buildStack as buildAnalyticsStack } from './stacks/analytics.ts';", + "import { buildStack as buildBastionHostStack } from './stacks/bastion-host.ts';", + "import { buildStack as buildBootstrapStack } from './stacks/bootstrap.ts';", + "import { buildStack as buildClusterStack } from './stacks/cluster.ts';", + "import { buildStack as buildClusterManagerStack } from './stacks/cluster-manager.ts';", + "import { buildStack as buildDirectoryServiceStack } from './stacks/directoryservice.ts';", + "import { buildStack as buildEcsStack } from './stacks/ecs.ts';", + "import { buildStack as buildIdentityProviderStack } from './stacks/identity-provider.ts';", + "import { buildStack as buildMetricsStack } from './stacks/metrics.ts';", + "import { buildStack as buildSchedulerStack } from './stacks/scheduler.ts';", + "import { buildStack as buildSharedStorageStack } from './stacks/shared-storage.ts';", + "import { buildStack as buildVirtualDesktopControllerStack } from './stacks/vdc.ts';", + ].join("\n"); + if (!source.includes(localImport)) { + throw new Error("cannot locate the stack registry import point"); + } + source = source.replace(localImport, `${localImport}\n${staticImports}`); + + const registryStart = source.indexOf("/**\n * Module name -> the file under `stacks/`"); + const registryEnd = source.indexOf("/** `Utils.get_as_bool", registryStart); + if (registryStart < 0 || registryEnd < 0) { + throw new Error("cannot locate the dynamic stack registry"); + } + const staticRegistry = `/** + * Every stack is a static bundle input. The returned promise preserves the + * StackRegistry contract used by buildApp. + */ +export const DEFAULT_STACK_REGISTRY: StackRegistry = { + "analytics": async () => buildAnalyticsStack, + "bastion-host": async () => buildBastionHostStack, + "bootstrap": async () => buildBootstrapStack, + "cluster": async () => buildClusterStack, + "cluster-manager": async () => buildClusterManagerStack, + "directoryservice": async () => buildDirectoryServiceStack, + "ecs": async () => buildEcsStack, + "identity-provider": async () => buildIdentityProviderStack, + "metrics": async () => buildMetricsStack, + "scheduler": async () => buildSchedulerStack, + "shared-storage": async () => buildSharedStorageStack, + "virtual-desktop-controller": async () => buildVirtualDesktopControllerStack, +}; + +`; + source = `${source.slice(0, registryStart)}${staticRegistry}${source.slice(registryEnd)}`; + writeFileSync(appFile, source); +} + +/** + * Recursively measures regular files and records native add-ons. + * + * @param {string} root tree to measure + * @returns {{ bytes: number, files: number, nativeModules: string[] }} + */ +function measureTree(root) { + const result = { bytes: 0, files: 0, nativeModules: [] }; + const visit = (path) => { + const stats = statSync(path); + if (stats.isDirectory()) { + for (const name of readdirSync(path)) visit(join(path, name)); + return; + } + if (!stats.isFile()) return; + result.bytes += stats.size; + result.files += 1; + if (path.endsWith(".node")) result.nativeModules.push(path.slice(root.length + 1)); + }; + visit(root); + return result; +} + +/** + * Finds package names represented in the bundle metadata. + * + * @param {Record} metadata parsed bundle metadata + * @returns {string[]} + */ +function bundledPackages(metadata) { + const inputs = + metadata.inputs !== null && typeof metadata.inputs === "object" + ? Object.keys(metadata.inputs) + : []; + const packages = new Set(); + for (const input of inputs) { + const marker = "node_modules/"; + const markerIndex = input.lastIndexOf(marker); + if (markerIndex < 0) continue; + const parts = input.slice(markerIndex + marker.length).split("/"); + const packageName = parts[0]?.startsWith("@") + ? `${parts[0]}/${parts[1] ?? ""}` + : parts[0]; + if (packageName !== undefined && !packageName.endsWith("/")) { + packages.add(packageName); + } + } + return [...packages].sort(); +} + +/** + * Lists external imports that the runtime does not provide. + * + * @param {Record} metadata parsed bundle metadata + * @returns {string[]} + */ +function externalRuntimeImports(metadata) { + const builtins = new Set( + builtinModules.flatMap((name) => [ + name, + name.startsWith("node:") ? name.slice(5) : `node:${name}`, + ]), + ); + const outputs = + metadata.outputs !== null && typeof metadata.outputs === "object" + ? Object.values(metadata.outputs) + : []; + const imports = []; + for (const output of outputs) { + if (output === null || typeof output !== "object") continue; + const entries = Array.isArray(output.imports) ? output.imports : []; + for (const entry of entries) { + if (entry === null || typeof entry !== "object") continue; + if (entry.external !== true || typeof entry.path !== "string") continue; + if (!builtins.has(entry.path)) imports.push(entry.path); + } + } + return [...new Set(imports)].sort(); +} + +/** + * Copies the package resource groups that production command paths read. + * + * @param {string} destination output resources directory + * @param {string} lambdaAssets prebuilt Lambda asset directory + * @returns {Record} exact source bytes by resource group + */ +function copyRuntimeResources(destination, lambdaAssets) { + const resourceRoot = firstExisting( + [ + join(PACKAGE_ROOT, "resources"), + join(PACKAGE_ROOT, "..", "idea-administrator", "resources"), + ], + "administrator resources", + ); + const groups = ["cdk", "config", "input_params", "integration_tests", "lambda_functions", "policies"]; + const measurements = {}; + mkdirSync(destination, { recursive: true }); + for (const group of groups) { + const source = firstExisting([join(resourceRoot, group)], `resource group ${group}`); + cpSync(source, join(destination, group), { recursive: true }); + measurements[group] = measureTree(source).bytes; + } + + // The container module's config templates live in this package until resource ownership moves + // here, so they are overlaid the way the dist build overlays them. Without this the released + // artifact cannot render the configuration of a new cluster at all, because every new cluster + // runs its control plane as container tasks and `ecs/settings.yml` is not in the copied tree. + const containerTemplates = firstExisting( + [join(PACKAGE_ROOT, "resources-ecs", "config")], + "container module config templates", + ); + cpSync(containerTemplates, join(destination, "config"), { recursive: true }); + measurements.config += measureTree(containerTemplates).bytes; + + const bootstrap = firstExisting( + [ + join(resourceRoot, "bootstrap"), + join(PACKAGE_ROOT, "..", "idea-bootstrap"), + ], + "bootstrap resources", + ); + cpSync(bootstrap, join(destination, "bootstrap"), { recursive: true }); + measurements.bootstrap = measureTree(bootstrap).bytes; + + cpSync(lambdaAssets, join(destination, "lambda_assets"), { recursive: true }); + measurements.lambda_assets = measureTree(lambdaAssets).bytes; + return measurements; +} + +/** + * Writes the executable entry file. It imports the application bundle and + * mirrors the bundle's error-to-exit-code handling using only the runtime. + * + * @param {string} output artifact root + */ +function writeLauncher(output) { + const launcher = join(output, "bin", "ideactl"); + mkdirSync(dirname(launcher), { recursive: true }); + writeFileSync( + launcher, + `#!/usr/bin/env node +const { run } = await import("../dist/src/cli/main.js"); +run().then( + (code) => { + if (code !== 0) process.exitCode = code; + }, + (error) => { + console.error(error); + process.exitCode = 1; + }, +); +`, + ); + chmodSync(launcher, 0o755); +} + +/** + * Builds, validates, measures, and optionally archives the direct-runtime + * distribution. + */ +function main() { + const { output, archive } = parseArguments(process.argv.slice(2)); + const temporary = mkdtempSync(join(tmpdir(), "ideactl-shell-build-")); + const copiedSource = join(temporary, "src"); + const optionalWatcherShim = join(temporary, "optional-watcher.cjs"); + const lambdaAssets = join(temporary, "lambda_assets"); + const bundleFile = join(output, "dist", "src", "cli", "main.js"); + const metadataFile = join(output, "bundle-metadata.json"); + + try { + rmSync(output, { recursive: true, force: true }); + mkdirSync(dirname(bundleFile), { recursive: true }); + cpSync(join(PACKAGE_ROOT, "src"), copiedSource, { recursive: true }); + makeStackImportsStatic(copiedSource); + writeFileSync( + optionalWatcherShim, + `"use strict"; +module.exports = { + watch() { + throw new Error("template watching is not available in the release artifact"); + }, +}; +`, + ); + + run( + "npm", + [ + "exec", + "--yes", + `--package=esbuild@${BUNDLER_VERSION}`, + "--", + "esbuild", + join(copiedSource, "cli", "main.ts"), + "--bundle", + "--platform=node", + "--format=esm", + "--target=node22", + "--legal-comments=external", + `--alias:chokidar=${optionalWatcherShim}`, + `--banner:js=import { createRequire as __ideactlCreateRequire } from "node:module"; const require = __ideactlCreateRequire(import.meta.url);`, + `--metafile=${metadataFile}`, + `--outfile=${bundleFile}`, + ], + { ...process.env, NODE_PATH: join(PACKAGE_ROOT, "node_modules") }, + ); + chmodSync(bundleFile, 0o755); + + const resourceRoot = firstExisting( + [ + join(PACKAGE_ROOT, "resources"), + join(PACKAGE_ROOT, "..", "idea-administrator", "resources"), + ], + "administrator resources", + ); + run(join(PACKAGE_ROOT, "scripts", "build-lambda-zips.sh"), [ + join(resourceRoot, "lambda_functions"), + lambdaAssets, + ]); + const resources = copyRuntimeResources( + join(output, "dist", "resources"), + lambdaAssets, + ); + mkdirSync(join(output, "dist", "node_modules"), { recursive: true }); + cpSync( + firstExisting( + [ + join(PACKAGE_ROOT, "node_modules", "aws-cdk"), + ], + "pinned deployment CLI", + ), + join(output, "dist", "node_modules", "aws-cdk"), + { recursive: true }, + ); + copyFileSync(join(PACKAGE_ROOT, "cdk.json"), join(output, "dist", "cdk.json")); + mkdirSync(join(output, "dist", "src"), { recursive: true }); + copyFileSync( + firstExisting( + [ + join(PACKAGE_ROOT, "IDEA_VERSION.txt"), + join(PACKAGE_ROOT, "..", "..", "..", "IDEA_VERSION.txt"), + ], + "IDEA_VERSION.txt", + ), + join(output, "dist", "src", "IDEA_VERSION.txt"), + ); + writeLauncher(output); + + const metadata = JSON.parse(readFileSync(metadataFile, "utf8")); + const externalImports = externalRuntimeImports(metadata); + if (externalImports.length > 0) { + throw new Error(`external runtime imports found: ${externalImports.join(", ")}`); + } + const packages = bundledPackages(metadata); + rmSync(metadataFile); + const bundle = measureTree(bundleFile); + const deploymentCli = measureTree(join(output, "dist", "node_modules", "aws-cdk")); + const resourceTree = measureTree(join(output, "dist", "resources")); + const artifactBeforeManifest = measureTree(output); + const nativeModules = [ + ...bundle.nativeModules, + ...deploymentCli.nativeModules, + ...resourceTree.nativeModules, + ]; + if (nativeModules.length > 0) { + throw new Error(`native modules found in artifact: ${nativeModules.join(", ")}`); + } + + const manifest = { + schemaVersion: 1, + packageVersion: PACKAGE_JSON.version, + runtime: { + node: PACKAGE_JSON.engines.node, + packageManagerRequired: false, + compilerRequired: false, + nativeModules, + externalImports, + }, + contentBytes: { + applicationBundle: bundle.bytes, + deploymentCli: deploymentCli.bytes, + resources: resourceTree.bytes, + artifactBeforeManifest: artifactBeforeManifest.bytes, + }, + resourceSourceBytes: resources, + bundledPackages: packages, + resourceReads: [ + "resources/bootstrap", + "resources/cdk", + "resources/config", + "resources/input_params", + "resources/integration_tests", + "resources/lambda_assets when prebuilt, otherwise resources/lambda_functions", + "resources/policies", + ], + }; + writeFileSync(join(output, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`); + + run(process.execPath, [join(output, "bin", "ideactl"), "about"]); + run(process.execPath, [ + join(output, "dist", "node_modules", "aws-cdk", "bin", "cdk"), + "--version", + ]); + + const finalArtifact = measureTree(output); + console.log(`application bundle bytes: ${String(bundle.bytes)}`); + console.log(`deployment CLI bytes: ${String(deploymentCli.bytes)}`); + console.log(`resource bytes: ${String(resourceTree.bytes)}`); + console.log(`artifact bytes: ${String(finalArtifact.bytes)}`); + console.log(`native modules: ${nativeModules.length === 0 ? "none" : nativeModules.join(", ")}`); + + if (archive !== undefined) { + mkdirSync(dirname(archive), { recursive: true }); + rmSync(archive, { force: true }); + run("tar", ["-czf", archive, "-C", dirname(output), basename(output)]); + console.log(`archive bytes: ${String(statSync(archive).size)}`); + console.log(`archive: ${archive}`); + } + } finally { + rmSync(temporary, { recursive: true, force: true }); + } +} + +main(); diff --git a/source/idea/ideactl/scripts/ci-gates.mjs b/source/idea/ideactl/scripts/ci-gates.mjs new file mode 100644 index 00000000..4fd8ccef --- /dev/null +++ b/source/idea/ideactl/scripts/ci-gates.mjs @@ -0,0 +1,725 @@ +#!/usr/bin/env node +/** + * Runs the package gates that need no cloud credentials. + */ + +import { spawnSync } from "node:child_process"; +import { + existsSync, + readFileSync, + readdirSync, + statSync, +} from "node:fs"; +import { createRequire } from "node:module"; +import { dirname, extname, join, relative, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const SCRIPT_DIRECTORY = dirname(fileURLToPath(import.meta.url)); +const DEFAULT_PACKAGE_ROOT = resolve(SCRIPT_DIRECTORY, ".."); +const DEFAULT_WORKFLOW_ROOT = resolve(DEFAULT_PACKAGE_ROOT, "../../..", ".github/workflows"); +const require = createRequire(import.meta.url); +const EXACT_VERSION = + /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/u; +const TEST_FILE = /\.test\.ts$/u; +const SKIP_DECLARATION = + /\bskip\s*:|(?:\btest|\bit|\bdescribe|\bsuite)\.skip\s*\(/u; +const TEXT_EXTENSIONS = new Set([ + "", + ".cjs", + ".css", + ".html", + ".ini", + ".js", + ".jinja2", + ".json", + ".md", + ".mjs", + ".py", + ".sh", + ".sql", + ".ts", + ".txt", + ".yaml", + ".yml", +]); +const EXCLUDED_DIRECTORIES = new Set([ + ".git", + "cdk.out", + "dist", + "node_modules", +]); +const EXCLUDED_RELATIVE_DIRECTORIES = [ + "docs/port", + "tools/e2e/reference", + "tools/parity/fixtures", + "tools/parity/live", +]; +const ALLOWED_TICKET_LIKE_PREFIXES = new Set([ + "BSD", + "FS", + "GPL", + "SHA", + "UTF", +]); +/** + * Escapes a literal for use inside a regular expression. + * + * @param {string} value literal text + * @returns {string} + */ +function escapeRegExp(value) { + return value.replaceAll(/[.*+?^${}()|[\]\\]/gu, "\\$&"); +} + +const PROHIBITED_NAMES = [ + "Q2xhdWRl", + "Q3Vyc29y", + "R1BU", + "aWRlYS1jb20=", + "aWRlYS1jb2xsYWI=", + "cG9ueXRhaWw=", +].map((value) => Buffer.from(value, "base64").toString("utf8").toLowerCase()); +const HUMAN_ONLY_GATES = [ + "Capture real cluster settings, module rows, synthesis reads, context, and deployed templates with a read-only federated identity, then run every stack comparison.", + "Run the empty live infrastructure diff and review each target change set with the target deployment identity before deployment.", + "Run and record a fresh development install and an existing development upgrade as separate rehearsals. Drain and announce the scheduler before the upgrade.", + "Capture a restricted-partition fixture and run its partition-specific comparisons. No credential-free capture exists yet.", +]; + +/** + * Returns true for a JSON object with string keys. + * + * @param {unknown} value parsed value + * @returns {value is Record} + */ +function isRecord(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** + * Reads and validates a JSON object. + * + * @param {string} file file to read + * @returns {Record} + */ +function readJsonRecord(file) { + const value = JSON.parse(readFileSync(file, "utf8")); + if (!isRecord(value)) { + throw new Error(`${file} must contain a JSON object`); + } + return value; +} + +/** + * Reads an object member and validates that every value is a string. + * + * @param {Record} record containing object + * @param {string} key member name + * @returns {Record} + */ +function readStringMap(record, key) { + const value = record[key]; + if (!isRecord(value)) { + throw new Error(`${key} must be an object`); + } + const result = {}; + for (const [name, version] of Object.entries(value)) { + if (typeof version !== "string") { + throw new Error(`${key}.${name} must be a string`); + } + result[name] = version; + } + return result; +} + +/** + * Audits direct dependencies against exact lockfile resolutions. + * + * @param {string} packageRoot package directory + */ +export function checkDependencyPins(packageRoot) { + const manifest = readJsonRecord(join(packageRoot, "package.json")); + const lockfile = readJsonRecord(join(packageRoot, "package-lock.json")); + const packages = lockfile.packages; + if (!isRecord(packages)) { + throw new Error("package-lock.json packages must be an object"); + } + const lockRoot = packages[""]; + if (!isRecord(lockRoot)) { + throw new Error("package-lock.json must describe the root package"); + } + + let checked = 0; + for (const section of ["dependencies", "devDependencies"]) { + const declared = readStringMap(manifest, section); + const locked = readStringMap(lockRoot, section); + for (const [name, version] of Object.entries(declared)) { + if (!EXACT_VERSION.test(version)) { + throw new Error(`${section}.${name} must be an exact version, found ${version}`); + } + if (locked[name] !== version) { + throw new Error(`${section}.${name} differs between package.json and package-lock.json`); + } + const installed = packages[`node_modules/${name}`]; + if (!isRecord(installed) || installed.version !== version) { + throw new Error(`${section}.${name} does not match its resolved lockfile version`); + } + checked += 1; + } + } + if (checked === 0) { + throw new Error("package.json must declare at least one dependency"); + } + console.log(`PASS dependency pins (${checked} direct packages)`); +} + +/** + * Returns true when a relative path is intentionally private or generated. + * + * @param {string} relativePath slash-separated path + * @returns {boolean} + */ +function isExcludedPath(relativePath) { + const parts = relativePath.split("/"); + if (parts.some((part) => EXCLUDED_DIRECTORIES.has(part))) { + return true; + } + return EXCLUDED_RELATIVE_DIRECTORIES.some( + (directory) => + relativePath === directory || relativePath.startsWith(`${directory}/`), + ); +} + +/** + * Walks text files without following symbolic links. + * + * @param {string} root directory to walk + * @returns {string[]} + */ +function listTextFiles(root) { + if (!existsSync(root)) { + return []; + } + const files = []; + const pending = [root]; + while (pending.length > 0) { + const directory = pending.pop(); + if (directory === undefined) { + continue; + } + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const absolute = join(directory, entry.name); + const relativePath = relative(root, absolute).split("\\").join("/"); + if (isExcludedPath(relativePath) || entry.isSymbolicLink()) { + continue; + } + if (entry.isDirectory()) { + pending.push(absolute); + continue; + } + if ( + entry.isFile() && + statSync(absolute).size <= 1_000_000 && + (entry.name === "Dockerfile" || TEXT_EXTENSIONS.has(extname(entry.name))) + ) { + files.push(absolute); + } + } + } + return files.sort(); +} + +/** + * Reports the one-based line containing an offset. + * + * @param {string} text complete file text + * @param {number} offset character offset + * @returns {number} + */ +function lineAt(text, offset) { + return text.slice(0, offset).split("\n").length; +} + +/** + * Adds one hygiene error with a stable path and line. + * + * @param {string[]} errors destination + * @param {string} root scan root + * @param {string} file matching file + * @param {string} text complete file text + * @param {number} offset character offset + * @param {string} reason failure reason + */ +function addHygieneError(errors, root, file, text, offset, reason) { + const path = relative(root, file).split("\\").join("/"); + errors.push(`${path}:${lineAt(text, offset)} ${reason}`); +} + +/** + * Checks one file for public repository hygiene violations. + * + * @param {string[]} errors destination + * @param {string} root scan root + * @param {string} file file to inspect + */ +function checkHygieneFile(errors, root, file) { + const bytes = readFileSync(file); + if (bytes.includes(0)) { + return; + } + const text = bytes.toString("utf8"); + const relativePath = relative(root, file).split("\\").join("/"); + const inTest = relativePath.startsWith("test/"); + const uuidRanges = [ + ...text.matchAll( + /\b[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}\b/gu, + ), + ].map((match) => [ + match.index, + match.index + match[0].length, + ]); + + for (const match of text.matchAll( + /(? match.index >= start && match.index < end, + ) + ) { + continue; + } + const value = match[0]; + const synthetic = + value === ["123456", "789012"].join("") || /^(\d)\1{11}$/u.test(value); + if (!inTest || !synthetic) { + addHygieneError( + errors, + root, + file, + text, + match.index, + "contains a prohibited account identifier", + ); + } + } + + for (const prohibited of PROHIBITED_NAMES) { + // A trailing letter or digit means a longer word, not the forbidden name: a compute-node + // resource name starts with the same letters as one of the cluster names. A trailing + // hyphen is still the name, because every resource derived from it carries one. + const match = new RegExp(`${escapeRegExp(prohibited)}(?![a-z0-9])`, "i").exec(text); + if (match !== null) { + addHygieneError( + errors, + root, + file, + text, + match.index, + "contains a prohibited private or product name", + ); + } + } + + const emDash = text.indexOf(String.fromCodePoint(0x2014)); + if (emDash >= 0) { + addHygieneError( + errors, + root, + file, + text, + emDash, + "contains an em dash", + ); + } + + for (const match of text.matchAll(/\b([A-Z]{2,10})-\d+\b/gu)) { + const prefix = match[1]; + if (prefix !== undefined && !ALLOWED_TICKET_LIKE_PREFIXES.has(prefix)) { + addHygieneError( + errors, + root, + file, + text, + match.index, + "contains a ticket-like identifier", + ); + } + } + + const secretPatterns = [ + /-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/gu, + /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/gu, + /\bgh[pousr]_[A-Za-z0-9_]{20,}\b/gu, + ]; + for (const pattern of secretPatterns) { + const match = pattern.exec(text); + if (match !== null) { + addHygieneError( + errors, + root, + file, + text, + match.index, + "contains secret material", + ); + } + } + + // A batch-system principal is `user@host`, which is not an email address and has no example + // domain to use. The batch server matches a connection against the name it reverse-resolves the + // caller's address to, so the host part must be a provider-shaped private DNS name or the grant + // silently matches nothing. Addresses in the documentation range are the correct example to write + // there, so accept a private DNS name built from one and keep everything else failing. + const documentationHost = /^ip-(192-0-2|198-51-100|203-0-113)-\d{1,3}\.[a-z0-9-]+\.compute\.internal$/u; + for (const match of text.matchAll( + /[A-Za-z0-9._%+-]+@([A-Za-z0-9.-]+\.[A-Za-z]{2,})/gu, + )) { + const host = match[1] ?? ""; + if (documentationHost.test(host)) continue; + if (host.toLowerCase() !== "example.invalid") { + addHygieneError( + errors, + root, + file, + text, + match.index, + "contains a non-example email address", + ); + } + } + + const copyrightPattern = new RegExp( + [ + "copy", + "right.{0,40}amazon|amazon.{0,40}copy", + "right", + ].join(""), + "iu", + ); + const copyright = copyrightPattern.exec(text); + if (copyright !== null) { + addHygieneError( + errors, + root, + file, + text, + copyright.index, + "contains a prohibited copyright header", + ); + } +} + +/** + * Audits public package files and workflow definitions. + * + * @param {string} packageRoot package directory + * @param {string | undefined} workflowRoot workflow directory + */ +export function checkHygiene(packageRoot, workflowRoot) { + const roots = [packageRoot]; + if (workflowRoot !== undefined && existsSync(workflowRoot)) { + roots.push(workflowRoot); + } + const errors = []; + let checked = 0; + for (const root of roots) { + for (const file of listTextFiles(root)) { + checkHygieneFile(errors, root, file); + checked += 1; + } + } + if (errors.length > 0) { + throw new Error(`repository hygiene failed:\n${errors.join("\n")}`); + } + console.log(`PASS repository hygiene (${checked} public files)`); +} + +/** + * Loads the explicit file-level skip allowances. + * + * @param {string} packageRoot package directory + * @returns {Record} + */ +function readSkipAllowances(packageRoot) { + const file = join(packageRoot, "scripts/ci-skip-allowances.json"); + if (!existsSync(file)) { + return {}; + } + const value = readJsonRecord(file); + const result = {}; + for (const [path, reason] of Object.entries(value)) { + if (typeof reason !== "string" || reason.trim() === "") { + throw new Error(`skip allowance ${path} must have a non-empty reason`); + } + result[path] = reason; + } + return result; +} + +/** + * Rejects test files that can skip without a file-level allowance. + * + * @param {string} packageRoot package directory + */ +export function checkSkipAllowances(packageRoot) { + const testRoot = join(packageRoot, "test"); + const allowances = readSkipAllowances(packageRoot); + const declaredSkips = []; + for (const file of listTextFiles(testRoot)) { + if (!TEST_FILE.test(file)) { + continue; + } + const source = readFileSync(file, "utf8"); + if (SKIP_DECLARATION.test(source)) { + declaredSkips.push(relative(packageRoot, file).split("\\").join("/")); + } + } + const missing = declaredSkips.filter((file) => allowances[file] === undefined); + const stale = Object.keys(allowances).filter((file) => !declaredSkips.includes(file)); + if (missing.length > 0 || stale.length > 0) { + const failures = [ + ...missing.map((file) => `${file} declares a skip without an allowance`), + ...stale.map((file) => `${file} has a stale skip allowance`), + ]; + throw new Error(`test skip policy failed:\n${failures.join("\n")}`); + } + console.log(`PASS test skip allowances (${declaredSkips.length} allowed files)`); +} + +/** + * Parses every workflow as a YAML object. + * + * @param {string} workflowRoot workflow directory + */ +export function checkWorkflows(workflowRoot) { + const yamlModule = require("js-yaml"); + if ( + !isRecord(yamlModule) || + typeof yamlModule.load !== "function" + ) { + throw new Error("YAML parser is unavailable"); + } + const workflows = readdirSync(workflowRoot, { withFileTypes: true }) + .filter( + (entry) => + entry.isFile() && + (entry.name.endsWith(".yaml") || entry.name.endsWith(".yml")), + ) + .map((entry) => join(workflowRoot, entry.name)) + .sort(); + if (workflows.length === 0) { + throw new Error(`no workflow files found under ${workflowRoot}`); + } + for (const workflow of workflows) { + const parsed = yamlModule.load(readFileSync(workflow, "utf8")); + if (!isRecord(parsed)) { + throw new Error(`${workflow} must contain a YAML object`); + } + } + console.log(`PASS workflow parsing (${workflows.length} workflows)`); +} + +/** + * Runs a subprocess and forwards all output. + * + * @param {string} label gate label + * @param {string} command executable + * @param {string[]} args arguments + * @param {string} cwd working directory + */ +function runChecked(label, command, args, cwd) { + const env = { ...process.env }; + delete env.NODE_TEST_CONTEXT; + const result = spawnSync(command, args, { + cwd, + encoding: "utf8", + env, + maxBuffer: 64 * 1024 * 1024, + }); + if (result.stdout !== "") { + process.stdout.write(result.stdout); + } + if (result.stderr !== "") { + process.stderr.write(result.stderr); + } + if (result.error !== undefined) { + throw new Error(`${label} could not start: ${result.error.message}`); + } + if (result.status !== 0) { + throw new Error(`${label} failed with exit ${result.status ?? "signal"}`); + } + console.log(`PASS ${label}`); +} + +/** + * Runs the package type check. + * + * @param {string} packageRoot package directory + */ +export function runTypeCheck(packageRoot) { + const compiler = join(DEFAULT_PACKAGE_ROOT, "node_modules/typescript/bin/tsc"); + runChecked( + "type check", + process.execPath, + [compiler, "--noEmit", "-p", join(packageRoot, "tsconfig.json")], + packageRoot, + ); +} + +/** + * Runs the complete package test suite with native type stripping. + * + * @param {string} packageRoot package directory + */ +export function runTests(packageRoot) { + runChecked( + "full test suite", + process.execPath, + ["--test", "test/**/*.test.ts"], + packageRoot, + ); +} + +/** + * Compares the committed synthetic template with the current synthesis. + * + * @param {string} packageRoot package directory + * @param {string | undefined} testFile optional test override + */ +export function runSyntheticParity(packageRoot, testFile) { + runChecked( + "synthetic template parity", + process.execPath, + ["--test", testFile ?? "test/ci/synthetic-parity.test.ts"], + packageRoot, + ); +} + +/** + * Prints the credential-backed gates that public automation cannot run. + */ +export function printHumanOnlyGates() { + console.log("HUMAN-ONLY CREDENTIAL GATES"); + for (const gate of HUMAN_ONLY_GATES) { + console.log(`- ${gate}`); + } +} + +/** + * Parses the small command-line interface. + * + * @param {string[]} argv process arguments + * @returns {{ + * command: string; + * packageRoot: string; + * workflowRoot: string | undefined; + * parityTest: string | undefined; + * }} + */ +function parseArguments(argv) { + const command = argv[0] ?? "all"; + let packageRoot = DEFAULT_PACKAGE_ROOT; + let workflowRoot; + let parityTest; + for (let index = 1; index < argv.length; index += 1) { + const flag = argv[index]; + const value = argv[index + 1]; + if ( + !["--root", "--workflows", "--parity-test"].includes(flag ?? "") || + value === undefined || + value === "" + ) { + throw new Error(`invalid argument: ${flag ?? ""}`); + } + if (flag === "--root") { + packageRoot = resolve(value); + } else if (flag === "--workflows") { + workflowRoot = resolve(value); + } else { + parityTest = value; + } + index += 1; + } + return { + command, + packageRoot, + workflowRoot: + workflowRoot ?? + (packageRoot === DEFAULT_PACKAGE_ROOT ? DEFAULT_WORKFLOW_ROOT : undefined), + parityTest, + }; +} + +/** + * Runs one requested gate or the complete credential-free set. + * + * @param {string[]} argv process arguments + */ +export async function runCli(argv) { + const options = parseArguments(argv); + const actions = { + dependencies: () => checkDependencyPins(options.packageRoot), + hygiene: () => checkHygiene(options.packageRoot, options.workflowRoot), + human: () => printHumanOnlyGates(), + parity: () => runSyntheticParity(options.packageRoot, options.parityTest), + skips: () => checkSkipAllowances(options.packageRoot), + tests: () => runTests(options.packageRoot), + typecheck: () => runTypeCheck(options.packageRoot), + workflows: () => { + if (options.workflowRoot === undefined) { + throw new Error("workflow parsing requires --workflows"); + } + checkWorkflows(options.workflowRoot); + }, + }; + + if (options.command === "all") { + const failures = []; + for (const gate of [ + "dependencies", + "hygiene", + "skips", + "workflows", + "typecheck", + "parity", + "tests", + ]) { + try { + actions[gate](); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(`FAIL ${message}`); + failures.push(gate); + } + } + if (failures.length === 0) { + console.log("PASS all credential-free gates"); + } else { + console.error(`FAIL credential-free gates: ${failures.join(", ")}`); + process.exitCode = 1; + } + actions.human(); + return; + } + + const action = actions[options.command]; + if (action === undefined) { + throw new Error(`unknown gate: ${options.command}`); + } + action(); +} + +const entry = process.argv[1]; +if ( + entry !== undefined && + pathToFileURL(resolve(entry)).href === import.meta.url +) { + try { + await runCli(process.argv.slice(2)); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(`FAIL ${message}`); + process.exitCode = 1; + } +} diff --git a/source/idea/ideactl/scripts/ci-skip-allowances.json b/source/idea/ideactl/scripts/ci-skip-allowances.json new file mode 100644 index 00000000..7fbd02c3 --- /dev/null +++ b/source/idea/ideactl/scripts/ci-skip-allowances.json @@ -0,0 +1,12 @@ +{ + "test/M-BINARY2/smoke.test.ts": "The release executables and archives are build outputs, absent until the distribution build has run.", + "test/cutover-killers/cutover-killers.test.ts": "A Java runtime is optional only in a checkout that declares it has no prerequisites.", + "test/g-backup/backup-continuity.test.ts": "Captured deployed templates are private and optional.", + "test/g-shapes/alternate-shapes.test.ts": "Private alternate-shape references are optional.", + "test/installer/installer-params.test.ts": "Captured installer values are private and optional.", + "test/jinja-branches/python-parity.test.ts": "The Python rendering oracle is optional only in a checkout that declares it has no prerequisites.", + "test/retain-stateful/retain-stateful.test.ts": "Captured deployed templates are private and optional; with none present there is nothing to itemise.", + "test/shapes/shape-matrix.test.ts": "Private alternate-shape references are optional.", + "test/upgrade-rehearsal/rehearse.test.ts": "The captured development fixture is private and optional.", + "test/w19/cluster-config-db.test.ts": "The captured synchronization oracle and local database service are optional." +} diff --git a/source/idea/ideactl/scripts/copy-resources.mjs b/source/idea/ideactl/scripts/copy-resources.mjs new file mode 100644 index 00000000..c4bab702 --- /dev/null +++ b/source/idea/ideactl/scripts/copy-resources.mjs @@ -0,0 +1,50 @@ +// Copies the runtime inputs into dist/ so the built package is self-contained. +import { cpSync, mkdirSync, copyFileSync, rmSync } from "node:fs"; +import { join, dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const repositoryRoot = resolve(packageRoot, "..", "..", ".."); +const administratorResources = join( + repositoryRoot, + "source", + "idea", + "idea-administrator", + "resources", +); +const outputResources = resolve( + process.env.IDEACTL_RESOURCE_OUTPUT_DIR ?? join(packageRoot, "dist", "resources"), +); +const bootstrapSource = resolve( + process.env.IDEACTL_BOOTSTRAP_SOURCE_DIR ?? + join(repositoryRoot, "source", "idea", "idea-bootstrap"), +); +const runtimeResourceDirectories = [ + "cdk", + "config", + "input_params", + "integration_tests", + "lambda_functions", + "policies", +]; + +rmSync(outputResources, { recursive: true, force: true }); +mkdirSync(outputResources, { recursive: true }); +for (const directory of runtimeResourceDirectories) { + cpSync( + join(administratorResources, directory), + join(outputResources, directory), + { recursive: true }, + ); +} +// The container module's config templates live in this package until resource ownership moves +// here. They are overlaid onto the copied tree so a released layout has one templates directory. +cpSync(join(packageRoot, "resources-ecs", "config"), join(outputResources, "config"), { + recursive: true, +}); +cpSync(bootstrapSource, join(outputResources, "bootstrap"), { recursive: true }); +copyFileSync( + join(repositoryRoot, "IDEA_VERSION.txt"), + join(dirname(outputResources), "IDEA_VERSION.txt"), +); +console.log("resources copied to dist/resources"); diff --git a/source/idea/ideactl/scripts/ideactl-linux.Dockerfile b/source/idea/ideactl/scripts/ideactl-linux.Dockerfile new file mode 100644 index 00000000..f3d8db8b --- /dev/null +++ b/source/idea/ideactl/scripts/ideactl-linux.Dockerfile @@ -0,0 +1,11 @@ +# syntax=docker/dockerfile:1.7 +# Carries the Linux release file. Amazon Linux needs libatomic for the official Node binary. + +FROM public.ecr.aws/amazonlinux/amazonlinux:2023 + +RUN dnf install -y libatomic && dnf clean all + +COPY ideactl /usr/local/bin/ideactl +RUN chmod 0755 /usr/local/bin/ideactl + +ENTRYPOINT ["/usr/local/bin/ideactl"] diff --git a/source/idea/ideactl/src/cdk/app.ts b/source/idea/ideactl/src/cdk/app.ts new file mode 100644 index 00000000..ee66baf1 --- /dev/null +++ b/source/idea/ideactl/src/cdk/app.ts @@ -0,0 +1,249 @@ +/** + * The CDK CLI entry point accepts: + * + * --cluster-name X --aws-region Y --module-id Z --module-name N --deployment-id U + * --termination-protection true|false [--aws-profile P] + * [--config-file F] [--synth-reads F] + * + * `--config-file` and `--synth-reads` replay the cluster settings and synth-time reads from files. + * Exactly one stack is built, then `app.synth()`. + */ + +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { parseArgs } from 'node:util'; +import { pathToFileURL } from 'node:url'; + +import { App, Aspects, CfnDeletionPolicy, CfnResource, type Environment, type IAspect } from 'aws-cdk-lib'; +import { AwsSolutionsChecks } from 'cdk-nag'; +import type { IConstruct } from 'constructs'; + +import { ClusterConfig } from '../config/cluster-config.ts'; +import { ideaVersion } from '../version.ts'; +import { makeContext, type IdeaContext } from './constructs/base.ts'; +import { isStatefulType } from './stateful.ts'; +import { liveSynthReads, replaySynthReads, type SynthReads } from './synth-reads.ts'; + +export interface CdkAppOptions { + clusterName: string; + awsRegion: string; + moduleId: string; + moduleName: string; + deploymentId: string; + terminationProtection: boolean; + awsProfile?: string; + configFile?: string; + synthReadsFile?: string; +} + +/** + * Sets `UpdateReplacePolicy: Retain` on every stateful resource in the tree. + * + * `DeletionPolicy` is left exactly as the stack set it. The two attributes answer different + * questions: this one governs the old resource when an update forces a replacement, where losing + * the data is never the intent, while `DeletionPolicy` governs a deliberate teardown, where it + * usually is. Retaining on teardown would leave litter the delete sweep then reports as a failure. + * + * Neither attribute is handed to the resource provider, so adding this to a stack that is already + * deployed changes CloudFormation's own bookkeeping and nothing about the resource itself. + */ +export class RetainStatefulOnUpdateReplace implements IAspect { + visit(node: IConstruct): void { + if (!CfnResource.isCfnResource(node)) return; + if (!isStatefulType(node.cfnResourceType)) return; + node.cfnOptions.updateReplacePolicy = CfnDeletionPolicy.RETAIN; + } +} + +/** + * What a stack module exports: it builds itself under the app when called. A builder that needs a + * synth-time AWS read returns a promise; `buildApp` awaits it, so the resource is in the tree + * before `app.synth()` writes the template. + */ +export type StackBuilder = (props: StackBuildProps) => void | Promise; + +export interface StackBuildProps { + app: App; + ctx: IdeaContext; + moduleName: string; + deploymentId: string; + terminationProtection: boolean; + env: Environment; +} + +export type StackRegistry = Record Promise>; + +/** + * Module name -> the file under `stacks/` that exports `buildStack`. The specifier is built at + * runtime so an unavailable stack module does not break type checking. + */ +const STACK_MODULES: Record = { + analytics: 'analytics', + 'bastion-host': 'bastion-host', + bootstrap: 'bootstrap', + cluster: 'cluster', + 'cluster-manager': 'cluster-manager', + directoryservice: 'directoryservice', + ecs: "ecs", + 'identity-provider': 'identity-provider', + metrics: 'metrics', + scheduler: 'scheduler', + 'shared-storage': 'shared-storage', + 'virtual-desktop-controller': 'vdc', +}; + +const MODULE_EXT = import.meta.url.endsWith('.ts') ? '.ts' : '.js'; + +export const DEFAULT_STACK_REGISTRY: StackRegistry = Object.fromEntries( + Object.entries(STACK_MODULES).map(([moduleName, file]) => [ + moduleName, + async () => { + const specifier = new URL(`./stacks/${file}${MODULE_EXT}`, import.meta.url).href; + const loaded = (await import(specifier)) as { buildStack?: StackBuilder }; + if (typeof loaded.buildStack !== 'function') { + throw new Error(`stack module for '${moduleName}' does not export buildStack()`); + } + return loaded.buildStack; + }, + ]), +); + +/** `Utils.get_as_bool(value, default)` for the strings the CLI and the environment hand us. */ +export function asBool(value: string | undefined, defaultValue: boolean): boolean { + if (value === undefined) return defaultValue; + const normalized = value.trim().toLowerCase(); + if (normalized === '') return defaultValue; + if (['true', 'yes', 'y', '1'].includes(normalized)) return true; + if (['false', 'no', 'n', '0'].includes(normalized)) return false; + return defaultValue; +} + +export function parseCdkAppArgs(argv: string[]): CdkAppOptions { + const { values } = parseArgs({ + args: argv, + options: { + 'cluster-name': { type: 'string' }, + 'aws-region': { type: 'string' }, + 'aws-profile': { type: 'string' }, + 'module-id': { type: 'string' }, + 'module-name': { type: 'string' }, + 'deployment-id': { type: 'string' }, + 'termination-protection': { type: 'string' }, + 'config-file': { type: 'string' }, + 'synth-reads': { type: 'string' }, + }, + strict: true, + allowPositionals: false, + }); + + const required = (name: keyof typeof values): string => { + const value = values[name]; + if (value === undefined || value === '') throw new Error(`--${String(name)} is required`); + return value; + }; + + const options: CdkAppOptions = { + clusterName: required('cluster-name'), + awsRegion: required('aws-region'), + moduleId: required('module-id'), + moduleName: required('module-name'), + deploymentId: required('deployment-id'), + terminationProtection: asBool(values['termination-protection'], true), + }; + if (values['aws-profile'] !== undefined) options.awsProfile = values['aws-profile']; + if (values['config-file'] !== undefined) options.configFile = values['config-file']; + if (values['synth-reads'] !== undefined) options.synthReadsFile = values['synth-reads']; + return options; +} + +/** + * Context the CDK CLI would normally hand us in `CDK_CONTEXT_JSON`: `cdk.json`'s `context` block + * and `cdk.context.json` from the working directory. Passed as `App({context})` **defaults**, so + * the CLI's values still win when the app runs under it, and a standalone synth resolves + * `Vpc.fromLookup` from the same file. + */ +export function readLocalContext(cwd: string = process.cwd()): Record { + const context: Record = {}; + const cdkJson = join(cwd, 'cdk.json'); + if (existsSync(cdkJson)) { + const parsed = JSON.parse(readFileSync(cdkJson, 'utf8')) as { context?: Record }; + Object.assign(context, parsed.context ?? {}); + } + const contextJson = join(cwd, 'cdk.context.json'); + if (existsSync(contextJson)) { + Object.assign(context, JSON.parse(readFileSync(contextJson, 'utf8')) as Record); + } + return context; +} + +async function loadConfig(options: CdkAppOptions): Promise { + if (options.configFile !== undefined) { + return ClusterConfig.fromFile(readFileSync(options.configFile, 'utf8')); + } + return ClusterConfig.fromDynamoDb(options.clusterName, options.awsRegion); +} + +/** Builds the app and the one stack. Returns the app so a caller can synth it itself. */ +export async function buildApp( + options: CdkAppOptions, + registry: StackRegistry = DEFAULT_STACK_REGISTRY, +): Promise { + const loadStack = registry[options.moduleName]; + if (loadStack === undefined) { + throw new Error( + `module not supported: '${options.moduleName}'. supported modules: ${Object.keys(registry).sort().join(', ')}`, + ); + } + + const synthReads: SynthReads = + options.synthReadsFile !== undefined + ? replaySynthReads(options.synthReadsFile) + : liveSynthReads(options.awsRegion, options.awsProfile); + + const config = await loadConfig(options); + const ctx = makeContext({ + config, + awsRegion: options.awsRegion, + awsProfile: options.awsProfile, + moduleId: options.moduleId, + releaseVersion: ideaVersion(), + synthReads, + }); + + // The default emits `Metadata.aws:cdk:path` for standalone synths and tests. + const app = new App({ context: { 'aws:cdk:enable-path-metadata': true, ...readLocalContext() } }); + + if (asBool(process.env.IDEA_ADMIN_ENABLE_CDK_NAG_SCAN, true)) { + Aspects.of(app).add(new AwsSolutionsChecks()); + } + + // An upgrade must never be able to delete state. Applied to the app so every stack is covered, + // including any added later, and applied as an aspect so it runs after the constructs have set + // their own removal policies. + Aspects.of(app).add(new RetainStatefulOnUpdateReplace()); + + const { account } = await synthReads.callerIdentity(); + const env: Environment = { account, region: options.awsRegion }; + + const buildStack = await loadStack(); + await buildStack({ + app, + ctx, + moduleName: options.moduleName, + deploymentId: options.deploymentId, + terminationProtection: options.terminationProtection, + env, + }); + + return app; +} + +/** `CdkApp.invoke`: build the one stack, then synth. */ +export async function main(argv: string[] = process.argv.slice(2)): Promise { + const app = await buildApp(parseCdkAppArgs(argv)); + app.synth(); +} + +if (process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href) { + await main(); +} diff --git a/source/idea/ideactl/src/cdk/base-stack.ts b/source/idea/ideactl/src/cdk/base-stack.ts new file mode 100644 index 00000000..d86ed626 --- /dev/null +++ b/source/idea/ideactl/src/cdk/base-stack.ts @@ -0,0 +1,202 @@ +/** + * Owns one `Stack` that module stacks add constructs under. The stack name and + * construct id are `${cluster}-${moduleId}`, which is the root of every + * `aws:cdk:path` and feeds `Names.uniqueId` for imported peers. + */ + +import { CustomResource, DefaultStackSynthesizer, Stack, type Environment } from 'aws-cdk-lib'; +import type { Construct, IConstruct } from 'constructs'; +import * as cognito from 'aws-cdk-lib/aws-cognito'; + +import { shake256Hex } from '../util/shake256.ts'; +import { getTargetGroupName } from '../util/names.ts'; +import type { IdeaContext, IdeaNagSuppression } from './constructs/base.ts'; +import { + IDEA_TAG_CLUSTER_NAME, + IDEA_TAG_MODULE_ID, + IDEA_TAG_MODULE_NAME, + IDEA_TAG_MODULE_VERSION, + METRICS_PROVIDER_AMAZON_MANAGED_PROMETHEUS, + addBackupTags, + addCommonTags, + addNagSuppression, + resourceName, +} from './constructs/base.ts'; + +export interface IdeaBaseStackProps { + scope: Construct; + ctx: IdeaContext; + /** The module *name* (`virtual-desktop-controller`), not the id, it becomes `idea:ModuleName`. */ + moduleName: string; + deploymentId: string; + terminationProtection: boolean; + env: Environment; +} + +/** Converts `Key=k,Value=v` strings to a record. */ +export function convertCustomTags(customTags: string[]): Record { + const result: Record = {}; + for (const customTag of customTags) { + const commaIndex = customTag.indexOf(','); + if (commaIndex < 0) continue; + const keyToken = customTag.slice(0, commaIndex); + const valueToken = customTag.slice(commaIndex + 1); + const keyParts = keyToken.split('Key='); + const valueParts = valueToken.split('Value='); + if (keyParts.length < 2 || valueParts.length < 2) continue; + const key = (keyParts[1] as string).trim(); + const value = (valueParts[1] as string).trim(); + if (key === '' || value === '') continue; + result[key] = value; + } + return result; +} + +export class IdeaBaseStack { + readonly context: IdeaContext; + readonly stack: Stack; + readonly stackName: string; + readonly clusterName: string; + readonly moduleId: string; + readonly moduleName: string; + readonly awsRegion: string; + readonly deploymentId: string; + readonly releaseVersion: string; + + constructor(props: IdeaBaseStackProps) { + const ctx = props.ctx; + this.context = ctx; + this.clusterName = ctx.clusterName; + this.moduleId = ctx.moduleId; + this.moduleName = props.moduleName; + this.awsRegion = ctx.awsRegion; + this.deploymentId = props.deploymentId; + this.releaseVersion = ctx.releaseVersion; + this.stackName = `${this.clusterName}-${this.moduleId}`; + + // Custom tags come first. IDEA tags win a key collision. + const tags: Record = { + ...convertCustomTags(ctx.config.getList('global-settings.custom_tags', [])), + [IDEA_TAG_MODULE_ID]: this.moduleId, + [IDEA_TAG_MODULE_NAME]: this.moduleName, + [IDEA_TAG_MODULE_VERSION]: this.releaseVersion, + [IDEA_TAG_CLUSTER_NAME]: this.clusterName, + }; + + this.stack = new Stack(props.scope, this.stackName, { + description: `ModuleId: ${this.moduleId}, Cluster: ${this.clusterName}, Version: ${this.releaseVersion}`, + env: props.env, + stackName: this.stackName, + tags, + terminationProtection: props.terminationProtection, + synthesizer: new DefaultStackSynthesizer({ + qualifier: shake256Hex(this.clusterName, 5), + bucketPrefix: 'cdk/', + fileAssetsBucketName: ctx.config.getString('cluster.cluster_s3_bucket', undefined, { + required: true, + }) as string, + }), + }); + } + + /** Builds a resource name with the module id. */ + buildResourceName(name: string, regionSuffix = false): string { + return resourceName(this.context, name, regionSuffix); + } + + /** + * Adds common tags with the module id as the name. The prefix list and EC2 + * state-change topic use `Name=-`. + */ + addCommonTags(construct: IConstruct): void { + addCommonTags(this.context, construct, this.moduleId); + } + + addBackupTags(construct: IConstruct): void { + addBackupTags(this.context, construct); + } + + addNagSuppression(suppressions: IdeaNagSuppression[], construct: IConstruct, applyToChildren = false): void { + addNagSuppression(construct, suppressions, applyToChildren); + } + + /** Builds a target group name and throws over 32 characters. */ + getTargetGroupName(identifier: string): string { + return getTargetGroupName(this.clusterName, this.moduleId, identifier); + } + + /** + * Adds the `Custom::ClusterSettings` resource that ends every module stack. + * Its id is `${cluster}-${moduleId}-settings`. + */ + updateClusterSettings(clusterSettings: Record): CustomResource { + const serviceToken = this.context.config.getString('cluster.cluster_settings_lambda_arn', undefined, { + required: true, + }) as string; + return new CustomResource(this.stack, `${this.clusterName}-${this.moduleId}-settings`, { + serviceToken, + properties: { + cluster_name: this.clusterName, + module_id: this.moduleId, + version: this.releaseVersion, + settings: clusterSettings, + }, + resourceType: 'Custom::ClusterSettings', + }); + } + + isMetricsProviderAmazonManagedPrometheus(): boolean { + const provider = this.context.config.getString('metrics.provider'); + if (provider === undefined || provider === '') return false; + return provider === METRICS_PROVIDER_AMAZON_MANAGED_PROMETHEUS; + } + + /** Returns the managed policies for EC2 instances. */ + getEc2InstanceManagedPolicies(): string[] { + const config = this.context.config; + const policies: string[] = [ + config.getString('cluster.iam.policies.amazon_ssm_managed_instance_core_arn', undefined, { + required: true, + }) as string, + // Logs always go to CloudWatch, regardless of the metrics provider. + config.getString('cluster.iam.policies.cloud_watch_agent_server_arn', undefined, { + required: true, + }) as string, + ]; + if (this.isMetricsProviderAmazonManagedPrometheus()) { + policies.push( + config.getString('cluster.iam.policies.amazon_prometheus_remote_write_arn', undefined, { + required: true, + }) as string, + ); + } + return [...policies, ...config.getList('cluster.iam.ec2_managed_policy_arns', [])]; + } + + /** Looks up the user pool at construct id `${cluster}-user-pool`. */ + lookupUserPool(): cognito.IUserPool { + return cognito.UserPool.fromUserPoolId( + this.stack, + `${this.clusterName}-user-pool`, + this.context.config.getString('identity-provider.cognito.user_pool_id', undefined, { + required: true, + }) as string, + ); + } + + /** Adds module administrators and users groups. */ + buildAccessControlGroups(userPool: cognito.IUserPool): void { + new cognito.CfnUserPoolGroup(this.stack, `${this.moduleId}-administrators-group`, { + description: `Module administrators group for module id: ${this.moduleId}, cluster: ${this.clusterName}`, + groupName: `${this.moduleId}-administrators-module-group`, + precedence: 3, + userPoolId: userPool.userPoolId, + }); + new cognito.CfnUserPoolGroup(this.stack, `${this.moduleId}-users-group`, { + description: `Module user group for module id: ${this.moduleId}, cluster: ${this.clusterName}`, + groupName: `${this.moduleId}-users-module-group`, + precedence: 4, + userPoolId: userPool.userPoolId, + }); + } +} diff --git a/source/idea/ideactl/src/cdk/code-asset.ts b/source/idea/ideactl/src/cdk/code-asset.ts new file mode 100644 index 00000000..89077d8e --- /dev/null +++ b/source/idea/ideactl/src/cdk/code-asset.ts @@ -0,0 +1,180 @@ +/** + * Locates Lambda code assets. + * + * A Lambda asset root is a directory whose top level holds the handler package and the shared + * commons package, plus any third-party dependencies the handler imports: + * + * //handler.py + * /idea_lambda_commons/... + * / + * + * The handler string is `.handler.handler`, so the package has to be a directory inside + * the asset root. `dist/resources/lambda_functions/` is the source of one package only, + * which is why it cannot be handed to `lambda.Code.fromAsset` as it stands. + * + * Two asset roots are supported: + * + * 1. `dist/resources/lambda_assets/`, built once by `scripts/build-lambda-zips.sh`; + * 2. an on-demand build under `~/.idea/build/lambda//pkg`, assembled from + * `dist/resources/lambda_functions`. + */ + +import { createHash } from 'node:crypto'; +import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); + +const COMMONS_PACKAGE = 'idea_lambda_commons'; +const CHECKSUM_FILE = 'source.checksum.sha'; +const BUILD_DIR = 'pkg'; + +/** Finds `dist/resources` for source and built execution. */ +export function distResourcesDir(): string { + const candidates = [join(HERE, '..', '..', 'resources'), join(HERE, '..', '..', 'dist', 'resources')]; + const found = candidates.find((candidate) => existsSync(candidate)); + if (found === undefined) { + throw new Error(`dist resources directory not found; looked in ${candidates.join(', ')}`); + } + return found; +} + +/** `~/.idea/build/lambda`, or under `IDEA_USER_HOME`. */ +function lambdaBuildDir(): string { + const home = process.env.IDEA_USER_HOME ?? join(homedir(), '.idea'); + return join(home, 'build', 'lambda'); +} + +/** Content checksum over one or more directories: sorted relative paths plus file bytes. */ +function checksumForDirs(dirs: string[]): string { + const hash = createHash('sha256'); + for (const dir of dirs) { + const walk = (current: string, prefix: string): void => { + for (const entry of readdirSync(current, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) { + const full = join(current, entry.name); + const relative = prefix === '' ? entry.name : `${prefix}/${entry.name}`; + if (entry.isDirectory()) { + walk(full, relative); + continue; + } + hash.update(relative); + hash.update(readFileSync(full)); + } + }; + hash.update(dir); + if (existsSync(dir) && statSync(dir).isDirectory()) walk(dir, ''); + } + return hash.digest('hex'); +} + +/** The interpreter used to install a handler's third-party dependencies. */ +function pythonBin(): string | undefined { + const candidates = [process.env.PYTHON, 'python3.13', 'python3'].filter( + (candidate): candidate is string => candidate !== undefined && candidate !== '', + ); + for (const candidate of candidates) { + const probe = spawnSync(candidate, ['--version'], { stdio: 'ignore' }); + if (probe.status === 0) return candidate; + } + return undefined; +} + +export class IdeaCodeAsset { + readonly lambdaPackageName: string; + + constructor(lambdaPackageName: string) { + this.lambdaPackageName = lambdaPackageName; + } + + /** `IdeaCodeAsset.lambda_handler`. */ + get lambdaHandler(): string { + return `${this.lambdaPackageName}.handler.handler`; + } + + /** The directory handed to `lambda.Code.fromAsset`. */ + assetPath(): string { + const resources = distResourcesDir(); + const prebuilt = join(resources, 'lambda_assets', this.lambdaPackageName); + if (existsSync(prebuilt)) return prebuilt; + + const functions = join(resources, 'lambda_functions'); + const source = join(functions, this.lambdaPackageName); + if (!existsSync(source)) { + throw new Error( + `lambda package not found: ${this.lambdaPackageName}; looked in ${prebuilt}, ${source}`, + ); + } + return this.buildLambda(functions, source); + } + + /** + * Assembles the asset root the handler string needs, reusing the previous build while the + * sources are unchanged. A handler with a `requirements.txt` needs a Python interpreter with + * pip; without one the build refuses instead of producing an asset that cannot import. + */ + private buildLambda(functionsDir: string, sourceDir: string): string { + const commonsDir = join(functionsDir, COMMONS_PACKAGE); + if (!existsSync(commonsDir)) { + throw new Error(`lambda commons package not found: ${commonsDir}`); + } + const buildRoot = join(lambdaBuildDir(), this.lambdaPackageName); + const pkg = join(buildRoot, BUILD_DIR); + const checksumPath = join(buildRoot, CHECKSUM_FILE); + const checksum = checksumForDirs([sourceDir, commonsDir]); + + if (existsSync(pkg) && existsSync(checksumPath) && readFileSync(checksumPath, 'utf-8').trim() === checksum) { + return pkg; + } + + rmSync(buildRoot, { recursive: true, force: true }); + mkdirSync(pkg, { recursive: true }); + cpSync(commonsDir, join(pkg, COMMONS_PACKAGE), { recursive: true }); + cpSync(sourceDir, join(pkg, this.lambdaPackageName), { recursive: true }); + + const requirements = join(pkg, this.lambdaPackageName, 'requirements.txt'); + if (existsSync(requirements)) { + const moved = join(pkg, 'requirements.txt'); + renameSync(requirements, moved); + const python = pythonBin(); + if (python === undefined) { + rmSync(buildRoot, { recursive: true, force: true }); + throw new Error( + `lambda package ${this.lambdaPackageName} has dependencies in requirements.txt and no ` + + 'python interpreter was found to install them. Build the assets once with ' + + 'scripts/build-lambda-zips.sh, or set PYTHON to an interpreter with pip.', + ); + } + const install = spawnSync( + python, + [ + '-m', + 'pip', + 'install', + '-r', + 'requirements.txt', + '--platform', + 'manylinux2014_x86_64', + '--only-binary=:all:', + '--target', + '.', + '--upgrade', + ], + { cwd: pkg, stdio: 'inherit' }, + ); + if (install.status !== 0) { + rmSync(buildRoot, { recursive: true, force: true }); + throw new Error( + `failed to install dependencies for lambda package ${this.lambdaPackageName} ` + + `(${python} -m pip exited ${String(install.status)}). Build the assets once with ` + + 'scripts/build-lambda-zips.sh.', + ); + } + } + + writeFileSync(checksumPath, `${checksum}\n`); + return pkg; + } +} diff --git a/source/idea/ideactl/src/cdk/constructs/analytics.ts b/source/idea/ideactl/src/cdk/constructs/analytics.ts new file mode 100644 index 00000000..f6fac722 --- /dev/null +++ b/source/idea/ideactl/src/cdk/constructs/analytics.ts @@ -0,0 +1,153 @@ +/** + * `OpenSearch` extends the L2 directly. Defaults apply only when the caller provides no value, + * and are load-bearing for the template. + */ + +import { RemovalPolicy } from 'aws-cdk-lib'; +import type { Construct } from 'constructs'; +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import type * as kms from 'aws-cdk-lib/aws-kms'; +import * as opensearch from 'aws-cdk-lib/aws-opensearchservice'; + +import { ArnBuilder } from '../../config/arn-builder.ts'; +import { isEmpty } from '../../config/cluster-config.ts'; +import type { IdeaContext } from './base.ts'; +import { addCommonTags, addNagSuppression, resourceName } from './base.ts'; +import type { ExistingSocaCluster } from './existing-resources.ts'; + +export interface OpenSearchProps { + cluster: ExistingSocaCluster; + securityGroups: ec2.ISecurityGroup[]; + dataNodes: number; + dataNodeInstanceType: string; + ebsVolumeSize: number; + /** Defaults to `DESTROY` when omitted. */ + removalPolicy?: RemovalPolicy; + version?: opensearch.EngineVersion; + /** Defaults to `true`. */ + createServiceLinkedRole?: boolean; + accessPolicies?: iam.PolicyStatement[]; + advancedOptions?: Record; + automatedSnapshotStartHour?: number; + capacity?: opensearch.CapacityConfig; + cognitoDashboardsAuth?: opensearch.CognitoOptions; + customEndpoint?: opensearch.CustomEndpointOptions; + domainName?: string; + ebs?: opensearch.EbsOptions; + enableVersionUpgrade?: boolean; + encryptionAtRest?: opensearch.EncryptionAtRestOptions; + /** Holds an `IKey` used only by `encryptionAtRest`. */ + kmsKeyArn?: kms.IKey; + enforceHttps?: boolean; + fineGrainedAccessControl?: opensearch.AdvancedSecurityOptions; + logging?: opensearch.LoggingOptions; + nodeToNodeEncryption?: boolean; + tlsSecurityPolicy?: opensearch.TLSSecurityPolicy; + useUnsignedBasicAuth?: boolean; + vpcSubnets?: ec2.SubnetSelection[]; + zoneAwareness?: opensearch.ZoneAwarenessConfig; +} + +/** Applies defaults before calling the L2 constructor. */ +function domainProps(ctx: IdeaContext, name: string, props: OpenSearchProps): opensearch.DomainProps { + const dataNodes = props.dataNodes; + + let zoneAwareness = props.zoneAwareness; + if (zoneAwareness === undefined) { + zoneAwareness = + dataNodes > 1 + ? { enabled: true, availabilityZoneCount: Math.min(3, dataNodes) } + : { enabled: false }; + } + + const domainName = props.domainName ?? resourceName(ctx, name).toLowerCase(); + + let accessPolicies = props.accessPolicies; + if (accessPolicies === undefined) { + const arnBuilder = new ArnBuilder(ctx.config); + accessPolicies = [ + new iam.PolicyStatement({ + principals: [new iam.AnyPrincipal()], + actions: ['es:ESHttp*'], + resources: [arnBuilder.getArn('es', `domain/${domainName}/*`)], + }), + ]; + } + + return { + version: props.version ?? opensearch.EngineVersion.OPENSEARCH_2_19, + accessPolicies, + advancedOptions: props.advancedOptions ?? { 'rest.action.multi.allow_explicit_index': 'true' }, + // 0 in every deployment, and CDK drops `SnapshotOptions` because 0 is falsy. Do not "fix" it. + automatedSnapshotStartHour: props.automatedSnapshotStartHour ?? 0, + capacity: props.capacity ?? { + dataNodeInstanceType: props.dataNodeInstanceType, + dataNodes, + }, + cognitoDashboardsAuth: props.cognitoDashboardsAuth, + customEndpoint: props.customEndpoint, + domainName, + ebs: props.ebs ?? { + volumeSize: props.ebsVolumeSize, + volumeType: ec2.EbsDeviceVolumeType.GP3, + }, + enableVersionUpgrade: props.enableVersionUpgrade, + encryptionAtRest: props.encryptionAtRest ?? { enabled: true, kmsKey: props.kmsKeyArn }, + enforceHttps: props.enforceHttps ?? true, + fineGrainedAccessControl: props.fineGrainedAccessControl, + logging: props.logging, + nodeToNodeEncryption: props.nodeToNodeEncryption, + removalPolicy: props.removalPolicy ?? RemovalPolicy.DESTROY, + securityGroups: props.securityGroups, + tlsSecurityPolicy: props.tlsSecurityPolicy, + useUnsignedBasicAuth: props.useUnsignedBasicAuth, + vpc: props.cluster.vpc, + vpcSubnets: props.vpcSubnets ?? [{ subnets: props.cluster.privateSubnets.slice(0, dataNodes) }], + zoneAwareness, + }; +} + +export class OpenSearch extends opensearch.Domain { + constructor(ctx: IdeaContext, name: string, scope: Construct, props: OpenSearchProps) { + super(scope, name, domainProps(ctx, name, props)); + + addCommonTags(ctx, this, name); + + if (props.createServiceLinkedRole !== false) { + let awsServiceName = ctx.config.getString('global-settings.opensearch.aws_service_name'); + if (isEmpty(awsServiceName)) { + const dnsSuffix = ctx.config.getString('cluster.aws.dns_suffix', undefined, { + required: true, + }) as string; + awsServiceName = `es.${dnsSuffix}`; + } + // DO NOT CHANGE THE DESCRIPTION OF THE ROLE: it is what AWS matches an existing SLR against. + const serviceLinkedRole = new iam.CfnServiceLinkedRole( + this, + resourceName(ctx, 'es-service-linked-role'), + { + awsServiceName, + description: 'Role for ES to access resources in the VPC', + }, + ); + this.node.addDependency(serviceLinkedRole); + } + + addNagSuppression(this, [ + { + rule_id: 'AwsSolutions-OS3', + reason: 'Access to OpenSearch cluster is restricted within a VPC', + }, + { + rule_id: 'AwsSolutions-OS4', + reason: + 'Use existing resources flow to provision an even more scalable OpenSearch cluster with dedicated master nodes', + }, + { + rule_id: 'AwsSolutions-OS5', + reason: 'Access to OpenSearch cluster is restricted within a VPC', + }, + ]); + } +} diff --git a/source/idea/ideactl/src/cdk/constructs/backup.ts b/source/idea/ideactl/src/cdk/constructs/backup.ts new file mode 100644 index 00000000..a1285749 --- /dev/null +++ b/source/idea/ideactl/src/cdk/constructs/backup.ts @@ -0,0 +1,98 @@ +/** + * A plain holder around the `backup.BackupPlan` / `backup.BackupSelection` L2s. Construct ids are + * `` and `-selection`, the plan's id is the *plan name* + * (`-`). + * + * `disableDefaultBackupPolicy: true` keeps the L2 from attaching + * `AWSBackupServiceRolePolicyForBackup` to the role it is handed; the cluster stack builds that + * role with the copied policies itself. + */ + +import { Duration } from 'aws-cdk-lib'; +import type { Construct } from 'constructs'; +import * as backup from 'aws-cdk-lib/aws-backup'; +import * as events from 'aws-cdk-lib/aws-events'; +import type * as iam from 'aws-cdk-lib/aws-iam'; + +import { convertCustomTags } from '../base-stack.ts'; +import { valueAsDict, valueAsInt, valueAsList, valueAsString, valueAsBool } from './storage.ts'; + +export interface BackupPlanProps { + backupPlanName: string; + /** The `.backups.backup_plan` subtree. */ + backupPlanConfig: Record | undefined; + backupVault: backup.IBackupVault; + backupRole: iam.IRole; +} + +export class BackupPlan { + readonly backupPlan: backup.BackupPlan; + readonly backupSelection: backup.BackupSelection; + + constructor(scope: Construct, props: BackupPlanProps) { + const config = props.backupPlanConfig; + + const rules: backup.BackupPlanRule[] = []; + const ruleConfigs = valueAsDict('rules', config) ?? {}; + for (const [ruleName, rawRule] of Object.entries(ruleConfigs)) { + const rule = rawRule as Record | undefined; + const deleteAfterDays = valueAsInt('delete_after_days', rule); + const startWindowMinutes = valueAsInt('start_window_minutes', rule); + const completionWindowMinutes = valueAsInt('completion_window_minutes', rule); + const scheduleExpression = valueAsString('schedule_expression', rule); + const moveToColdStorageAfterDays = valueAsInt('move_to_cold_storage_after_days', rule); + + // All four rule properties are required. + if (deleteAfterDays === undefined) throw new Error(`backup rule ${ruleName}: delete_after_days is required`); + if (startWindowMinutes === undefined) { + throw new Error(`backup rule ${ruleName}: start_window_minutes is required`); + } + if (completionWindowMinutes === undefined) { + throw new Error(`backup rule ${ruleName}: completion_window_minutes is required`); + } + if (scheduleExpression === undefined) { + throw new Error(`backup rule ${ruleName}: schedule_expression is required`); + } + + rules.push( + new backup.BackupPlanRule({ + ruleName, + backupVault: props.backupVault, + startWindow: Duration.minutes(startWindowMinutes as number), + completionWindow: Duration.minutes(completionWindowMinutes as number), + deleteAfter: Duration.days(deleteAfterDays as number), + moveToColdStorageAfter: + moveToColdStorageAfterDays === undefined + ? undefined + : Duration.days(moveToColdStorageAfterDays as number), + scheduleExpression: events.Schedule.expression(scheduleExpression), + }), + ); + } + + this.backupPlan = new backup.BackupPlan(scope, props.backupPlanName, { + backupPlanName: props.backupPlanName, + backupPlanRules: rules, + backupVault: props.backupVault, + windowsVss: valueAsBool('enable_windows_vss', config, false), + }); + + const selection = valueAsDict('selection', config) ?? {}; + const selectionTags = convertCustomTags((valueAsList('tags', selection) ?? []) as string[]); + const resources = Object.entries(selectionTags).map(([key, value]) => + backup.BackupResource.fromTag(key, value, backup.TagOperation.STRING_EQUALS), + ); + + this.backupSelection = new backup.BackupSelection(scope, `${props.backupPlanName}-selection`, { + backupPlan: this.backupPlan, + resources, + backupSelectionName: `${props.backupPlanName}-selection`, + role: props.backupRole, + disableDefaultBackupPolicy: true, + }); + } + + getBackupPlanArn(): string { + return this.backupPlan.backupPlanArn; + } +} diff --git a/source/idea/ideactl/src/cdk/constructs/base.ts b/source/idea/ideactl/src/cdk/constructs/base.ts new file mode 100644 index 00000000..ab4a085f --- /dev/null +++ b/source/idea/ideactl/src/cdk/constructs/base.ts @@ -0,0 +1,164 @@ +/** + * Helpers for construct identifiers, tags, physical names, and suppressions. + */ + +import type { IConstruct } from 'constructs'; +import { Aws, Stack, Tags } from 'aws-cdk-lib'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import { NagSuppressions } from 'cdk-nag'; + +import type { ClusterConfig } from '../../config/cluster-config.ts'; +import type { SynthReads } from '../synth-reads.ts'; +import { + buildInstanceProfileArn, + buildResourceName, + buildTrimmedResourceName, + getKmsKeyArn, +} from '../../util/names.ts'; + +/** `ideadatamodel.constants` tag keys used by the CDK app. */ +export const IDEA_TAG_NAME = 'Name'; +export const IDEA_TAG_CLUSTER_NAME = 'idea:ClusterName'; +export const IDEA_TAG_MODULE_ID = 'idea:ModuleId'; +export const IDEA_TAG_MODULE_NAME = 'idea:ModuleName'; +export const IDEA_TAG_MODULE_VERSION = 'idea:ModuleVersion'; +export const IDEA_TAG_BACKUP_PLAN = 'idea:BackupPlan'; +export const IDEA_TAG_NODE_TYPE = 'idea:NodeType'; + +export const MODULE_CLUSTER = 'cluster'; +export const METRICS_PROVIDER_AMAZON_MANAGED_PROMETHEUS = 'amazon_managed_prometheus'; +export const DIRECTORYSERVICE_OPENLDAP = 'openldap'; +export const DIRECTORYSERVICE_ACTIVE_DIRECTORY = 'activedirectory'; +export const DIRECTORYSERVICE_AWS_MANAGED_ACTIVE_DIRECTORY = 'aws_managed_activedirectory'; + +export interface IdeaNagSuppression { + rule_id: string; + reason: string; +} + +/** + * Context built once per synth and handed to every construct. + */ +export interface IdeaContext { + readonly config: ClusterConfig; + /** `cluster.cluster_name`, required. */ + readonly clusterName: string; + readonly awsRegion: string; + readonly awsProfile: string | undefined; + /** The module id this process is synthesizing. */ + readonly moduleId: string; + /** `ideaadministrator.__version__`, the release stamped into descriptions and settings. */ + readonly releaseVersion: string; + readonly synthReads: SynthReads; +} + +export function makeContext(input: { + config: ClusterConfig; + awsRegion: string; + awsProfile?: string; + moduleId: string; + releaseVersion: string; + synthReads: SynthReads; +}): IdeaContext { + return { + config: input.config, + clusterName: input.config.getString('cluster.cluster_name', undefined, { required: true }) as string, + awsRegion: input.awsRegion, + awsProfile: input.awsProfile, + moduleId: input.moduleId, + releaseVersion: input.releaseVersion, + synthReads: input.synthReads, + }; +} + +/** `SocaBaseConstruct.get_construct_id`: the name, verbatim. */ +export function constructId(name: string): string { + return name; +} + +/** `Utils.to_title_case`: `-`/`_` to spaces, title case, spaces dropped. */ +export function toTitleCase(value: string): string { + return value + .replace(/[-_]/g, ' ') + // python str.title() upper-cases the first cased character of every run and lowers the rest + .replace(/[A-Za-z]+/g, (word) => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase()) + .replace(/\s+/g, ''); +} + +/** `SocaBaseConstruct.build_resource_name`. */ +export function resourceName(ctx: IdeaContext, name: string, regionSuffix = false): string { + return buildResourceName( + ctx.clusterName, + name, + regionSuffix ? (ctx.config.getString('cluster.aws.region', undefined, { required: true }) as string) : undefined, + ); +} + +/** `SocaBaseConstruct.build_trimmed_resource_name`. */ +export function trimmedResourceName( + ctx: IdeaContext, + name: string, + regionSuffix = false, + trimLength = 64, +): string { + return buildTrimmedResourceName( + ctx.clusterName, + name, + regionSuffix ? (ctx.config.getString('cluster.aws.region', undefined, { required: true }) as string) : undefined, + trimLength, + ); +} + +/** `SocaBaseConstruct.add_common_tags`: `Name` first, then `idea:ClusterName`. */ +export function addCommonTags(ctx: IdeaContext, construct: IConstruct, name: string): void { + Tags.of(construct).add(IDEA_TAG_NAME, resourceName(ctx, name)); + Tags.of(construct).add(IDEA_TAG_CLUSTER_NAME, ctx.clusterName); +} + +/** `SocaBaseConstruct.add_backup_tags`. */ +export function addBackupTags(ctx: IdeaContext, construct: IConstruct): void { + Tags.of(construct).add(IDEA_TAG_BACKUP_PLAN, `${ctx.clusterName}-${MODULE_CLUSTER}`); +} + +/** `SocaBaseConstruct.build_service_principal`: `.${AWS::URLSuffix}`. */ +export function buildServicePrincipal(serviceName: string): iam.ServicePrincipal { + return new iam.ServicePrincipal(`${serviceName}.${Aws.URL_SUFFIX}`); +} + +/** `SocaBaseConstruct.get_kms_key_arn`. */ +export function kmsKeyArn(ctx: IdeaContext, keyId: string): string { + return getKmsKeyArn( + keyId, + ctx.config.getString('cluster.aws.partition', undefined, { required: true }) as string, + ctx.config.getString('cluster.aws.region', undefined, { required: true }) as string, + ctx.config.getString('cluster.aws.account_id', undefined, { required: true }) as string, + ); +} + +/** `SocaBaseConstruct.build_instance_profile_arn`. */ +export function instanceProfileArn(ctx: IdeaContext, instanceProfileRef: string): string { + return buildInstanceProfileArn( + ctx.config.getString('cluster.aws.partition', undefined, { required: true }) as string, + ctx.config.getString('cluster.aws.account_id', undefined, { required: true }) as string, + instanceProfileRef, + ); +} + +export function isDsActivedirectory(ctx: IdeaContext): boolean { + const provider = ctx.config.getString('directoryservice.provider'); + return provider === DIRECTORYSERVICE_AWS_MANAGED_ACTIVE_DIRECTORY || provider === DIRECTORYSERVICE_ACTIVE_DIRECTORY; +} + +/** `SocaBaseConstruct.add_nag_suppression`. */ +export function addNagSuppression( + construct: IConstruct, + suppressions: IdeaNagSuppression[], + applyToChildren = false, +): void { + const rules = suppressions.map((suppression) => ({ id: suppression.rule_id, reason: suppression.reason })); + if (Stack.isStack(construct)) { + NagSuppressions.addStackSuppressions(construct, rules, applyToChildren); + } else { + NagSuppressions.addResourceSuppressions(construct, rules, applyToChildren); + } +} diff --git a/source/idea/ideactl/src/cdk/constructs/common.ts b/source/idea/ideactl/src/cdk/constructs/common.ts new file mode 100644 index 00000000..186898af --- /dev/null +++ b/source/idea/ideactl/src/cdk/constructs/common.ts @@ -0,0 +1,508 @@ +/** + * Common CDK constructs that call the helpers from `base.ts`. + */ + +import { Duration, RemovalPolicy, CustomResource as CdkCustomResource, CfnOutput } from 'aws-cdk-lib'; +import type { Construct } from 'constructs'; +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as kinesis from 'aws-cdk-lib/aws-kinesis'; +import * as kms from 'aws-cdk-lib/aws-kms'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import * as logs from 'aws-cdk-lib/aws-logs'; +import * as sns from 'aws-cdk-lib/aws-sns'; +import * as sqs from 'aws-cdk-lib/aws-sqs'; + +import { isEmpty } from '../../config/cluster-config.ts'; +import { IdeaCodeAsset } from '../code-asset.ts'; +import { renderPolicy } from '../policy.ts'; +import type { IdeaContext } from './base.ts'; +import { + addCommonTags, + addNagSuppression, + buildServicePrincipal, + constructId, + kmsKeyArn, + resourceName, + toTitleCase, + trimmedResourceName, +} from './base.ts'; + +/** `LOG_RETENTION_DAYS`, the retention values a log group accepts, mapped to the cdk enum. */ +export const LOG_RETENTION_DAYS: Record = { + 1: logs.RetentionDays.ONE_DAY, + 3: logs.RetentionDays.THREE_DAYS, + 5: logs.RetentionDays.FIVE_DAYS, + 7: logs.RetentionDays.ONE_WEEK, + 14: logs.RetentionDays.TWO_WEEKS, + 30: logs.RetentionDays.ONE_MONTH, + 60: logs.RetentionDays.TWO_MONTHS, + 90: logs.RetentionDays.THREE_MONTHS, + 120: logs.RetentionDays.FOUR_MONTHS, + 150: logs.RetentionDays.FIVE_MONTHS, + 180: logs.RetentionDays.SIX_MONTHS, + 365: logs.RetentionDays.ONE_YEAR, + 400: logs.RetentionDays.THIRTEEN_MONTHS, + 545: logs.RetentionDays.EIGHTEEN_MONTHS, + 731: logs.RetentionDays.TWO_YEARS, + 1827: logs.RetentionDays.FIVE_YEARS, + 3653: logs.RetentionDays.TEN_YEARS, +}; + +const MAX_NAME_LENGTH = 64; + +// --- Lambda ----------------------------------------------------------------------------------- + +export interface LambdaFunctionProps { + ideaCodeAsset?: IdeaCodeAsset; + code?: lambda.Code; + handler?: string; + description?: string; + memorySize?: number; + runtime?: lambda.Runtime; + timeoutSeconds?: number; + vpc?: ec2.IVpc; + securityGroups?: ec2.ISecurityGroup[]; + vpcSubnets?: ec2.SubnetSelection; + logRetention?: logs.RetentionDays; + logRetentionRole?: iam.IRole; + environment?: Record; + role?: iam.IRole; +} + +function lambdaFunctionName(ctx: IdeaContext, name: string): string { + const functionName = resourceName(ctx, name); + return functionName.length > MAX_NAME_LENGTH ? trimmedResourceName(ctx, name, false, MAX_NAME_LENGTH) : functionName; +} + +export class LambdaFunction extends lambda.Function { + constructor(ctx: IdeaContext, name: string, scope: Construct, props: LambdaFunctionProps = {}) { + let code = props.code; + let handler = props.handler; + if (props.ideaCodeAsset !== undefined) { + code = lambda.Code.fromAsset(props.ideaCodeAsset.assetPath()); + handler = props.ideaCodeAsset.lambdaHandler; + } else if (code === undefined || handler === undefined) { + throw new Error('Provide either idea_code_asset or (code and handler)'); + } + + super(scope, constructId(name), { + functionName: lambdaFunctionName(ctx, name), + description: props.description, + memorySize: props.memorySize ?? 128, + runtime: props.runtime ?? lambda.Runtime.PYTHON_3_13, + timeout: Duration.seconds(props.timeoutSeconds ?? 60), + logRetention: props.logRetention, + handler, + environment: props.environment, + code, + role: props.role, + vpc: props.vpc, + securityGroups: props.securityGroups, + vpcSubnets: props.vpcSubnets, + logRetentionRole: props.logRetentionRole, + }); + + addCommonTags(ctx, this, name); + addNagSuppression(this, [ + { rule_id: 'AwsSolutions-L1', reason: 'Lambda runtime uses Python 3.13 by default.' }, + ]); + } +} + +// --- IAM -------------------------------------------------------------------------------------- + +export interface PolicyProps { + policyTemplateName: string; + vars?: Record; + moduleId?: string; +} + +export class Policy extends iam.Policy { + constructor(ctx: IdeaContext, name: string, scope: Construct, props: PolicyProps) { + super(scope, constructId(name), { + document: iam.PolicyDocument.fromJson( + renderPolicy(props.policyTemplateName, { + config: ctx.config, + moduleId: props.moduleId, + vars: props.vars, + }), + ), + }); + + addCommonTags(ctx, this, name); + addNagSuppression(this, [ + { + rule_id: 'AwsSolutions-IAM5', + reason: 'Wild-card policies are scoped with conditions and/or applicable prefixes.', + }, + ]); + } +} + +export interface ManagedPolicyProps extends PolicyProps { + description: string; + managedPolicyName: string; +} + +export class ManagedPolicy extends iam.ManagedPolicy { + constructor(ctx: IdeaContext, name: string, scope: Construct, props: ManagedPolicyProps) { + super(scope, constructId(name), { + managedPolicyName: props.managedPolicyName, + description: props.description, + document: iam.PolicyDocument.fromJson( + renderPolicy(props.policyTemplateName, { + config: ctx.config, + moduleId: props.moduleId, + vars: props.vars, + }), + ), + }); + + addCommonTags(ctx, this, name); + addNagSuppression(this, [ + { + rule_id: 'AwsSolutions-IAM5', + reason: + 'AWS Managed Policies are expected to be customized and scoped down.' + + 'AWS Managed policies are copied over to enable these customizations.', + }, + ]); + } +} + +export interface RoleProps { + description: string; + /** Service names, turned into `.${AWS::URLSuffix}` principals. */ + assumedBy: string[]; + inlinePolicies?: iam.Policy[]; + /** Managed policy ARNs, or AWS managed policy names. */ + managedPolicies?: string[]; +} + +function roleName(ctx: IdeaContext, name: string): string { + const built = resourceName(ctx, name, true); + return built.length > MAX_NAME_LENGTH ? trimmedResourceName(ctx, name, true, MAX_NAME_LENGTH) : built; +} + +export class Role extends iam.Role { + constructor(ctx: IdeaContext, name: string, scope: Construct, props: RoleProps) { + super(scope, constructId(name), { + roleName: roleName(ctx, name), + description: props.description, + assumedBy: new iam.CompositePrincipal(...props.assumedBy.map(buildServicePrincipal)), + }); + + addCommonTags(ctx, this, name); + + for (const policy of props.inlinePolicies ?? []) { + this.attachInlinePolicy(policy); + } + for (const policy of props.managedPolicies ?? []) { + if (policy.startsWith('arn:')) { + this.addManagedPolicy(iam.ManagedPolicy.fromManagedPolicyArn(this, policy.split('/')[1] as string, policy)); + } else { + this.addManagedPolicy(iam.ManagedPolicy.fromAwsManagedPolicyName(policy)); + } + } + } +} + +export class InstanceProfile extends iam.CfnInstanceProfile { + constructor(ctx: IdeaContext, name: string, scope: Construct, roles: iam.Role[]) { + super(scope, constructId(name), { + instanceProfileName: resourceName(ctx, name, true), + roles: roles.map((role) => role.roleName), + }); + + addCommonTags(ctx, this, name); + } +} + +// --- Custom resources ------------------------------------------------------------------------- + +export interface CustomResourceProviderProps { + ideaCodeAsset: IdeaCodeAsset; + policyStatements?: iam.PolicyStatement[]; + policyTemplateName?: string; + removalPolicy?: RemovalPolicy; + resourceType?: string; + runtime?: lambda.Runtime; + lambdaTimeoutSeconds?: number; + lambdaLogRetentionRole?: iam.IRole; +} + +/** + * The policy, role, and lambda trio behind a `Custom::*` type. Construct ids are + * `-lambda-policy`, `-role`, and `-lambda`, all at the parent scope; + * the function depends on the role and then the policy (CloudFormation renders `DependsOn` sorted). + */ +export class CustomResourceProvider { + readonly ctx: IdeaContext; + readonly name: string; + readonly scope: Construct; + readonly resourceType: string; + readonly removalPolicy: RemovalPolicy | undefined; + readonly lambdaPolicy: Policy | undefined; + readonly lambdaRole: Role; + readonly lambdaFunction: LambdaFunction; + + constructor(ctx: IdeaContext, name: string, scope: Construct, props: CustomResourceProviderProps) { + this.ctx = ctx; + this.name = name; + this.scope = scope; + this.removalPolicy = props.removalPolicy; + + const prefix = 'Custom::'; + const resourceType = props.resourceType as string; + if (isEmpty(resourceType)) { + this.resourceType = `${prefix}${toTitleCase(name)}`; + } else { + this.resourceType = resourceType.startsWith(prefix) ? resourceType : `${prefix}${resourceType}`; + } + + const policyTemplateName = props.policyTemplateName as string; + if (!isEmpty(policyTemplateName)) { + this.lambdaPolicy = new Policy(ctx, `${name}-lambda-policy`, scope, { policyTemplateName }); + } + + this.lambdaRole = new Role(ctx, `${name}-role`, scope, { + description: `Role for ${this.resourceType} for Cluster: ${ctx.clusterName}`, + assumedBy: ['lambda'], + }); + if (this.lambdaPolicy !== undefined) { + this.lambdaRole.attachInlinePolicy(this.lambdaPolicy); + } + + this.lambdaFunction = new LambdaFunction(ctx, `${name}-lambda`, scope, { + ideaCodeAsset: props.ideaCodeAsset, + description: `${this.resourceType} Lambda Function for Cluster: ${ctx.clusterName}`, + timeoutSeconds: props.lambdaTimeoutSeconds ?? 60, + role: this.lambdaRole, + logRetentionRole: props.lambdaLogRetentionRole, + runtime: props.runtime ?? lambda.Runtime.PYTHON_3_13, + }); + addNagSuppression(this.lambdaFunction, [ + { rule_id: 'AwsSolutions-L1', reason: 'Python Runtime is selected for stability.' }, + ]); + + for (const statement of props.policyStatements ?? []) { + this.lambdaFunction.addToRolePolicy(statement); + } + + this.lambdaFunction.node.addDependency(this.lambdaRole); + if (this.lambdaPolicy !== undefined) { + this.lambdaFunction.node.addDependency(this.lambdaPolicy); + } + } + + invoke(name: string, properties: Record): CdkCustomResource { + const customResource = new CdkCustomResource(this.scope, name, { + serviceToken: this.lambdaFunction.functionArn, + properties, + removalPolicy: this.removalPolicy, + resourceType: this.resourceType, + }); + customResource.node.addDependency(this.lambdaFunction); + return customResource; + } +} + +export class CreateTagsCustomResource extends CustomResourceProvider { + constructor(ctx: IdeaContext, scope: Construct, lambdaLogRetentionRole?: iam.IRole) { + super(ctx, 'ec2-create-tags', scope, { + ideaCodeAsset: new IdeaCodeAsset('idea_custom_resource_create_tags'), + policyTemplateName: 'custom-resource-ec2-create-tags.yml', + resourceType: 'EC2CreateTags', + lambdaLogRetentionRole, + }); + } + + apply(name: string, resourceId: string, tags: Record): CdkCustomResource { + const awsTags = Object.entries(tags).map(([key, value]) => ({ Key: key, Value: String(value) })); + return this.invoke(name, { ResourceId: resourceId, Tags: awsTags }); + } +} + +// --- Messaging -------------------------------------------------------------------------------- + +export interface SQSQueueProps { + contentBasedDeduplication?: boolean; + dataKeyReuse?: Duration; + deadLetterQueue?: sqs.DeadLetterQueue; + deduplicationScope?: sqs.DeduplicationScope; + deliveryDelay?: Duration; + /** Defaults to `true`. */ + encryptAtRest?: boolean; + encryption?: sqs.QueueEncryption; + /** A KMS key id or ARN from config, not a key object. */ + encryptionMasterKey?: string; + fifo?: boolean; + fifoThroughputLimit?: sqs.FifoThroughputLimit; + maxMessageSizeBytes?: number; + queueName?: string; + receiveMessageWaitTime?: Duration; + removalPolicy?: RemovalPolicy; + retentionPeriod?: Duration; + visibilityTimeout?: Duration; + isDeadLetterQueue?: boolean; +} + +function queueEncryption( + ctx: IdeaContext, + id: string, + scope: Construct, + props: SQSQueueProps, +): { encryption: sqs.QueueEncryption; encryptionMasterKey: kms.IKey | undefined } { + if (props.encryptAtRest === false) { + return { encryption: sqs.QueueEncryption.UNENCRYPTED, encryptionMasterKey: undefined }; + } + if (!isEmpty(props.encryptionMasterKey)) { + return { + encryption: sqs.QueueEncryption.KMS, + encryptionMasterKey: kms.Key.fromKeyArn( + scope, + `${id}-kms-key`, + kmsKeyArn(ctx, props.encryptionMasterKey as string), + ), + }; + } + return { encryption: props.encryption ?? sqs.QueueEncryption.KMS_MANAGED, encryptionMasterKey: undefined }; +} + +export class SQSQueue extends sqs.Queue { + constructor(ctx: IdeaContext, id: string, scope: Construct, props: SQSQueueProps = {}) { + const encrypted = props.encryptAtRest !== false; + const { encryption, encryptionMasterKey } = queueEncryption(ctx, id, scope, props); + + super(scope, constructId(id), { + contentBasedDeduplication: props.contentBasedDeduplication, + dataKeyReuse: props.dataKeyReuse, + deadLetterQueue: props.deadLetterQueue, + deduplicationScope: props.deduplicationScope, + deliveryDelay: props.deliveryDelay, + encryption, + encryptionMasterKey, + fifo: props.fifo, + fifoThroughputLimit: props.fifoThroughputLimit, + maxMessageSizeBytes: props.maxMessageSizeBytes, + queueName: props.queueName, + receiveMessageWaitTime: props.receiveMessageWaitTime, + removalPolicy: props.removalPolicy, + retentionPeriod: props.retentionPeriod, + visibilityTimeout: props.visibilityTimeout, + }); + + addCommonTags(ctx, this, id); + + if (encrypted) { + this.addToResourcePolicy( + new iam.PolicyStatement({ + sid: 'AlwaysEncrypted', + effect: iam.Effect.DENY, + actions: ['sqs:*'], + conditions: { Bool: { 'aws:SecureTransport': 'false' } }, + resources: [this.queueArn], + principals: [new iam.AnyPrincipal()], + }), + ); + } else { + addNagSuppression(this, [ + { + rule_id: 'AwsSolutions-SQS2', + reason: 'SQS encryption key is configurable, but is not provided in cluster config.', + }, + ]); + addNagSuppression(this, [ + { + rule_id: 'AwsSolutions-SQS4', + reason: 'SQS encryption key is configurable, but is not provided in cluster config.', + }, + ]); + } + + if (props.isDeadLetterQueue === true) { + addNagSuppression(this, [{ rule_id: 'AwsSolutions-SQS3', reason: 'Dead letter queue' }]); + } + } +} + +export interface SNSTopicProps { + fifo?: boolean; + /** A KMS key id or ARN from config; falls back to the `alias/aws/sns` managed key. */ + masterKey?: string; + displayName?: string; + topicName?: string; + policyStatements?: iam.PolicyStatement[]; +} + +export class SNSTopic extends sns.Topic { + constructor(ctx: IdeaContext, id: string, scope: Construct, props: SNSTopicProps = {}) { + const topicName = isEmpty(props.topicName) ? resourceName(ctx, id) : (props.topicName as string); + const displayName = isEmpty(props.displayName) ? topicName : (props.displayName as string); + const masterKey = isEmpty(props.masterKey) + ? kms.Alias.fromAliasName(scope, `${id}-kms-key-default`, 'alias/aws/sns') + : kms.Key.fromKeyArn(scope, `${id}-kms-key`, kmsKeyArn(ctx, props.masterKey as string)); + + super(scope, constructId(id), { displayName, fifo: props.fifo, topicName, masterKey }); + + addCommonTags(ctx, this, id); + + for (const statement of props.policyStatements ?? []) { + this.addToResourcePolicy(statement); + } + + this.addToResourcePolicy( + new iam.PolicyStatement({ + sid: 'AlwaysEncrypted', + effect: iam.Effect.DENY, + actions: ['SNS:Publish'], + conditions: { Bool: { 'aws:SecureTransport': 'false' } }, + resources: [this.topicArn], + principals: [new iam.AnyPrincipal()], + }), + ); + } +} + +// --- Streams and outputs ---------------------------------------------------------------------- + +export interface KinesisStreamProps { + streamName: string; + streamMode: kinesis.StreamMode; + shardCount?: number; + removalPolicy?: RemovalPolicy; +} + +export class KinesisStream extends kinesis.Stream { + constructor(ctx: IdeaContext, name: string, scope: Construct, props: KinesisStreamProps) { + const kmsKeyId = ctx.config.getString('analytics.kinesis.kms_key_id'); + const encryptionKey = + kmsKeyId !== undefined + ? kms.Key.fromKeyArn(scope, 'kinesis-kms-key', kmsKeyArn(ctx, kmsKeyId)) + : kms.Alias.fromAliasName(scope, 'kinesis-kms-key-default', 'alias/aws/kinesis'); + super(scope, constructId(name), { + streamName: `${ctx.clusterName}-${props.streamName}`, + streamMode: props.streamMode, + encryption: kinesis.StreamEncryption.KMS, + encryptionKey, + shardCount: props.shardCount, + removalPolicy: props.removalPolicy, + }); + + addCommonTags(ctx, this, name); + } +} + +/** `common.py:Output`, a thin `CfnOutput` wrapper; the construct id is the name verbatim. */ +export function output( + scope: Construct, + name: string, + props: { value: string; description?: string; exportName?: string }, +): CfnOutput { + return new CfnOutput(scope, name, { + value: props.value, + description: props.description, + exportName: props.exportName, + }); +} diff --git a/source/idea/ideactl/src/cdk/constructs/directory-service.ts b/source/idea/ideactl/src/cdk/constructs/directory-service.ts new file mode 100644 index 00000000..c157ac01 --- /dev/null +++ b/source/idea/ideactl/src/cdk/constructs/directory-service.ts @@ -0,0 +1,363 @@ +/** + * All four classes are plain holders. They create no wrapper resource, and every resource lands + * directly under the caller's scope. + */ + +import { randomUUID } from 'node:crypto'; + +import { Duration, Fn, RemovalPolicy, SecretValue, Tags } from 'aws-cdk-lib'; +import type { Construct } from 'constructs'; +import * as cognito from 'aws-cdk-lib/aws-cognito'; +import * as ds from 'aws-cdk-lib/aws-directoryservice'; +import type * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager'; + +import { IdeaCodeAsset } from '../code-asset.ts'; +import { isEmpty } from '../../config/cluster-config.ts'; +import type { IdeaContext, IdeaNagSuppression } from './base.ts'; +import { IDEA_TAG_MODULE_NAME, addCommonTags, addNagSuppression, resourceName } from './base.ts'; +import { CustomResourceProvider } from './common.ts'; +import { DNSResolverEndpoint, DNSResolverRule } from './dns.ts'; +import type { ExistingSocaCluster } from './existing-resources.ts'; + +/** `constants.MODULE_DIRECTORYSERVICE`. */ +export const MODULE_DIRECTORYSERVICE = 'directoryservice'; +/** `constants.GROUP_TYPE_CLUSTER`. */ +const GROUP_TYPE_CLUSTER = 'cluster'; + +const SECRET_ROTATION_SUPPRESSION = (reason: string): IdeaNagSuppression[] => [ + { rule_id: 'AwsSolutions-SMG4', reason }, +]; + +// --- credentials ------------------------------------------------------------------------------ + +/** + * `DirectoryServiceCredentials`: the root username/password secrets, created once at cluster + * creation. Construct ids are `-admin-username` / `-admin-password`, so the + * provider name is part of the logical ID. When `directoryservice.root_credentials_provided` is + * true nothing is created and the ARNs come from config instead. + * + * The secrets carry only the `idea:ModuleName` tag (access is granted by tag) and **no** removal + * policy, so CloudFormation's default applies. + */ +export class DirectoryServiceCredentials { + readonly ctx: IdeaContext; + readonly credentialsProvided: boolean; + readonly adminUsername: secretsmanager.CfnSecret | undefined; + readonly adminPassword: secretsmanager.CfnSecret | undefined; + + constructor(ctx: IdeaContext, _name: string, scope: Construct, adminUsername: string, adminPassword?: string) { + this.ctx = ctx; + this.credentialsProvided = ctx.config.getBool('directoryservice.root_credentials_provided', false); + if (this.credentialsProvided) return; + + const kmsKeyId = ctx.config.getString('cluster.secretsmanager.kms_key_id'); + const provider = ctx.config.getString('directoryservice.provider', undefined, { + required: true, + }) as string; + + const adminUsernameKey = `${provider}-admin-username`; + this.adminUsername = new secretsmanager.CfnSecret(scope, adminUsernameKey, { + description: `${provider} Root Username, Cluster: ${ctx.clusterName}`, + kmsKeyId, + name: resourceName(ctx, adminUsernameKey), + secretString: adminUsername, + }); + Tags.of(this.adminUsername).add(IDEA_TAG_MODULE_NAME, MODULE_DIRECTORYSERVICE); + + const adminPasswordKey = `${provider}-admin-password`; + this.adminPassword = new secretsmanager.CfnSecret(scope, adminPasswordKey, { + description: `${provider} Root Password, Cluster: ${ctx.clusterName}`, + kmsKeyId, + name: resourceName(ctx, adminPasswordKey), + ...(adminPassword === undefined || adminPassword.trim() === '' + ? { generateSecretString: { excludeCharacters: '$@;"\\\'', passwordLength: 16 } } + : { secretString: adminPassword }), + }); + Tags.of(this.adminPassword).add(IDEA_TAG_MODULE_NAME, MODULE_DIRECTORYSERVICE); + + const suppressions = SECRET_ROTATION_SUPPRESSION( + 'Secret rotation not applicable for DirectoryService credentials.', + ); + addNagSuppression(this.adminUsername, suppressions); + addNagSuppression(this.adminPassword, suppressions); + } + + getUsernameSecretArn(): string { + if (this.credentialsProvided) { + return this.ctx.config.getString('directoryservice.root_username_secret_arn', undefined, { + required: true, + }) as string; + } + return (this.adminUsername as secretsmanager.CfnSecret).ref; + } + + getPasswordSecretArn(): string { + if (this.credentialsProvided) { + return this.ctx.config.getString('directoryservice.root_password_secret_arn', undefined, { + required: true, + }) as string; + } + return (this.adminPassword as secretsmanager.CfnSecret).ref; + } +} + +/** + * `OAuthClientIdAndSecret`: `-client-id` / `-client-secret` secrets, tagged with + * the module name that is allowed to read them. Both get `RemovalPolicy.DESTROY`, which on an L1 + * sets `DeletionPolicy` **and** `UpdateReplacePolicy` to `Delete`. + */ +export class OAuthClientIdAndSecret { + readonly clientId: secretsmanager.CfnSecret; + readonly clientSecret: secretsmanager.CfnSecret; + + constructor( + ctx: IdeaContext, + secretNamePrefix: string, + moduleName: string, + scope: Construct, + clientId: string, + clientSecret: string, + ) { + const kmsKeyId = ctx.config.getString('cluster.secretsmanager.kms_key_id'); + + const clientIdKey = `${secretNamePrefix}-client-id`; + this.clientId = new secretsmanager.CfnSecret(scope, clientIdKey, { + description: `${secretNamePrefix} ClientId, Cluster: ${ctx.clusterName}`, + kmsKeyId, + name: resourceName(ctx, clientIdKey), + secretString: clientId, + }); + this.clientId.applyRemovalPolicy(RemovalPolicy.DESTROY); + Tags.of(this.clientId).add(IDEA_TAG_MODULE_NAME, moduleName); + + const clientSecretKey = `${secretNamePrefix}-client-secret`; + this.clientSecret = new secretsmanager.CfnSecret(scope, clientSecretKey, { + description: `${secretNamePrefix} ClientSecret, Cluster: ${ctx.clusterName}`, + kmsKeyId, + name: resourceName(ctx, clientSecretKey), + secretString: clientSecret, + }); + this.clientSecret.applyRemovalPolicy(RemovalPolicy.DESTROY); + Tags.of(this.clientSecret).add(IDEA_TAG_MODULE_NAME, moduleName); + + const suppressions = SECRET_ROTATION_SUPPRESSION( + 'Secret rotation not applicable for OAuth 2.0 ClientId/Secret', + ); + addNagSuppression(this.clientId, suppressions); + addNagSuppression(this.clientSecret, suppressions); + } +} + +// --- AWS Managed Microsoft AD ----------------------------------------------------------------- + +export interface ActiveDirectoryProps { + cluster: ExistingSocaCluster; + subnets?: ec2.ISubnet[]; + /** Defaults to `false`. */ + enableSso?: boolean; +} + +/** + * `ActiveDirectory`: the credentials pair, the `AWS::DirectoryService::MicrosoftAD` (construct id + * = `name`), and the DNS forwarding chain, a `Custom::ADSecurityGroupId` lookup, an outbound + * resolver endpoint in the *same two* launch subnets, and a FORWARD rule to the AD's first two DNS + * addresses on port `'53'` (a string). + * + * The AD launches into the **first two** subnets of the list, in config order. + */ +export class ActiveDirectory { + readonly ctx: IdeaContext; + readonly name: string; + readonly credentials: DirectoryServiceCredentials; + readonly launchSubnets: string[]; + readonly adName: string; + readonly adShortName: string; + readonly adEdition: string; + readonly ad: ds.CfnMicrosoftAD; + + constructor(ctx: IdeaContext, name: string, scope: Construct, props: ActiveDirectoryProps) { + this.ctx = ctx; + this.name = name; + + this.credentials = new DirectoryServiceCredentials( + ctx, + 'ds-activedirectory-credentials', + scope, + 'Admin', + ); + + this.adName = ctx.config.getString('directoryservice.name', undefined, { required: true }) as string; + this.adShortName = ctx.config.getString('directoryservice.ad_short_name', undefined, { + required: true, + }) as string; + this.adEdition = ctx.config.getString('directoryservice.ad_edition', undefined, { + required: true, + }) as string; + + const subnets = props.subnets ?? props.cluster.privateSubnets; + this.launchSubnets = subnets.slice(0, 2).map((subnet) => subnet.subnetId); + + this.ad = new ds.CfnMicrosoftAD(scope, name, { + name: this.adName, + password: SecretValue.secretsManager(this.credentials.getPasswordSecretArn()).toString(), + vpcSettings: { subnetIds: this.launchSubnets, vpcId: props.cluster.vpc.vpcId }, + edition: this.adEdition, + enableSso: props.enableSso ?? false, + shortName: this.adShortName, + }); + addCommonTags(ctx, this.ad, name); + + this.buildDnsResolver(scope, props.cluster); + } + + private buildDnsResolver(scope: Construct, cluster: ExistingSocaCluster): void { + const getAdSecurityGroupResult = new CustomResourceProvider( + this.ctx, + 'get-ad-security-group-id', + scope, + { + ideaCodeAsset: new IdeaCodeAsset('idea_custom_resource_get_ad_security_group'), + lambdaTimeoutSeconds: 15, + policyTemplateName: 'custom-resource-get-ad-security-group.yml', + resourceType: 'ADSecurityGroupId', + }, + ).invoke(resourceName(this.ctx, this.name), { DirectoryId: this.ad.ref }); + + const endpoint = new DNSResolverEndpoint(this.ctx, this.ctx.clusterName, scope, { + subnetIds: this.launchSubnets, + securityGroupIds: [getAdSecurityGroupResult.getAttString('SecurityGroupId')], + }); + + new DNSResolverRule(this.ctx, this.name, scope, { + domainName: this.adName, + vpc: cluster.vpc, + resolverEndpointId: endpoint.resolverEndpoint.attrResolverEndpointId, + ipAddresses: [Fn.select(0, this.ad.attrDnsIpAddresses), Fn.select(1, this.ad.attrDnsIpAddresses)], + port: '53', + }); + } +} + +// --- Cognito user pool ------------------------------------------------------------------------ + +/** `GroupNameHelper.get_cluster_{administrators,managers}_group`. */ +function clusterGroupName(ctx: IdeaContext, key: string): string { + const groupName = ctx.config.getString(key, undefined, { required: true }) as string; + return groupName.endsWith(`${GROUP_TYPE_CLUSTER}-group`) + ? groupName + : `${groupName}-${GROUP_TYPE_CLUSTER}-group`; +} + +/** + * `UserPool`: the pool, its two cluster-wide groups and its Cognito domain. + * + * Every `if is None` default below is what the live pool contains; the identity-provider + * stack only passes `removalPolicy`, `userInvitation` and `lambdaTriggers`. + * + * Advanced security is disabled, so no `UserPoolAddOns` is emitted regardless of + * `identity-provider.cognito.advanced_security_mode`. + * + * When `identity-provider.cognito.domain_url` is empty the domain prefix is a **fresh uuid** on + * every synth, which replaces the live Cognito domain, the caller is expected to keep the key. + */ +export class UserPool { + readonly userPool: cognito.UserPool; + readonly domain: cognito.UserPoolDomain; + readonly secrets: OAuthClientIdAndSecret[] = []; + + constructor(ctx: IdeaContext, name: string, scope: Construct, props: cognito.UserPoolProps = {}) { + const userPoolName = props.userPoolName ?? `${ctx.clusterName}-user-pool`; + + this.userPool = new cognito.UserPool(scope, userPoolName, { + accountRecovery: props.accountRecovery ?? cognito.AccountRecovery.EMAIL_ONLY, + autoVerify: props.autoVerify ?? { email: true, phone: false }, + customAttributes: props.customAttributes ?? { + cluster_name: new cognito.StringAttribute({ mutable: true }), + aws_region: new cognito.StringAttribute({ mutable: true }), + password_last_set: new cognito.NumberAttribute({ mutable: true }), + password_max_age: new cognito.NumberAttribute({ mutable: true }), + }, + customSenderKmsKey: props.customSenderKmsKey, + deletionProtection: true, + deviceTracking: props.deviceTracking, + email: props.email, + enableSmsRole: props.enableSmsRole, + lambdaTriggers: props.lambdaTriggers, + mfa: props.mfa ?? cognito.Mfa.OPTIONAL, + mfaMessage: props.mfaMessage, + mfaSecondFactor: props.mfaSecondFactor ?? { otp: true, sms: false }, + passwordPolicy: props.passwordPolicy ?? { + minLength: 8, + requireDigits: true, + requireLowercase: true, + requireSymbols: true, + requireUppercase: true, + tempPasswordValidity: Duration.days(7), + }, + removalPolicy: props.removalPolicy ?? RemovalPolicy.DESTROY, + selfSignUpEnabled: props.selfSignUpEnabled ?? false, + signInAliases: props.signInAliases ?? { + username: true, + preferredUsername: false, + phone: false, + email: true, + }, + signInCaseSensitive: props.signInCaseSensitive ?? false, + smsRole: props.smsRole, + smsRoleExternalId: props.smsRoleExternalId, + standardAttributes: props.standardAttributes ?? { + email: { mutable: true, required: true }, + }, + userInvitation: props.userInvitation ?? { + emailSubject: `(${ctx.clusterName}) Your IDEA Account`, + emailBody: + '\n Hello {username},\n' + + '

\n' + + ` You have been invited to join the ${ctx.clusterName} cluster.\n` + + '
\n' + + ' Your temporary password is {####}\n' + + ' ', + }, + userPoolName, + userVerification: props.userVerification, + }); + addCommonTags(ctx, this.userPool, name); + + addNagSuppression(this.userPool, [ + { + rule_id: 'AwsSolutions-COG2', + reason: 'Suppress MFA warning. MFA provided by customer IdP/SSO methods.', + }, + ]); + addNagSuppression(this.userPool, [ + { + rule_id: 'AwsSolutions-COG3', + reason: 'suppress advanced security rule 1/to save cost, 2/Not supported in GovCloud', + }, + ]); + + new cognito.CfnUserPoolGroup(scope, `${userPoolName}-administrators-group`, { + description: 'Administrators group (Sudo Users)', + groupName: clusterGroupName(ctx, 'identity-provider.cognito.administrators_group_name'), + precedence: 1, + userPoolId: this.userPool.userPoolId, + }); + + new cognito.CfnUserPoolGroup(scope, `${userPoolName}-managers-group`, { + description: 'Managers group with limited administration access.', + groupName: clusterGroupName(ctx, 'identity-provider.cognito.managers_group_name'), + precedence: 2, + userPoolId: this.userPool.userPoolId, + }); + + const domainUrl = ctx.config.getString('identity-provider.cognito.domain_url'); + const domainPrefix = isEmpty(domainUrl) + ? `${ctx.clusterName}-${randomUUID()}` + : ((domainUrl as string).replace('https://', '').split('.')[0] as string); + + this.domain = this.userPool.addDomain('domain', { + cognitoDomain: { domainPrefix }, + }); + } +} diff --git a/source/idea/ideactl/src/cdk/constructs/dns.ts b/source/idea/ideactl/src/cdk/constructs/dns.ts new file mode 100644 index 00000000..faac3014 --- /dev/null +++ b/source/idea/ideactl/src/cdk/constructs/dns.ts @@ -0,0 +1,101 @@ +/** + * `DNSResolverEndpoint` and `DNSResolverRule` are plain holders that build L1 resources directly + * under the caller's scope. `PrivateHostedZone` is a CDK construct. + */ + +import type { Construct } from 'constructs'; +import type * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as route53 from 'aws-cdk-lib/aws-route53'; +import * as route53resolver from 'aws-cdk-lib/aws-route53resolver'; + +import type { IdeaContext } from './base.ts'; +import { addCommonTags } from './base.ts'; + +export interface DNSResolverEndpointProps { + securityGroupIds: string[]; + subnetIds: string[]; + /** Defaults to `OUTBOUND`. */ + direction?: string; +} + +/** + * `dns.py:DNSResolverEndpoint`. Construct id `-dns-resolver-endpoint`; one `IpAddresses` + * entry per subnet id, carrying the subnet only (the resolver picks the address). + */ +export class DNSResolverEndpoint { + readonly resolverEndpoint: route53resolver.CfnResolverEndpoint; + + constructor(ctx: IdeaContext, name: string, scope: Construct, props: DNSResolverEndpointProps) { + this.resolverEndpoint = new route53resolver.CfnResolverEndpoint(scope, `${name}-dns-resolver-endpoint`, { + direction: props.direction ?? 'OUTBOUND', + name, + ipAddresses: props.subnetIds.map((subnetId) => ({ subnetId })), + securityGroupIds: props.securityGroupIds, + }); + addCommonTags(ctx, this.resolverEndpoint, name); + } +} + +export interface DNSResolverRuleProps { + domainName: string; + vpc: ec2.IVpc; + resolverEndpointId: string; + ipAddresses: string[]; + /** Defaults to `FORWARD`. */ + ruleType?: string; + /** The AD caller passes the string `'53'`. */ + port?: string; +} + +/** + * `dns.py:DNSResolverRule`: the forward rule plus its VPC association. Construct ids + * `-dns-resolver-rule` and `-dns-resolver-rule-association`; the rule's `Name` + * property is the same `-dns-resolver-rule` string as its construct id. + * + * `add_common_tags` is called on the association too, but `AWS::Route53Resolver::ResolverRuleAssociation` + * has no tag property, so the aspect drops them and the live resource carries none. + */ +export class DNSResolverRule { + readonly resolverRule: route53resolver.CfnResolverRule; + readonly resolverRuleAssoc: route53resolver.CfnResolverRuleAssociation; + + constructor(ctx: IdeaContext, name: string, scope: Construct, props: DNSResolverRuleProps) { + this.resolverRule = new route53resolver.CfnResolverRule(scope, `${name}-dns-resolver-rule`, { + name: `${name}-dns-resolver-rule`, + domainName: props.domainName, + ruleType: props.ruleType ?? 'FORWARD', + resolverEndpointId: props.resolverEndpointId, + targetIps: props.ipAddresses.map((ip) => ({ ip, port: props.port })), + }); + addCommonTags(ctx, this.resolverRule, name); + + this.resolverRuleAssoc = new route53resolver.CfnResolverRuleAssociation( + scope, + `${name}-dns-resolver-rule-association`, + { + resolverRuleId: this.resolverRule.attrResolverRuleId, + vpcId: props.vpc.vpcId, + }, + ); + addCommonTags(ctx, this.resolverRuleAssoc, name); + } +} + +/** + * Construct id and tag name are both `-private-hosted-zone`, + * so `build_resource_name` prepends the cluster a second time and the `Name` tag reads + * `--private-hosted-zone`. Reproduce the doubled prefix. + */ +export class PrivateHostedZone extends route53.PrivateHostedZone { + constructor(ctx: IdeaContext, scope: Construct, vpc: ec2.IVpc) { + const name = `${ctx.clusterName}-private-hosted-zone`; + super(scope, name, { + vpc, + comment: `Private Hosted Zone for IDEA Cluster: ${ctx.clusterName}`, + zoneName: ctx.config.getString('cluster.route53.private_hosted_zone_name', undefined, { + required: true, + }) as string, + }); + addCommonTags(ctx, this, name); + } +} diff --git a/source/idea/ideactl/src/cdk/constructs/existing-resources.ts b/source/idea/ideactl/src/cdk/constructs/existing-resources.ts new file mode 100644 index 00000000..8c94b39e --- /dev/null +++ b/source/idea/ideactl/src/cdk/constructs/existing-resources.ts @@ -0,0 +1,209 @@ +/** + * Creates no resources: it imports the VPC (a context lookup resolved from `cdk.context.json`), + * the cluster IAM roles and the cluster security groups under the construct ids every other + * stack's logical IDs are hashed against (`vpc`, `-role`, `-security-group`). + */ + +import type { Construct } from 'constructs'; +import * as backup from 'aws-cdk-lib/aws-backup'; +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as kms from 'aws-cdk-lib/aws-kms'; +import * as opensearch from 'aws-cdk-lib/aws-opensearchservice'; +import * as route53 from 'aws-cdk-lib/aws-route53'; +import * as s3 from 'aws-cdk-lib/aws-s3'; +import * as sns from 'aws-cdk-lib/aws-sns'; + +import type { IdeaContext } from './base.ts'; +import { kmsKeyArn } from './base.ts'; + +/** + * `ExistingVpc`: the looked-up VPC plus the subnets named in + * `cluster.network.{public,private}_subnets`, ordered by the config list. + */ +export class ExistingVpc { + readonly ctx: IdeaContext; + readonly scope: Construct; + readonly vpcId: string; + vpc: ec2.IVpc; + private cachedPrivateSubnets: ec2.ISubnet[] | undefined; + private cachedPublicSubnets: ec2.ISubnet[] | undefined; + + constructor(ctx: IdeaContext, _name: string, scope: Construct) { + this.ctx = ctx; + this.scope = scope; + this.vpcId = ctx.config.getString('cluster.network.vpc_id', undefined, { required: true }) as string; + this.vpc = ec2.Vpc.fromLookup(this.scope, 'vpc', { vpcId: this.vpcId }); + } + + getPublicSubnetIds(): string[] { + return this.ctx.config.getList('cluster.network.public_subnets', []); + } + + getPrivateSubnetIds(): string[] { + return this.ctx.config.getList('cluster.network.private_subnets', []); + } + + getPublicSubnets(): ec2.ISubnet[] { + if (this.cachedPublicSubnets !== undefined) return this.cachedPublicSubnets; + const ids = this.getPublicSubnetIds(); + this.cachedPublicSubnets = + ids.length === 0 ? [] : orderByConfig(this.vpc.publicSubnets ?? [], ids); + return this.cachedPublicSubnets; + } + + /** + * Both `privateSubnets` and `isolatedSubnets` are searched: CDK buckets a subnet without a NAT + * gateway as isolated, and IDEA calls both "private". Order follows the config list. + */ + getPrivateSubnets(): ec2.ISubnet[] { + if (this.cachedPrivateSubnets !== undefined) return this.cachedPrivateSubnets; + const ids = this.getPrivateSubnetIds(); + if (ids.length === 0) { + this.cachedPrivateSubnets = []; + return this.cachedPrivateSubnets; + } + this.cachedPrivateSubnets = orderByConfig( + [...(this.vpc.privateSubnets ?? []), ...(this.vpc.isolatedSubnets ?? [])], + ids, + ); + return this.cachedPrivateSubnets; + } +} + +function orderByConfig(subnets: ec2.ISubnet[], ids: string[]): ec2.ISubnet[] { + return subnets + .filter((subnet) => ids.includes(subnet.subnetId)) + .sort((a, b) => ids.indexOf(a.subnetId) - ids.indexOf(b.subnetId)); +} + +/** `ExistingSocaCluster`: the VPC, the `cluster.iam.roles` and the `cluster.network.security_groups`. */ +export class ExistingSocaCluster { + readonly ctx: IdeaContext; + readonly scope: Construct; + readonly existingVpc: ExistingVpc; + readonly securityGroups: Record; + readonly roles: Record; + + constructor(ctx: IdeaContext, scope: Construct) { + this.ctx = ctx; + this.scope = scope; + this.existingVpc = new ExistingVpc(ctx, 'existing-vpc', scope); + this.securityGroups = {}; + this.roles = {}; + this.lookupRoles(); + this.lookupSecurityGroups(); + } + + get vpc(): ec2.IVpc { + return this.existingVpc.vpc; + } + + get publicSubnets(): ec2.ISubnet[] { + return this.existingVpc.getPublicSubnets(); + } + + get privateSubnets(): ec2.ISubnet[] { + return this.existingVpc.getPrivateSubnets(); + } + + private lookupRoles(): void { + const roles = this.ctx.config.getConfig('cluster.iam.roles', undefined, { required: true }) ?? {}; + for (const [name, arn] of Object.entries(roles)) { + this.roles[name] = iam.Role.fromRoleArn(this.scope, `${name}-role`, String(arn)); + } + } + + getRole(name: string): iam.IRole | undefined { + return this.roles[name]; + } + + private lookupSecurityGroups(): void { + const securityGroups = + this.ctx.config.getConfig('cluster.network.security_groups', undefined, { required: true }) ?? {}; + for (const [name, id] of Object.entries(securityGroups)) { + this.securityGroups[name] = ec2.SecurityGroup.fromSecurityGroupId( + this.scope, + `${name}-security-group`, + String(id), + ); + } + } + + getSecurityGroup(name: string): ec2.ISecurityGroup | undefined { + return this.securityGroups[name]; + } +} + +// --- Per-stack imports -------------------------------------------------------------------------- +// These create no resources, but their construct ids feed `Names.uniqueId` for dependent resources. + +function required(ctx: IdeaContext, key: string): string { + return ctx.config.getString(key, undefined, { required: true }) as string; +} + +/** `route53.HostedZone.from_hosted_zone_attributes(stack, 'cluster-dns', ...)`. */ +export function lookupClusterDns(ctx: IdeaContext, scope: Construct): route53.IHostedZone { + return route53.HostedZone.fromHostedZoneAttributes(scope, 'cluster-dns', { + hostedZoneId: required(ctx, 'cluster.route53.private_hosted_zone_id'), + zoneName: required(ctx, 'cluster.route53.private_hosted_zone_name'), + }); +} + +/** + * `cluster.ebs.kms_key_id` if set, else the account's `alias/aws/ebs`. VDC prefixes the id with + * the component name (`-ebs-kms-key(-default)`), every other stack does not. + */ +export function lookupEbsKmsKey(ctx: IdeaContext, scope: Construct, componentName?: string): kms.IKey { + const prefix = componentName === undefined ? '' : `${componentName}-`; + const kmsKeyId = ctx.config.getString('cluster.ebs.kms_key_id'); + return kmsKeyId === undefined + ? kms.Alias.fromAliasName(scope, `${prefix}ebs-kms-key-default`, 'alias/aws/ebs') + : kms.Key.fromKeyArn(scope, `${prefix}ebs-kms-key`, kmsKeyArn(ctx, kmsKeyId)); +} + +/** `ec2.KeyPair.from_key_pair_name(stack, f'{module_id}-key-pair' | f'{component}-key-pair', ...)`. */ +export function lookupKeyPair(ctx: IdeaContext, scope: Construct, id?: string): ec2.IKeyPair { + return ec2.KeyPair.fromKeyPairName( + scope, + id ?? `${ctx.moduleId}-key-pair`, + required(ctx, 'cluster.network.ssh_key_pair'), + ); +} + +/** `s3.Bucket.from_bucket_name(stack, 'cluster-s3-bucket', cluster.cluster_s3_bucket)`. */ +export function lookupClusterS3Bucket(ctx: IdeaContext, scope: Construct): s3.IBucket { + return s3.Bucket.fromBucketName(scope, 'cluster-s3-bucket', required(ctx, 'cluster.cluster_s3_bucket')); +} + +/** `opensearch.Domain.from_domain_endpoint(stack, 'existing-opensearch', https://)`. */ +export function lookupExistingOpensearch(ctx: IdeaContext, scope: Construct): opensearch.IDomain { + return opensearch.Domain.fromDomainEndpoint( + scope, + 'existing-opensearch', + `https://${required(ctx, 'analytics.opensearch.domain_vpc_endpoint_url')}`, + ); +} + +/** `sns.Topic.from_topic_arn(stack, f'{cluster}-{module_id}-ec2-state-change-topic', ...)`. */ +export function lookupEc2StateChangeTopic(ctx: IdeaContext, scope: Construct): sns.ITopic { + return sns.Topic.fromTopicArn( + scope, + `${ctx.clusterName}-${ctx.moduleId}-ec2-state-change-topic`, + required(ctx, 'cluster.ec2.state_change_notifications_sns_topic_arn'), + ); +} + +/** `iam.Role.from_role_arn(stack, 'backup-role', cluster.backups.role_arn)`. */ +export function lookupBackupRole(ctx: IdeaContext, scope: Construct): iam.IRole { + return iam.Role.fromRoleArn(scope, 'backup-role', required(ctx, 'cluster.backups.role_arn')); +} + +/** `backup.BackupVault.from_backup_vault_arn(stack, 'cluster-backup-vault', ...)`. */ +export function lookupClusterBackupVault(ctx: IdeaContext, scope: Construct): backup.IBackupVault { + return backup.BackupVault.fromBackupVaultArn( + scope, + 'cluster-backup-vault', + required(ctx, 'cluster.backups.backup_vault.arn'), + ); +} diff --git a/source/idea/ideactl/src/cdk/constructs/network.ts b/source/idea/ideactl/src/cdk/constructs/network.ts new file mode 100644 index 00000000..a5d25d90 --- /dev/null +++ b/source/idea/ideactl/src/cdk/constructs/network.ts @@ -0,0 +1,954 @@ +/** + * Security-group rule order and peer kinds determine template shape. CIDR peers, including token + * CIDRs, are inlined into the + * group's `SecurityGroupIngress`/`SecurityGroupEgress` lists, while prefix-list and security-group + * peers become separate `AWS::EC2::SecurityGroupIngress`/`Egress` resources whose construct id + * embeds the peer - a literal prefix-list id or the peer group's unique id verbatim, a token peer + * as `{IndirectPeer}` and then `'{IndirectPeer2}'`, `'{IndirectPeer3}'`, counted per group in + * call order (`SecurityGroupBase.renderPeer`). `allowAllOutbound` is false by default, so the + * explicit egress list renders instead of the L2's allow-all rule. + */ + +import { Fn, RemovalPolicy, Tags } from 'aws-cdk-lib'; +import type { Construct, IConstruct } from 'constructs'; +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as logs from 'aws-cdk-lib/aws-logs'; +import * as wafv2 from 'aws-cdk-lib/aws-wafv2'; + +import { ConfigKeyNotFound, isEmpty } from '../../config/cluster-config.ts'; +import type { IdeaContext, IdeaNagSuppression } from './base.ts'; +import { + IDEA_TAG_CLUSTER_NAME, + IDEA_TAG_NAME, + addCommonTags, + addNagSuppression as applyNagSuppression, + buildServicePrincipal, + constructId, + isDsActivedirectory, + resourceName, +} from './base.ts'; +import { LOG_RETENTION_DAYS, type CreateTagsCustomResource } from './common.ts'; + +/** `RemovalPolicy` lookup uses member names, not enum values. */ +function removalPolicyByName(name: string): RemovalPolicy { + if (!Object.prototype.hasOwnProperty.call(RemovalPolicy, name)) { + throw new Error(`'${name}' is not a valid RemovalPolicy`); + } + return RemovalPolicy[name as keyof typeof RemovalPolicy]; +} + +/** `config.get_int(key, required=True)`: a missing or NULL value raises. */ +function requiredInt(ctx: IdeaContext, key: string): number { + const value = ctx.config.getInt(key); + if (value === undefined) throw new ConfigKeyNotFound(`'${key}', key: ${key}`); + return value; +} + +export class ElasticIP extends ec2.CfnEIP { + constructor(ctx: IdeaContext, name: string, scope: Construct) { + super(scope, constructId(name)); + addCommonTags(ctx, this, name); + } +} + +// --- VPC -------------------------------------------------------------------------------------- + +/** + * The log group and role live at the stack scope and are created before the VPC. + */ +function buildFlowLogs(ctx: IdeaContext, scope: Construct): Record | undefined { + if (!ctx.config.getBool('cluster.network.vpc_flow_logs', false)) return undefined; + + const removalPolicy = ctx.config.getString('cluster.network.vpc_flow_logs_removal_policy', 'DESTROY'); + const logGroupName = ctx.config.getString( + 'cluster.network.vpc_flow_logs_group_name', + `${ctx.clusterName}-vpc-flow-logs`, + ); + const logGroup = new logs.LogGroup(scope, 'vpc-flow-logs-group', { + logGroupName, + removalPolicy: removalPolicyByName(removalPolicy), + }); + const iamRole = new iam.Role(scope, 'vpc-flow-logs-role', { + assumedBy: buildServicePrincipal('vpc-flow-logs'), + description: `IAM Role for VPC Flow Logs, Cluster: ${ctx.clusterName}`, + roleName: `${ctx.clusterName}-vpc-flow-logs-${ctx.awsRegion}`, + }); + return { + 'cloud-watch': { + destination: ec2.FlowLogDestination.toCloudWatchLogs(logGroup, iamRole), + trafficType: ec2.FlowLogTrafficType.ALL, + }, + }; +} + +/** `Vpc.build_subnet_configuration`: public always, private always, isolated only when configured. */ +function buildSubnetConfiguration(ctx: IdeaContext): ec2.SubnetConfiguration[] { + const result: ec2.SubnetConfiguration[] = [ + { + name: 'public', + cidrMask: ctx.config.getInt('cluster.network.subnet_config.public.cidr_mask', 26), + subnetType: ec2.SubnetType.PUBLIC, + }, + { + name: 'private', + cidrMask: ctx.config.getInt('cluster.network.subnet_config.private.cidr_mask', 18), + subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS, + }, + ]; + const isolatedCidrMask = ctx.config.getInt('cluster.network.subnet_config.isolated.cidr_mask'); + if (isolatedCidrMask !== undefined) { + result.push({ name: 'isolated', cidrMask: isolatedCidrMask, subnetType: ec2.SubnetType.PRIVATE_ISOLATED }); + } + return result; +} + +export class Vpc extends ec2.Vpc { + readonly ctx: IdeaContext; + + constructor(ctx: IdeaContext, name: string, scope: Construct) { + super(scope, constructId(name), { + ipAddresses: ec2.IpAddresses.cidr(ctx.config.getString('cluster.network.vpc_cidr_block') as string), + natGateways: ctx.config.getInt('cluster.network.nat_gateways'), + enableDnsSupport: true, + enableDnsHostnames: true, + maxAzs: ctx.config.getInt('cluster.network.max_azs'), + subnetConfiguration: buildSubnetConfiguration(ctx), + flowLogs: buildFlowLogs(ctx, scope), + }); + this.ctx = ctx; + addCommonTags(ctx, this, name); + } + + /** The `EIP` child of every public subnet that has one (one per NAT gateway). */ + get natGatewayIps(): ec2.CfnEIP[] { + const result: ec2.CfnEIP[] = []; + for (const subnet of this.publicSubnets) { + const eip = subnet.node.tryFindChild('EIP'); + if (eip === undefined) continue; + result.push(eip as ec2.CfnEIP); + } + return result; + } + + get publicSubnetIds(): string[] { + return this.publicSubnets.map((subnet) => subnet.subnetId); + } + + get privateSubnetIds(): string[] { + return this.privateSubnets.map((subnet) => subnet.subnetId); + } +} + +// --- Security groups -------------------------------------------------------------------------- + +export class SecurityGroup extends ec2.SecurityGroup { + readonly ctx: IdeaContext; + readonly vpc: ec2.IVpc; + + constructor( + ctx: IdeaContext, + name: string, + scope: Construct, + vpc: ec2.IVpc, + description: string, + allowAllOutbound = false, + ) { + super(scope, constructId(name), { + securityGroupName: resourceName(ctx, name), + vpc, + allowAllOutbound, + description, + }); + this.ctx = ctx; + this.vpc = vpc; + addCommonTags(ctx, this, name); + this.addNagSuppression([]); + } + + /** + * `AwsSolutions-EC23` is prepended to every suppression. At construction time it applies only + * to the `Resource` child, so later rules carry no metadata. + */ + addNagSuppression(suppressions: IdeaNagSuppression[], construct?: IConstruct, applyToChildren = true): void { + const updated: IdeaNagSuppression[] = [ + { rule_id: 'AwsSolutions-EC23', reason: 'suppress warning: parameter referencing intrinsic function' }, + ...suppressions, + ]; + applyNagSuppression(construct ?? this, updated, applyToChildren); + } + + addOutboundTrafficRule(): void { + this.addEgressRule(ec2.Peer.ipv4('0.0.0.0/0'), ec2.Port.tcpRange(0, 65535), 'Allow all egress for TCP'); + this.addEgressRule(ec2.Peer.ipv6('::/0'), ec2.Port.tcpRange(0, 65535), 'Allow all egress for TCP'); + } + + addApiIngressRule(): void { + this.addIngressRule( + ec2.Peer.ipv4(this.vpc.vpcCidrBlock), + ec2.Port.tcp(8443), + 'Allow HTTP traffic from all VPC nodes for API access', + ); + } + + addLoadbalancerIngressRule(loadbalancerSecurityGroup: ec2.ISecurityGroup): void { + this.addIngressRule(loadbalancerSecurityGroup, ec2.Port.tcp(8443), 'Allow HTTPs traffic from Load Balancer'); + } + + addBastionHostIngressRule(bastionHostSecurityGroup: ec2.ISecurityGroup): void { + this.addIngressRule(bastionHostSecurityGroup, ec2.Port.tcp(22), 'Allow SSH from Bastion Host'); + } + + addActiveDirectoryRules(): void { + this.addIngressRule( + ec2.Peer.ipv4(this.vpc.vpcCidrBlock), + ec2.Port.udpRange(0, 1024), + 'Allow UDP Traffic from VPC. Required for Directory Service', + ); + this.addEgressRule( + ec2.Peer.ipv4('0.0.0.0/0'), + ec2.Port.udpRange(0, 1024), + 'Allow UDP Traffic. Required for Directory Service', + ); + this.addEgressRule( + ec2.Peer.ipv6('::/0'), + ec2.Port.udpRange(0, 1024), + 'Allow UDP Traffic. Required for Directory Service', + ); + } +} + +/** The only group in a public subnet. */ +export class BastionHostSecurityGroup extends SecurityGroup { + readonly clusterPrefixListId: string; + + constructor(ctx: IdeaContext, name: string, scope: Construct, vpc: ec2.IVpc, clusterPrefixListId: string) { + super(ctx, name, scope, vpc, 'Bastion host security group'); + this.clusterPrefixListId = clusterPrefixListId; + this.setupIngress(); + this.setupEgress(); + + if (isDsActivedirectory(ctx)) { + this.addActiveDirectoryRules(); + } + } + + setupIngress(): void { + this.addIngressRule( + ec2.Peer.prefixList(this.clusterPrefixListId), + ec2.Port.tcp(22), + 'Allow SSH access from Cluster Prefix List to Bastion Host', + ); + + // entries are not checked for emptiness here, unlike the external load balancer's + for (const prefixListId of this.ctx.config.getList('cluster.network.prefix_list_ids', [])) { + this.addIngressRule( + ec2.Peer.prefixList(prefixListId), + ec2.Port.tcp(22), + 'Allow SSH access from Prefix List to Bastion Host', + ); + } + + this.addIngressRule(ec2.Peer.ipv4(this.vpc.vpcCidrBlock), ec2.Port.tcp(22), 'Allow SSH traffic from all VPC nodes'); + } + + setupEgress(): void { + this.addOutboundTrafficRule(); + } +} + +export class ExternalLoadBalancerSecurityGroup extends SecurityGroup { + readonly clusterPrefixListId: string; + readonly bastionHostSecurityGroup: ec2.ISecurityGroup; + + constructor( + ctx: IdeaContext, + name: string, + scope: Construct, + vpc: ec2.IVpc, + clusterPrefixListId: string, + bastionHostSecurityGroup: ec2.ISecurityGroup, + ) { + super(ctx, name, scope, vpc, 'External Application Load Balancer security group'); + this.clusterPrefixListId = clusterPrefixListId; + this.bastionHostSecurityGroup = bastionHostSecurityGroup; + this.setupIngress(); + this.setupEgress(); + } + + addPeerIngressRule(peer: ec2.IPeer, peerType: string): void { + this.addIngressRule(peer, ec2.Port.tcp(443), `Allow HTTPS access from ${peerType} to ALB`); + this.addIngressRule(peer, ec2.Port.tcp(80), `Allow HTTP access from ${peerType} to ALB`); + } + + setupIngress(): void { + // two rules on one token peer: `{IndirectPeer}` for 443, then `'{IndirectPeer2}'` for 80 + this.addPeerIngressRule(ec2.Peer.prefixList(this.clusterPrefixListId), 'Cluster Prefix List'); + + for (const prefixListId of this.ctx.config.getList('cluster.network.prefix_list_ids', [])) { + if (!isEmpty(prefixListId)) { + this.addPeerIngressRule(ec2.Peer.prefixList(prefixListId), 'Prefix List'); + } + } + + this.addIngressRule(this.bastionHostSecurityGroup, ec2.Port.tcp(80), 'Allow HTTP from Bastion Host'); + this.addIngressRule(this.bastionHostSecurityGroup, ec2.Port.tcp(443), 'Allow HTTPs from Bastion Host'); + } + + setupEgress(): void { + this.addOutboundTrafficRule(); + } + + /** + * One inline `/32` rule per NAT EIP, so instances behind the NAT (virtual desktops, private + * subnets) can reach the web portal and the APIs through the public ALB endpoint. + */ + addNatGatewayIpsIngressRule(natGatewayIps: ec2.CfnEIP[]): void { + for (const eip of natGatewayIps) { + this.addIngressRule(ec2.Peer.ipv4(`${eip.ref}/32`), ec2.Port.tcp(443), 'Allow NAT EIP to communicate to ALB.'); + } + } +} + +/** + * Attached to EFS and FSx file systems; open to every node in the VPC. Lustre rules are always + * provisioned because compute nodes can mount FSx Lustre on demand for /scratch. + */ +export class SharedStorageSecurityGroup extends SecurityGroup { + constructor(ctx: IdeaContext, name: string, scope: Construct, vpc: ec2.IVpc) { + super(ctx, name, scope, vpc, 'Shared Storage security group for EFS/FSx file systems'); + this.setupIngress(); + this.setupEgress(); + } + + setupIngress(): void { + const vpcCidr = ec2.Peer.ipv4(this.vpc.vpcCidrBlock); + // NFS + this.addIngressRule(vpcCidr, ec2.Port.tcp(2049), 'Allow NFS traffic from all VPC nodes to EFS'); + // FSx for Lustre + this.addIngressRule(vpcCidr, ec2.Port.tcp(988), 'Allow FSx Lustre traffic from all VPC nodes'); + this.addIngressRule(vpcCidr, ec2.Port.tcpRange(1021, 1023), 'Allow FSx Lustre traffic from all VPC nodes'); + } + + setupEgress(): void { + this.addOutboundTrafficRule(); + } +} + +export class OpenLDAPServerSecurityGroup extends SecurityGroup { + readonly bastionHostSecurityGroup: ec2.ISecurityGroup; + + constructor( + ctx: IdeaContext, + name: string, + scope: Construct, + vpc: ec2.IVpc, + bastionHostSecurityGroup: ec2.ISecurityGroup, + ) { + super(ctx, name, scope, vpc, 'OpenLDAP server security group'); + this.bastionHostSecurityGroup = bastionHostSecurityGroup; + this.setupIngress(); + this.setupEgress(); + } + + setupIngress(): void { + this.addIngressRule(ec2.Peer.ipv4(this.vpc.vpcCidrBlock), ec2.Port.tcp(389), 'Allow LDAP traffic from all VPC nodes'); + this.addApiIngressRule(); + this.addBastionHostIngressRule(this.bastionHostSecurityGroup); + } + + setupEgress(): void { + this.addOutboundTrafficRule(); + } +} + +/** The cluster-manager group. */ +export class WebPortalSecurityGroup extends SecurityGroup { + readonly bastionHostSecurityGroup: ec2.ISecurityGroup; + readonly loadbalancerSecurityGroup: ec2.ISecurityGroup; + + constructor( + ctx: IdeaContext, + name: string, + scope: Construct, + vpc: ec2.IVpc, + bastionHostSecurityGroup: ec2.ISecurityGroup, + loadbalancerSecurityGroup: ec2.ISecurityGroup, + ) { + super(ctx, name, scope, vpc, 'Web Portal security group'); + this.bastionHostSecurityGroup = bastionHostSecurityGroup; + this.loadbalancerSecurityGroup = loadbalancerSecurityGroup; + this.setupIngress(); + this.setupEgress(); + if (isDsActivedirectory(ctx)) { + this.addActiveDirectoryRules(); + } + } + + setupIngress(): void { + this.addApiIngressRule(); + this.addBastionHostIngressRule(this.bastionHostSecurityGroup); + this.addLoadbalancerIngressRule(this.loadbalancerSecurityGroup); + } + + setupEgress(): void { + this.addOutboundTrafficRule(); + } +} + +export class SchedulerSecurityGroup extends SecurityGroup { + readonly bastionHostSecurityGroup: ec2.ISecurityGroup; + readonly loadbalancerSecurityGroup: ec2.ISecurityGroup; + + constructor( + ctx: IdeaContext, + name: string, + scope: Construct, + vpc: ec2.IVpc, + bastionHostSecurityGroup: ec2.ISecurityGroup, + loadbalancerSecurityGroup: ec2.ISecurityGroup, + ) { + super(ctx, name, scope, vpc, 'Scheduler security group'); + this.bastionHostSecurityGroup = bastionHostSecurityGroup; + this.loadbalancerSecurityGroup = loadbalancerSecurityGroup; + this.setupIngress(); + this.setupEgress(); + if (isDsActivedirectory(ctx)) { + this.addActiveDirectoryRules(); + } + } + + setupIngress(): void { + this.addApiIngressRule(); + this.addIngressRule( + ec2.Peer.ipv4(this.vpc.vpcCidrBlock), + ec2.Port.tcpRange(0, 65535), + 'Allow all TCP traffic from VPC to scheduler', + ); + this.addBastionHostIngressRule(this.bastionHostSecurityGroup); + this.addLoadbalancerIngressRule(this.loadbalancerSecurityGroup); + } + + setupEgress(): void { + this.addOutboundTrafficRule(); + } +} + +export class ComputeNodeSecurityGroup extends SecurityGroup { + constructor(ctx: IdeaContext, name: string, scope: Construct, vpc: ec2.IVpc) { + super(ctx, name, scope, vpc, 'Compute Node security group'); + this.setupIngress(); + this.setupEgress(); + if (isDsActivedirectory(ctx)) { + this.addActiveDirectoryRules(); + } + } + + setupIngress(): void { + this.addIngressRule( + ec2.Peer.ipv4(this.vpc.vpcCidrBlock), + ec2.Port.tcpRange(0, 65535), + 'All TCP traffic from all VPC nodes to compute node', + ); + // a self peer is never inlined: `from :ALL TRAFFIC` + this.addIngressRule(this, ec2.Port.allTraffic(), 'Allow all traffic between compute nodes and EFA'); + } + + setupEgress(): void { + this.addOutboundTrafficRule(); + this.addEgressRule(this, ec2.Port.allTraffic(), 'Allow all traffic between compute nodes and EFA'); + } +} + +export interface VirtualDesktopBastionAccessSecurityGroupProps { + bastionHostSecurityGroup: ec2.ISecurityGroup; + description: string; + directoryServiceAccess: boolean; + componentName: string; +} + +/** Virtual desktop group with bastion access. */ +export class VirtualDesktopBastionAccessSecurityGroup extends SecurityGroup { + readonly componentName: string; + readonly bastionHostSecurityGroup: ec2.ISecurityGroup; + + constructor( + ctx: IdeaContext, + name: string, + scope: Construct, + vpc: ec2.IVpc, + props: VirtualDesktopBastionAccessSecurityGroupProps, + ) { + super(ctx, name, scope, vpc, props.description); + this.componentName = props.componentName; + this.bastionHostSecurityGroup = props.bastionHostSecurityGroup; + this.setupIngress(); + this.setupEgress(); + if (props.directoryServiceAccess && isDsActivedirectory(ctx)) { + this.addActiveDirectoryRules(); + } + } + + setupIngress(): void { + this.addApiIngressRule(); + this.addIngressRule( + ec2.Peer.ipv4(this.vpc.vpcCidrBlock), + ec2.Port.allTraffic(), + `Allow all Internal traffic TO ${this.componentName}`, + ); + this.addBastionHostIngressRule(this.bastionHostSecurityGroup); + } + + setupEgress(): void { + this.addOutboundTrafficRule(); + } +} + +export interface VirtualDesktopPublicLoadBalancerAccessSecurityGroupProps + extends VirtualDesktopBastionAccessSecurityGroupProps { + publicLoadbalancerSecurityGroup: ec2.ISecurityGroup; +} + +/** Virtual desktop group with bastion and public load balancer access. */ +export class VirtualDesktopPublicLoadBalancerAccessSecurityGroup extends SecurityGroup { + readonly componentName: string; + readonly publicLoadbalancerSecurityGroup: ec2.ISecurityGroup; + readonly bastionHostSecurityGroup: ec2.ISecurityGroup; + + constructor( + ctx: IdeaContext, + name: string, + scope: Construct, + vpc: ec2.IVpc, + props: VirtualDesktopPublicLoadBalancerAccessSecurityGroupProps, + ) { + super(ctx, name, scope, vpc, props.description); + this.componentName = props.componentName; + this.publicLoadbalancerSecurityGroup = props.publicLoadbalancerSecurityGroup; + this.bastionHostSecurityGroup = props.bastionHostSecurityGroup; + this.setupIngress(); + this.setupEgress(); + if (props.directoryServiceAccess && isDsActivedirectory(ctx)) { + this.addActiveDirectoryRules(); + } + } + + setupIngress(): void { + this.addApiIngressRule(); + this.addIngressRule( + ec2.Peer.ipv4(this.vpc.vpcCidrBlock), + ec2.Port.allTraffic(), + `Allow all Internal traffic TO ${this.componentName}`, + ); + this.addBastionHostIngressRule(this.bastionHostSecurityGroup); + this.addLoadbalancerIngressRule(this.publicLoadbalancerSecurityGroup); + } + + setupEgress(): void { + this.addOutboundTrafficRule(); + } +} + +export interface VirtualDesktopBrokerSecurityGroupProps { + bastionHostSecurityGroup: ec2.ISecurityGroup; + description: string; + componentName: string; + publicLoadbalancerSecurityGroup: ec2.ISecurityGroup; +} + +export class VirtualDesktopBrokerSecurityGroup extends SecurityGroup { + readonly componentName: string; + readonly publicLoadbalancerSecurityGroup: ec2.ISecurityGroup; + readonly bastionHostSecurityGroup: ec2.ISecurityGroup; + + constructor( + ctx: IdeaContext, + name: string, + scope: Construct, + vpc: ec2.IVpc, + props: VirtualDesktopBrokerSecurityGroupProps, + ) { + super(ctx, name, scope, vpc, props.description); + this.componentName = props.componentName; + this.publicLoadbalancerSecurityGroup = props.publicLoadbalancerSecurityGroup; + this.bastionHostSecurityGroup = props.bastionHostSecurityGroup; + this.setupIngress(); + this.setupEgress(); + } + + setupIngress(): void { + const brokerClientPort = requiredInt(this.ctx, 'virtual-desktop-controller.dcv_broker.client_communication_port'); + const brokerAgentPort = requiredInt(this.ctx, 'virtual-desktop-controller.dcv_broker.agent_communication_port'); + const brokerGatewayPort = requiredInt( + this.ctx, + 'virtual-desktop-controller.dcv_broker.gateway_communication_port', + ); + + const brokerPortList = [brokerClientPort, brokerAgentPort, brokerGatewayPort].sort((a, b) => a - b); + const minPort = brokerPortList[0] as number; + const maxPort = brokerPortList[brokerPortList.length - 1] as number; + const vpcCidr = ec2.Peer.ipv4(this.vpc.vpcCidrBlock); + + // one range rule when the three ports are consecutive (`sorted == range(min, max + 1)`) + if (brokerPortList.every((port, index) => port === minPort + index)) { + this.addIngressRule( + vpcCidr, + ec2.Port.tcpRange(minPort, maxPort), + `Allow VPC to broker ports ${minPort}-${maxPort}`, + ); + } else { + for (const port of brokerPortList) { + this.addIngressRule(vpcCidr, ec2.Port.tcp(port), `Allow VPC to broker port ${port}`); + } + } + + // broker to broker communications; hard-coded in the templates too + for (const port of [47100, 47200, 47500]) { + this.addIngressRule(this, ec2.Port.tcp(port), `Allow broker to broker port ${port}`); + } + this.addBastionHostIngressRule(this.bastionHostSecurityGroup); + this.addLoadbalancerIngressRule(this.publicLoadbalancerSecurityGroup); + } + + setupEgress(): void { + this.addOutboundTrafficRule(); + } +} + +/** The group with `allowAllOutbound` enabled. */ +export class VpcEndpointSecurityGroup extends SecurityGroup { + constructor(ctx: IdeaContext, name: string, scope: Construct, vpc: ec2.IVpc) { + super(ctx, name, scope, vpc, 'VPC Endpoints Security Group', true); + this.addIngressRule(ec2.Peer.ipv4(this.vpc.vpcCidrBlock), ec2.Port.tcp(443), 'Allow HTTPS traffic from VPC'); + } +} + +export class OpenSearchSecurityGroup extends SecurityGroup { + constructor(ctx: IdeaContext, name: string, scope: Construct, vpc: ec2.IVpc) { + super(ctx, name, scope, vpc, 'OpenSearch security group'); + this.setupIngress(); + this.setupEgress(); + } + + setupIngress(): void { + this.addIngressRule( + ec2.Peer.ipv4(this.vpc.vpcCidrBlock), + ec2.Port.tcp(443), + 'Allow HTTPS traffic from all VPC nodes to OpenSearch', + ); + } + + setupEgress(): void { + this.addOutboundTrafficRule(); + } +} + +/** No rules at all, so the L2 emits its "Disallow all traffic" placeholder egress. */ +export class DefaultClusterSecurityGroup extends SecurityGroup { + constructor(ctx: IdeaContext, name: string, scope: Construct, vpc: ec2.IVpc) { + super(ctx, name, scope, vpc, 'Default Cluster Security'); + } +} + +export class InternalLoadBalancerSecurityGroup extends SecurityGroup { + constructor(ctx: IdeaContext, name: string, scope: Construct, vpc: ec2.IVpc) { + super(ctx, name, scope, vpc, 'Internal load balancer security group'); + this.setupIngress(); + this.setupEgress(); + } + + setupIngress(): void { + // The OpenSearch description is part of the rendered rule. + this.addIngressRule( + ec2.Peer.ipv4(this.vpc.vpcCidrBlock), + ec2.Port.tcp(443), + 'Allow HTTPS traffic from all VPC nodes to OpenSearch', + ); + } + + setupEgress(): void { + this.addOutboundTrafficRule(); + } +} + +// --- VPC endpoints ---------------------------------------------------------------------------- + +/** + * `vpc/-gateway-endpoint` plus a `Custom::EC2CreateTags` of the same name at the stack + * scope, because tags set through the endpoint L2 never reached the endpoint. + */ +export class VpcGatewayEndpoint { + readonly ctx: IdeaContext; + readonly scope: Construct; + readonly name: string; + readonly endpoint: ec2.GatewayVpcEndpoint; + + constructor(ctx: IdeaContext, scope: Construct, service: string, vpc: ec2.IVpc, createTags: CreateTagsCustomResource) { + this.ctx = ctx; + this.scope = scope; + this.name = `${service}-gateway-endpoint`; + + this.endpoint = vpc.addGatewayEndpoint(constructId(this.name), { + service: new ec2.GatewayVpcEndpointAwsService(service), + }); + + createTags.apply(this.name, this.endpoint.vpcEndpointId, { + [IDEA_TAG_NAME]: this.name, + [IDEA_TAG_CLUSTER_NAME]: ctx.clusterName, + }); + } +} + +export class VpcInterfaceEndpoint { + readonly ctx: IdeaContext; + readonly scope: Construct; + readonly name: string; + readonly endpoint: ec2.InterfaceVpcEndpoint; + + constructor( + ctx: IdeaContext, + scope: Construct, + service: string, + vpc: ec2.IVpc, + vpcEndpointSecurityGroup: ec2.ISecurityGroup, + createTags: CreateTagsCustomResource, + ) { + this.ctx = ctx; + this.scope = scope; + this.name = `${service}-vpc-endpoint`; + + // Component groups do not exist when the cluster stack deploys, so every VPC node may use + // the interface endpoints. + this.endpoint = vpc.addInterfaceEndpoint(constructId(this.name), { + service: new ec2.InterfaceVpcEndpointAwsService(service), + open: true, + // private DNS is always on, GovCloud included, where private hosted zones may be unavailable + privateDnsEnabled: true, + lookupSupportedAzs: true, + securityGroups: [vpcEndpointSecurityGroup], + }); + + createTags.apply(this.name, this.endpoint.vpcEndpointId, { + [IDEA_TAG_NAME]: this.name, + [IDEA_TAG_CLUSTER_NAME]: ctx.clusterName, + }); + } + + /** `https://` + the DNS name of the first DNS entry (`:`). */ + getEndpointUrl(): string { + const dns = Fn.select(1, Fn.split(':', Fn.select(0, this.endpoint.vpcEndpointDnsEntries))); + return `https://${dns}`; + } +} + +// --- WAF -------------------------------------------------------------------------------------- + +/** + * AWS WAF WebACL for the external ALB: the ACL, and with CloudWatch logs enabled a log group and + * a logging configuration, all three at the stack scope under `--...` ids. The ACL + * and the logging configuration are L1s as direct children, so their logical ids carry no hash. + */ +export class WebAcl { + readonly ctx: IdeaContext; + readonly name: string; + readonly scope: Construct; + readonly createTags: CreateTagsCustomResource | undefined; + readonly webAcl: wafv2.CfnWebACL; + logGroup: logs.LogGroup | undefined; + loggingConfiguration: wafv2.CfnLoggingConfiguration | undefined; + + constructor(ctx: IdeaContext, name: string, scope: Construct, createTags?: CreateTagsCustomResource) { + this.ctx = ctx; + this.name = name; + this.scope = scope; + this.createTags = createTags; + + const clusterName = ctx.clusterName; + this.webAcl = new wafv2.CfnWebACL(scope, `${clusterName}-${name}-web-acl`, { + name: `${clusterName}-${name}`, + scope: 'REGIONAL', + defaultAction: { allow: {} }, + description: `WAF WebACL for ${clusterName} ${name}`, + rules: this.createManagedRules(), + visibilityConfig: { + cloudWatchMetricsEnabled: true, + metricName: `${clusterName}-${name}`, + sampledRequestsEnabled: true, + }, + tags: [ + { key: 'Name', value: `${clusterName}-${name}` }, + { key: 'idea:ClusterName', value: clusterName }, + { key: 'idea:Module', value: 'cluster' }, + ], + }); + + if (ctx.config.getBool('cluster.cloudwatch_logs.enabled', false)) { + this.setupCloudwatchLogging(scope, name); + } + } + + private createManagedRules(): wafv2.CfnWebACL.RuleProperty[] { + const rules: wafv2.CfnWebACL.RuleProperty[] = []; + + // blocks requests from IP addresses known to be malicious + rules.push({ + name: 'AWS-AWSManagedRulesAmazonIpReputationList', + priority: 0, + statement: { + managedRuleGroupStatement: { + vendorName: 'AWS', + name: 'AWSManagedRulesAmazonIpReputationList', + }, + }, + overrideAction: { none: {} }, + visibilityConfig: { + cloudWatchMetricsEnabled: true, + metricName: 'AWS-AWSManagedRulesAmazonIpReputationList', + sampledRequestsEnabled: true, + }, + }); + + // OWASP top 10; three rules excluded to avoid false positives + rules.push({ + name: 'AWS-AWSManagedRulesCommonRuleSet', + priority: 1, + statement: { + managedRuleGroupStatement: { + vendorName: 'AWS', + name: 'AWSManagedRulesCommonRuleSet', + version: 'Version_1.18', + excludedRules: [ + { name: 'SizeRestrictions_BODY' }, + { name: 'CrossSiteScripting_BODY' }, + { name: 'RestrictedExtensions_QUERYARGUMENTS' }, + ], + }, + }, + overrideAction: { none: {} }, + visibilityConfig: { + cloudWatchMetricsEnabled: true, + metricName: 'AWS-AWSManagedRulesCommonRuleSet', + sampledRequestsEnabled: true, + }, + }); + + // request patterns known to be malicious + rules.push({ + name: 'AWS-AWSManagedRulesKnownBadInputsRuleSet', + priority: 2, + statement: { + managedRuleGroupStatement: { + vendorName: 'AWS', + name: 'AWSManagedRulesKnownBadInputsRuleSet', + version: 'Version_1.22', + }, + }, + overrideAction: { none: {} }, + visibilityConfig: { + cloudWatchMetricsEnabled: true, + metricName: 'AWS-AWSManagedRulesKnownBadInputsRuleSet', + sampledRequestsEnabled: true, + }, + }); + + // optional bot control at the COMMON inspection level (billed per month and per request) + const botControlEnabled = this.ctx.config.getBool( + 'cluster.load_balancers.external_alb.waf.bot_control.enabled', + false, + ); + if (botControlEnabled) { + rules.push({ + name: 'AWS-AWSManagedRulesBotControlRuleSet', + priority: 3, + statement: { + managedRuleGroupStatement: { + vendorName: 'AWS', + name: 'AWSManagedRulesBotControlRuleSet', + version: 'Version_3.2', + excludedRules: [{ name: 'CategoryHttpLibrary' }, { name: 'SignalNonBrowserUserAgent' }], + managedRuleGroupConfigs: [ + { awsManagedRulesBotControlRuleSet: { inspectionLevel: 'COMMON' } }, + ], + }, + }, + overrideAction: { none: {} }, + visibilityConfig: { + cloudWatchMetricsEnabled: true, + metricName: 'AWS-AWSManagedRulesBotControlRuleSet', + sampledRequestsEnabled: true, + }, + }); + } + + return rules; + } + + private setupCloudwatchLogging(scope: Construct, name: string): void { + const clusterName = this.ctx.clusterName; + // WAF requires the log group name to start with `aws-waf-logs-` + const logGroupName = `aws-waf-logs-${clusterName}-cluster-waf-${name}`; + + // an unknown retention value keeps the CDK default (two years), with a warning + let retention: logs.RetentionDays | undefined; + const retentionDays = this.ctx.config.getInt('cluster.cloudwatch_logs.retention_in_days'); + if (retentionDays !== undefined) { + retention = LOG_RETENTION_DAYS[retentionDays]; + if (retention === undefined) { + console.warn( + `Invalid retention days value: ${retentionDays}. ` + + `Valid values are: [${Object.keys(LOG_RETENTION_DAYS).join(', ')}]. ` + + 'Using the CDK default retention (two years).', + ); + } + } + + this.logGroup = new logs.LogGroup(scope, `${clusterName}-${name}-waf-log-group`, { + logGroupName, + removalPolicy: RemovalPolicy.DESTROY, + retention, + }); + + Tags.of(this.logGroup).add('Name', `${clusterName}-${name}-waf-logs`); + Tags.of(this.logGroup).add('idea:ClusterName', clusterName); + Tags.of(this.logGroup).add('idea:Module', 'cluster'); + + // drop ALLOW actions from the log by default; the filter is a raw CloudFormation dict + const dropAllowLogs = this.ctx.config.getBool( + 'cluster.load_balancers.external_alb.waf.logging.drop_allow_actions', + true, + ); + const loggingFilter = dropAllowLogs + ? { + DefaultBehavior: 'KEEP', + Filters: [ + { + Behavior: 'DROP', + Requirement: 'MEETS_ANY', + Conditions: [{ ActionCondition: { Action: 'ALLOW' } }], + }, + ], + } + : undefined; + + this.loggingConfiguration = new wafv2.CfnLoggingConfiguration( + scope, + `${clusterName}-${name}-waf-logging-config`, + { + logDestinationConfigs: [this.logGroup.logGroupArn], + resourceArn: this.webAcl.attrArn, + loggingFilter, + }, + ); + + this.loggingConfiguration.node.addDependency(this.webAcl); + this.loggingConfiguration.node.addDependency(this.logGroup); + } + + get webAclArn(): string { + return this.webAcl.attrArn; + } + + get webAclId(): string { + return this.webAcl.attrId; + } +} diff --git a/source/idea/ideactl/src/cdk/constructs/storage.ts b/source/idea/ideactl/src/cdk/constructs/storage.ts new file mode 100644 index 00000000..34464fa7 --- /dev/null +++ b/source/idea/ideactl/src/cdk/constructs/storage.ts @@ -0,0 +1,243 @@ +/** + * Both classes are plain holders. The L1 file systems are created directly under the caller's + * scope and use `name` as the construct id. + */ + +import { CfnDeletionPolicy } from 'aws-cdk-lib'; +import type { Construct } from 'constructs'; +import type * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as efs from 'aws-cdk-lib/aws-efs'; +import * as fsx from 'aws-cdk-lib/aws-fsx'; + +import { isEmpty } from '../../config/cluster-config.ts'; +import type { IdeaContext } from './base.ts'; +import { addBackupTags, addCommonTags, resourceName } from './base.ts'; + +// --- `Utils.get_value_as_*` over a config subtree ---------------------------------------------- +// +// A DynamoDB NULL arrives as `null`. Missing, null, and empty string, list, or object values use +// the default instead of JavaScript's `??`. + +/** `ModelUtils.value_exists`. */ +export function valueExists(key: string, obj: Record | undefined): boolean { + if (obj === undefined || obj === null) return false; + if (!(key in obj)) return false; + const value = obj[key]; + if (value === null || value === undefined) return false; + if (typeof value === 'string' || typeof value === 'object') return !isEmpty(value); + return true; +} + +/** `Utils.get_value_as_string`: strings are stripped, everything else goes through `str()`. */ +export function valueAsString(key: string, obj: Record | undefined): string | undefined { + if (!valueExists(key, obj)) return undefined; + const value = (obj as Record)[key]; + if (typeof value === 'string') { + const stripped = value.trim(); + return stripped.length === 0 ? undefined : stripped; + } + if (typeof value === 'boolean') return value ? 'True' : 'False'; + return String(value); +} + +/** Booleans are returned unchanged. */ +export function valueAsInt(key: string, obj: Record | undefined): number | boolean | undefined { + if (!valueExists(key, obj)) return undefined; + const value = (obj as Record)[key]; + if (typeof value === 'boolean') return value; + if (typeof value === 'number') return Math.trunc(value); + if (typeof value === 'string') { + const text = value.trim(); + // `int()`/`float()` parse neither a base prefix nor an empty string, so neither does this + if (text.length > 0 && !/^[+-]?0[xXoObB]/.test(text)) { + const parsed = Number(text); + if (Number.isFinite(parsed)) return Math.trunc(parsed); + } + } + return undefined; +} + +/** `Utils.get_value_as_bool`. */ +export function valueAsBool( + key: string, + obj: Record | undefined, + defaultValue: boolean, +): boolean { + if (!valueExists(key, obj)) return defaultValue; + const value = (obj as Record)[key]; + if (typeof value === 'boolean') return value; + if (typeof value === 'number') return Boolean(value); + if (typeof value === 'string') { + const lowered = value.trim().toLowerCase(); + if (['true', 'yes', 'y', '1'].includes(lowered)) return true; + if (['false', 'no', 'n', '0'].includes(lowered)) return false; + } + return defaultValue; +} + +/** `Utils.get_value_as_dict`: an empty dict reads as absent. */ +export function valueAsDict( + key: string, + obj: Record | undefined, +): Record | undefined { + if (!valueExists(key, obj)) return undefined; + const value = (obj as Record)[key]; + return typeof value === 'object' && value !== null && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +/** `Utils.get_value_as_list`: an empty list reads as absent. */ +export function valueAsList(key: string, obj: Record | undefined): unknown[] | undefined { + if (!valueExists(key, obj)) return undefined; + const value = (obj as Record)[key]; + return Array.isArray(value) ? value : undefined; +} + +/** + * `CfnDeletionPolicy` lookup uses member names. `DESTROY` is rewritten to `DELETE`, and an + * unknown value raises instead of silently emitting no policy. + */ +export function cfnDeletionPolicy(memberName: string | undefined): CfnDeletionPolicy { + const policy = (CfnDeletionPolicy as unknown as Record)[ + memberName ?? '' + ]; + if (policy === undefined) { + throw new Error(`${memberName} is not a valid CfnDeletionPolicy`); + } + return policy; +} + +// --- EFS -------------------------------------------------------------------------------------- + +export interface AmazonEFSProps { + vpc: ec2.IVpc; + securityGroup: ec2.ISecurityGroup; + /** The `shared-storage..efs` subtree. */ + efsConfig: Record | undefined; + subnets?: ec2.ISubnet[]; +} + +/** + * `storage.py:AmazonEFS`. One `CfnFileSystem` (construct id = `name`) plus one `CfnMountTarget` + * per subnet, **scoped to the file system**, so the logical ID repeats the file-system segment. + * + * `DeletionPolicy` comes from `.removal_policy` and no `UpdateReplacePolicy` is set. + * + * CloudWatch monitoring is always undefined, so it creates no resources. + */ +export class AmazonEFS { + readonly ctx: IdeaContext; + readonly name: string; + readonly fileSystem: efs.CfnFileSystem; + readonly mountTargets: efs.CfnMountTarget[]; + /** Present in configuration but unused. */ + readonly cloudWatchMonitoring: undefined; + + constructor(ctx: IdeaContext, name: string, scope: Construct, props: AmazonEFSProps) { + this.ctx = ctx; + this.name = name; + + const config = props.efsConfig; + const kmsKeyId = valueAsString('kms_key_id', config); + const transitionToIa = valueAsString('transition_to_ia', config); + const encrypted = valueAsBool('encrypted', config, true); + const throughputMode = valueAsString('throughput_mode', config) ?? 'bursting'; + const performanceMode = valueAsString('performance_mode', config) ?? 'generalPurpose'; + let removalPolicy = valueAsString('removal_policy', config); + if (removalPolicy === 'DESTROY') removalPolicy = 'DELETE'; + const deletionPolicy = cfnDeletionPolicy(removalPolicy); + + this.fileSystem = new efs.CfnFileSystem(scope, name, { + encrypted, + fileSystemTags: [{ key: 'Name', value: resourceName(ctx, name) }], + kmsKeyId, + throughputMode, + performanceMode, + lifecyclePolicies: isEmpty(transitionToIa) ? undefined : [{ transitionToIa }], + fileSystemPolicy: { + Version: '2012-10-17', + Id: 'efs-prevent-anonymous-access-policy', + Statement: [ + { + Sid: 'efs-statement', + Effect: 'Allow', + Principal: { AWS: '*' }, + Action: [ + 'elasticfilesystem:ClientRootAccess', + 'elasticfilesystem:ClientWrite', + 'elasticfilesystem:ClientMount', + ], + Condition: { Bool: { 'elasticfilesystem:AccessedViaMountTarget': 'true' } }, + }, + ], + }, + }); + addCommonTags(ctx, this.fileSystem, name); + addBackupTags(ctx, this.fileSystem); + this.fileSystem.cfnOptions.deletionPolicy = deletionPolicy; + + const subnets = props.subnets ?? props.vpc.privateSubnets; + this.mountTargets = subnets.map((subnet, index) => { + const mountTarget = new efs.CfnMountTarget(this.fileSystem, `${name}-mount-target-${index + 1}`, { + fileSystemId: this.fileSystem.ref, + securityGroups: [props.securityGroup.securityGroupId], + subnetId: subnet.subnetId, + }); + addCommonTags(ctx, mountTarget, name); + return mountTarget; + }); + } +} + +// --- FSx for Lustre --------------------------------------------------------------------------- + +export interface FSxForLustreProps { + vpc: ec2.IVpc; + securityGroup: ec2.ISecurityGroup; + /** The `shared-storage..fsx_lustre` subtree. */ + fsxLustreConfig: Record | undefined; + subnets?: ec2.ISubnet[]; +} + +/** + * `storage.py:FSxForLustre`. A single `AWS::FSx::FileSystem` in the **first** subnet only; no + * deletion policy is set, so it inherits CloudFormation's default (Delete). + */ +export class FSxForLustre { + readonly fileSystem: fsx.CfnFileSystem; + + constructor(ctx: IdeaContext, name: string, scope: Construct, props: FSxForLustreProps) { + const config = props.fsxLustreConfig; + const deploymentType = valueAsString('deployment_type', config); + const storageType = valueAsString('storage_type', config); + // `drive_cache_type` accepts numeric values, so string values are omitted. + const driveCacheTypeAsInt = valueAsInt('drive_cache_type', config); + + let perUnitStorageThroughput: number | boolean | undefined; + let driveCacheType: number | boolean | undefined; + if (storageType === 'SSD') { + if (deploymentType === 'PERSISTENT_1') { + perUnitStorageThroughput = valueAsInt('per_unit_storage_throughput', config); + } + } else { + driveCacheType = driveCacheTypeAsInt; + } + + const subnets = props.subnets ?? props.vpc.privateSubnets; + this.fileSystem = new fsx.CfnFileSystem(scope, name, { + fileSystemType: 'LUSTRE', + subnetIds: [(subnets[0] as ec2.ISubnet).subnetId], + lustreConfiguration: { + deploymentType, + perUnitStorageThroughput: perUnitStorageThroughput as number | undefined, + driveCacheType: driveCacheType as unknown as string | undefined, + }, + securityGroupIds: [props.securityGroup.securityGroupId], + kmsKeyId: valueAsString('kms_key_id', config), + storageCapacity: valueAsInt('storage_capacity', config) as number | undefined, + }); + addCommonTags(ctx, this.fileSystem, name); + addBackupTags(ctx, this.fileSystem); + } +} diff --git a/source/idea/ideactl/src/cdk/policy.ts b/source/idea/ideactl/src/cdk/policy.ts new file mode 100644 index 00000000..9fa68c28 --- /dev/null +++ b/source/idea/ideactl/src/cdk/policy.ts @@ -0,0 +1,152 @@ +/** + * Renders IAM policy documents from `resources/policies/*.yml` Jinja templates. + * + * Templates receive a `context` with cluster values, config, ARNs, and utilities. + * Facades map snake_case names to camelCase members and bind keyword arguments. + */ + +import { existsSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { load } from 'js-yaml'; +import type nunjucks from 'nunjucks'; + +import { ArnBuilder } from '../config/arn-builder.ts'; +import type { ClusterConfig } from '../config/cluster-config.ts'; +import { jinjaEnv, renderTemplate, toYaml } from '../config/jinja.ts'; + +export interface PolicyVars { + config: ClusterConfig; + /** `Policy(module_id=...)`; only cluster-manager.yml, compute-node.yml and scheduler.yml read it. */ + moduleId?: string; + /** `Policy(vars=SocaAnyPayload(...))`, reached as `context.vars.`. */ + vars?: Record; + /** Override for the `resources/policies` directory; defaults to the packaged one. */ + policiesDir?: string; +} + +const HERE = dirname(fileURLToPath(import.meta.url)); + +/** Finds the package resources directory. */ +export function resourcesDir(): string { + const candidates = [ + join(HERE, '..', '..', 'resources'), + join(HERE, '..', '..', '..', 'idea-administrator', 'resources'), + ]; + const found = candidates.find((candidate) => existsSync(candidate)); + if (found === undefined) throw new Error(`resources directory not found; looked in ${candidates.join(', ')}`); + return found; +} + +/** Binds trailing keyword arguments collected by the renderer. */ +function bindKeywords(args: unknown[], params?: readonly string[]): unknown[] { + const last = args[args.length - 1] as Record | undefined; + if (last === null || typeof last !== 'object' || last.__keywords !== true) return args; + const positional = args.slice(0, -1); + for (const [name, value] of Object.entries(last)) { + if (name === '__keywords') continue; + const index = params?.indexOf(name) ?? -1; + if (index < 0) throw new Error(`unexpected keyword argument '${name}'`); + while (positional.length < index) positional.push(undefined); + positional[index] = value; + } + return positional; +} + +const snakeToCamel = (name: string): string => name.replace(/_([a-z0-9])/g, (_match, c: string) => c.toUpperCase()); + +/** Keyword parameter order for the ARN builder member called with keywords. */ +const ARN_KEYWORDS: Record = { + get_arn: ['service', 'resource', 'aws_account_id', 'aws_region'], +}; + +function snakeCaseFacade(target: object, keywords: Record): object { + return new Proxy(Object.create(null) as Record, { + get(_holder, property) { + if (typeof property !== 'string') return undefined; + const member = (target as Record)[snakeToCamel(property)]; + if (typeof member !== 'function') return member; + return (...args: unknown[]) => member.apply(target, bindKeywords(args, keywords[property])); + }, + }); +} + +/** Exposes configuration getters to templates. */ +function configFacade(config: ClusterConfig): object { + const split = (args: unknown[]) => { + const last = args[args.length - 1] as Record | undefined; + const keywords = last !== null && typeof last === 'object' && last.__keywords === true ? last : {}; + const positional = keywords === last ? args.slice(0, -1) : args; + return { + key: positional[0] as string, + fallback: positional.length > 1 ? positional[1] : keywords.default, + options: { required: keywords.required === true }, + }; + }; + return { + get_string: (...args: unknown[]) => { + const { key, fallback, options } = split(args); + return config.getString(key, fallback as string, options); + }, + get_bool: (...args: unknown[]) => { + const { key, fallback, options } = split(args); + return config.getBool(key, fallback as boolean, options); + }, + get_int: (...args: unknown[]) => { + const { key, fallback, options } = split(args); + return config.getInt(key, fallback as number, options); + }, + get_list: (...args: unknown[]) => { + const { key, fallback, options } = split(args); + const list = config.getList(key, fallback as unknown[], options); + // Empty lists must be falsy in template conditions and iterate zero times. + return list === undefined || list.length === 0 ? undefined : list; + }, + get_module_id: (moduleName: string) => config.moduleId(moduleName), + is_module_enabled: (moduleName: string) => config.isModuleEnabled(moduleName), + }; +} + +const envCache = new Map(); + +function policyEnv(policiesDir: string): nunjucks.Environment { + let env = envCache.get(policiesDir); + if (env === undefined) { + env = jinjaEnv(policiesDir); + envCache.set(policiesDir, env); + } + return env; +} + +/** Builds the context object used to render policy templates. */ +export function policyContext(policyVars: PolicyVars): object { + const { config } = policyVars; + return { + cluster_name: config.getString('cluster.cluster_name'), + module_id: policyVars.moduleId ?? null, + aws_region: config.getString('cluster.aws.region', undefined, { required: true }), + aws_dns_suffix: config.getString('cluster.aws.dns_suffix', undefined, { required: true }), + aws_partition: config.getString('cluster.aws.partition', undefined, { required: true }), + aws_account_id: config.getString('cluster.aws.account_id', undefined, { required: true }), + config: configFacade(config), + arns: snakeCaseFacade(new ArnBuilder(config), ARN_KEYWORDS), + vars: policyVars.vars ?? {}, + utils: { to_yaml: toYaml }, + }; +} + +/** Renders and parses a policy document for `PolicyDocument.fromJson`. */ +export function renderPolicy(policyName: string, policyVars: PolicyVars): object { + const dir = policyVars.policiesDir ?? join(resourcesDir(), 'policies'); + const text = renderTemplate(policyEnv(dir), policyName, { context: policyContext(policyVars) }); + try { + // Include the template name in the parser error. + return load(text, { filename: policyName }) as object; + } catch (error) { + // Report numbered source content and preserve the parser error. + const numbered = text.split('\n').map((line, index) => `${String(index + 1).padStart(5)}: ${line}`); + console.error(`failed to decode policy json: ${policyName} - ${String(error)}. Content:\n${numbered.join('\n')}`); + throw error; + } +} diff --git a/source/idea/ideactl/src/cdk/stacks/analytics.ts b/source/idea/ideactl/src/cdk/stacks/analytics.ts new file mode 100644 index 00000000..1553bceb --- /dev/null +++ b/source/idea/ideactl/src/cdk/stacks/analytics.ts @@ -0,0 +1,438 @@ +/** + * Two values are regenerated on every synth: the `UpdateToken` that makes the + * private-IP custom resource re-run, and the uuid tail of the dashboard target group name, which + * replaces that target group on every deploy so the endpoints lambda is re-pointed at a fresh one. + * + * `analytics.opensearch.use_existing`, the GovCloud Kinesis branches, and the service-linked-role + * branch require their respective configuration conditions. + */ + +import { randomUUID } from 'node:crypto'; + +import { + CfnDeletionPolicy, + CfnOutput, + CustomResource as CdkCustomResource, + Fn, + RemovalPolicy, +} from 'aws-cdk-lib'; +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as elbv2 from 'aws-cdk-lib/aws-elasticloadbalancingv2'; +import * as kinesis from 'aws-cdk-lib/aws-kinesis'; +import * as kms from 'aws-cdk-lib/aws-kms'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import { KinesisEventSource } from 'aws-cdk-lib/aws-lambda-event-sources'; +import * as logs from 'aws-cdk-lib/aws-logs'; +import * as opensearch from 'aws-cdk-lib/aws-opensearchservice'; + +import type { StackBuildProps } from '../app.ts'; +import { IdeaBaseStack } from '../base-stack.ts'; +import { IdeaCodeAsset } from '../code-asset.ts'; +import { OpenSearch } from '../constructs/analytics.ts'; +import { kmsKeyArn } from '../constructs/base.ts'; +import { + CustomResourceProvider, + KinesisStream, + LambdaFunction, + Policy, + Role, +} from '../constructs/common.ts'; +import { ExistingSocaCluster, lookupExistingOpensearch } from '../constructs/existing-resources.ts'; +import { OpenSearchSecurityGroup } from '../constructs/network.ts'; + +export const MODULE_ANALYTICS = 'analytics'; + +/** `constants.CAVEATS['KINESIS_STREAMS_CLOUDFORMATION_UNSUPPORTED_STREAMMODEDETAILS_REGION_LIST']`. */ +export const KINESIS_STREAM_MODE_UNSUPPORTED_REGIONS = ['us-gov-east-1', 'us-gov-west-1']; + +/** `RemovalPolicy` lookup uses member names, not enum values. */ +function removalPolicyByName(name: string): RemovalPolicy { + if (!Object.prototype.hasOwnProperty.call(RemovalPolicy, name)) { + throw new Error(`'${name}' is not a valid RemovalPolicy`); + } + return RemovalPolicy[name as keyof typeof RemovalPolicy]; +} + +export class AnalyticsStack extends IdeaBaseStack { + readonly cluster: ExistingSocaCluster; + securityGroup: ec2.ISecurityGroup | undefined; + opensearch!: opensearch.IDomain; + kinesisStream!: KinesisStream; + + /** + * `serviceLinkedRoleExists` is the answer to the `iam:ListRoles` probe. `buildStack` resolves + * it before constructing the stack. + */ + constructor( + props: StackBuildProps, + serviceLinkedRoleExists: boolean, + existingDomainDataNodes?: number, + ) { + super({ + scope: props.app, + ctx: props.ctx, + moduleName: props.moduleName, + deploymentId: props.deploymentId, + terminationProtection: props.terminationProtection, + env: props.env, + }); + + this.cluster = new ExistingSocaCluster(this.context, this.stack); + + this.buildSecurityGroup(); + + if (this.context.config.getBool('analytics.opensearch.use_existing', false)) { + this.opensearch = lookupExistingOpensearch(this.context, this.stack); + this.buildDashboardEndpoints(existingDomainDataNodes); + this.addNagSuppression( + [ + { + rule_id: 'AwsSolutions-KDS3', + reason: 'Kinesis Data Stream is encrypted with customer-managed KMS key', + }, + ], + this.stack, + ); + } else { + this.buildOpenSearch(!serviceLinkedRoleExists); + this.buildDashboardEndpoints(); + this.addNagSuppression( + [ + { + rule_id: 'AwsSolutions-IAM5', + reason: 'CDK L2 construct does not support custom LogGroup permissions', + }, + { rule_id: 'AwsSolutions-IAM4', reason: 'Usage is required for Service Linked Role' }, + { + rule_id: 'AwsSolutions-L1', + reason: 'CDK L2 construct does not offer options to customize the Lambda runtime', + }, + { + rule_id: 'AwsSolutions-KDS3', + reason: 'Kinesis Data Stream is encrypted with customer-managed KMS key', + }, + ], + this.stack, + ); + if (this.dataNodes() === 1) { + this.addNagSuppression( + [ + { + rule_id: 'AwsSolutions-OS7', + reason: 'OpenSearch domain has 1 data node disabling Zone Awareness', + }, + ], + this.stack, + ); + } + } + + this.buildAnalyticsInputStream(); + this.buildClusterSettings(); + } + + dataNodes(): number { + return this.context.config.getInt('analytics.opensearch.data_nodes', 0, { required: true }); + } + + buildSecurityGroup(): void { + this.securityGroup = new OpenSearchSecurityGroup( + this.context, + `${this.moduleId}-opensearch-security-group`, + this.stack, + this.cluster.vpc, + ); + } + + buildOpenSearch(createServiceLinkedRole: boolean): void { + const config = this.context.config; + const dataNodes = this.dataNodes(); + const dataNodeInstanceType = config.getString('analytics.opensearch.data_node_instance_type', '', { + required: true, + }); + const ebsVolumeSize = config.getInt('analytics.opensearch.ebs_volume_size', 0, { required: true }); + const nodeToNodeEncryption = config.getBool('analytics.opensearch.node_to_node_encryption', false, { + required: true, + }); + const removalPolicy = config.getString('analytics.opensearch.removal_policy', '', { required: true }); + const appLogRemovalPolicy = config.getString('analytics.opensearch.logging.app_log_removal_policy', 'DESTROY'); + const searchLogRemovalPolicy = config.getString( + 'analytics.opensearch.logging.search_log_removal_policy', + 'DESTROY', + ); + const slowIndexLogRemovalPolicy = config.getString( + 'analytics.opensearch.logging.slow_index_log_removal_policy', + 'DESTROY', + ); + + const kmsKeyId = config.getString('analytics.opensearch.kms_key_id'); + const encryptionKey = + kmsKeyId === undefined + ? undefined + : kms.Key.fromKeyArn(this.stack, 'opensearch-kms-key', kmsKeyArn(this.context, kmsKeyId)); + + // The logging options construct log groups before the domain. + const logging: opensearch.LoggingOptions = { + slowSearchLogEnabled: config.getBool('analytics.opensearch.logging.slow_search_log_enabled', false, { + required: true, + }), + slowSearchLogGroup: new logs.LogGroup(this.stack, 'analytics-search-log-group', { + logGroupName: `/${this.clusterName}/${this.moduleId}/search-log`, + removalPolicy: removalPolicyByName(searchLogRemovalPolicy), + }), + appLogEnabled: config.getBool('analytics.opensearch.logging.app_log_enabled', false, { required: true }), + appLogGroup: new logs.LogGroup(this.stack, 'analytics-app-log-group', { + logGroupName: `/${this.clusterName}/${this.moduleId}/app-log`, + removalPolicy: removalPolicyByName(appLogRemovalPolicy), + }), + slowIndexLogEnabled: config.getBool('analytics.opensearch.logging.slow_index_log_enabled', false, { + required: true, + }), + slowIndexLogGroup: new logs.LogGroup(this.stack, 'analytics-slow-index-log-group', { + logGroupName: `/${this.clusterName}/${this.moduleId}/slow-index-log`, + removalPolicy: removalPolicyByName(slowIndexLogRemovalPolicy), + }), + // Audit logs need fine-grained access control; provision the domain manually and import it + // through the use-existing flow to turn them on. + auditLogEnabled: false, + }; + + // The construct id is the literal `analytics`, not the module id: a cluster with a custom + // analytics module id still names this construct (and therefore its logical id) `analytics`. + this.opensearch = new OpenSearch(this.context, MODULE_ANALYTICS, this.stack, { + cluster: this.cluster, + securityGroups: [this.securityGroup as ec2.ISecurityGroup], + dataNodes, + dataNodeInstanceType, + ebsVolumeSize, + removalPolicy: removalPolicyByName(removalPolicy), + nodeToNodeEncryption, + kmsKeyArn: encryptionKey, + createServiceLinkedRole, + logging, + }); + } + + buildDashboardEndpoints(existingDomainDataNodes?: number): void { + const config = this.context.config; + const clusterEndpointsLambdaArn = config.getString('cluster.cluster_endpoints_lambda_arn', '', { + required: true, + }); + const externalHttpsListenerArn = config.getString( + 'cluster.load_balancers.external_alb.https_listener_arn', + '', + { required: true }, + ); + const pathPatterns = config.getList('analytics.opensearch.endpoints.external.path_patterns', [], { + required: true, + }); + const priority = config.getInt('analytics.opensearch.endpoints.external.priority', 0, { required: true }); + + let domainName = this.opensearch.domainName; + let dataNodes: number; + if (config.getBool('analytics.opensearch.use_existing', false)) { + // The import reads the name off the endpoint, which carries a `vpc-` prefix that + // `describe_domain` rejects. The stripped name is what the custom resource gets too. + if (domainName.startsWith('vpc-')) domainName = domainName.replace('vpc-', ''); + if (existingDomainDataNodes === undefined) { + throw new Error(`no opensearch:DescribeDomain answer for ${domainName}`); + } + dataNodes = existingDomainDataNodes; + } else { + dataNodes = this.dataNodes(); + } + + const opensearchPrivateIps = new CustomResourceProvider( + this.context, + 'opensearch-private-ips', + this.stack, + { + ideaCodeAsset: new IdeaCodeAsset('idea_custom_resource_opensearch_private_ips'), + lambdaTimeoutSeconds: 180, + policyTemplateName: 'custom-resource-opensearch-private-ips.yml', + resourceType: 'OpenSearchPrivateIPAddresses', + }, + ).invoke('opensearch-private-ips', { + DomainName: domainName, + // Regenerated on every synth so the custom resource re-reads the domain's ENIs on deploy. + UpdateToken: randomUUID(), + }); + + const ipAddresses = opensearchPrivateIps.getAttString('IpAddresses'); + const targets: elbv2.CfnTargetGroup.TargetDescriptionProperty[] = []; + for (let i = 0; i < dataNodes; i += 1) { + targets.push({ id: Fn.select(i, Fn.split(',', ipAddresses)) }); + } + + // The uuid tail replaces the target group on every deploy and updates the endpoint rule. + const deploymentId = randomUUID(); + const dashboardTargetGroup = new elbv2.CfnTargetGroup( + this.stack, + `${this.clusterName}-dashboard-target-group`, + { + port: 443, + protocol: 'HTTPS', + targetType: 'ip', + vpcId: this.cluster.vpc.vpcId, + name: `${this.getTargetGroupName('dashboard')}-${deploymentId}`.slice(0, 32), + targets, + healthCheckPath: '/', + }, + ); + dashboardTargetGroup.node.addDependency(opensearchPrivateIps); + + new CfnOutput(this.stack, 'IPAddresses', { value: ipAddresses }); + new CfnOutput(this.stack, 'NumberOfTargets', { value: String(targets.length) }); + + new CdkCustomResource(this.stack, 'dashboard-endpoint', { + serviceToken: clusterEndpointsLambdaArn, + properties: { + endpoint_name: `${this.moduleId}-dashboard-endpoint`, + listener_arn: externalHttpsListenerArn, + priority, + target_group_arn: dashboardTargetGroup.ref, + conditions: [{ Field: 'path-pattern', Values: pathPatterns }], + actions: [{ Type: 'forward', TargetGroupArn: dashboardTargetGroup.ref }], + tags: { + 'idea:ClusterName': this.clusterName, + 'idea:ModuleId': this.moduleId, + 'idea:ModuleName': MODULE_ANALYTICS, + }, + }, + resourceType: 'Custom::DashboardEndpointExternal', + }); + } + + buildAnalyticsInputStream(): void { + const config = this.context.config; + const streamConfig = config.getString('analytics.kinesis.stream_mode', '', { required: true }); + if (streamConfig !== 'PROVISIONED' && streamConfig !== 'ON_DEMAND') { + throw new Error('analytics.kinesis.stream_mode needs to be one of PROVISIONED or ON_DEMAND only'); + } + const streamMode = + streamConfig === 'PROVISIONED' ? kinesis.StreamMode.PROVISIONED : kinesis.StreamMode.ON_DEMAND; + const shardCount = + streamConfig === 'PROVISIONED' + ? config.getInt('analytics.kinesis.shard_count', 0, { required: true }) + : undefined; + + this.kinesisStream = new KinesisStream(this.context, `${this.moduleId}-kinesis-stream`, this.stack, { + streamName: `${this.moduleId}-kinesis-stream`, + streamMode, + shardCount, + removalPolicy: removalPolicyByName(config.getString('analytics.kinesis.removal_policy', 'DESTROY')), + }); + if (KINESIS_STREAM_MODE_UNSUPPORTED_REGIONS.includes(this.awsRegion)) { + (this.kinesisStream.node.defaultChild as kinesis.CfnStream).addPropertyDeletionOverride( + 'StreamModeDetails', + ); + } + + const lambdaName = `${this.moduleId}-sink-lambda`; + const streamProcessingLambdaRole = new Role(this.context, `${lambdaName}-role`, this.stack, { + description: `Role for ${lambdaName} function for Cluster: ${this.clusterName}`, + assumedBy: ['lambda'], + }); + streamProcessingLambdaRole.attachInlinePolicy( + new Policy(this.context, `${lambdaName}-policy`, this.stack, { + policyTemplateName: 'analytics-sink-lambda.yml', + }), + ); + + const streamProcessingLambda = new LambdaFunction(this.context, lambdaName, this.stack, { + ideaCodeAsset: new IdeaCodeAsset('idea_analytics_sink'), + description: 'Lambda to process analytics-kinesis-stream data', + timeoutSeconds: 900, + securityGroups: [this.securityGroup as ec2.ISecurityGroup], + role: streamProcessingLambdaRole, + environment: { opensearch_endpoint: this.opensearch.domainEndpoint }, + vpc: this.cluster.vpc, + vpcSubnets: { subnets: this.cluster.privateSubnets }, + }); + this.addNagSuppression( + [{ rule_id: 'AwsSolutions-L1', reason: 'Python Runtime is selected for stability.' }], + streamProcessingLambda, + ); + + if (this.awsRegion.startsWith('us-gov-')) { + // GovCloud rejects the tags the L2 event source puts on the mapping, so it is built by hand. + // The L2 is skipped entirely, which also means the role's `DefaultPolicy` never exists there + // and the function runs on the inline `analytics-sink-lambda.yml` policy alone. + const eventSourceMapping = new lambda.CfnEventSourceMapping(this.stack, `${lambdaName}-event-source`, { + functionName: streamProcessingLambda.functionName, + eventSourceArn: this.kinesisStream.streamArn, + startingPosition: 'LATEST', + batchSize: 100, + tags: [], + }); + eventSourceMapping.cfnOptions.deletionPolicy = CfnDeletionPolicy.DELETE; + eventSourceMapping.cfnOptions.updateReplacePolicy = CfnDeletionPolicy.DELETE; + eventSourceMapping.addDependency(streamProcessingLambda.node.defaultChild as lambda.CfnFunction); + } else { + streamProcessingLambda.addEventSource( + new KinesisEventSource(this.kinesisStream, { + batchSize: 100, + startingPosition: lambda.StartingPosition.LATEST, + }), + ); + } + } + + buildClusterSettings(): void { + const clusterSettings: Record = { + deployment_id: this.deploymentId, + 'opensearch.domain_name': this.opensearch.domainName, + 'opensearch.domain_arn': this.opensearch.domainArn, + 'opensearch.domain_endpoint': this.opensearch.domainEndpoint, + 'opensearch.dashboard_endpoint': `${this.opensearch.domainEndpoint}/_dashboards`, + 'kinesis.stream_name': this.kinesisStream.streamName, + 'kinesis.stream_arn': this.kinesisStream.streamArn, + }; + if (this.securityGroup !== undefined) { + clusterSettings['opensearch.security_group_id'] = this.securityGroup.securityGroupId; + } + this.updateClusterSettings(clusterSettings); + } +} + +/** `/aws-service-role/es.` then `/aws-service-role/opensearchservice.`. */ +export function serviceLinkedRolePathPrefixes(dnsSuffix: string): string[] { + return [`/aws-service-role/es.${dnsSuffix}`, `/aws-service-role/opensearchservice.${dnsSuffix}`]; +} + +/** Matches the OpenSearch L2 endpoint parser before the domain read occurs. */ +export function domainNameFromEndpoint(endpoint: string): string { + const hostname = new URL(`https://${endpoint}`).hostname; + const domain = hostname.split(".")[0]; + const components = domain.split("-"); + const suffix = `-${components[components.length - 1]}`; + return domain.split(suffix)[0]; +} + +/** Reads the deployed service-linked role or existing domain before building the construct tree. */ +export async function buildStack(props: StackBuildProps): Promise { + const dnsSuffix = props.ctx.config.getString('cluster.aws.dns_suffix', '', { required: true }); + const pathPrefixes = serviceLinkedRolePathPrefixes(dnsSuffix); + const useExisting = props.ctx.config.getBool('analytics.opensearch.use_existing', false); + + if (useExisting) { + const endpoint = props.ctx.config.getString('analytics.opensearch.domain_vpc_endpoint_url', '', { + required: true, + }); + let domainName = domainNameFromEndpoint(endpoint); + if (domainName.startsWith('vpc-')) domainName = domainName.replace('vpc-', ''); + const domain = await props.ctx.synthReads.describeDomain(domainName); + const instanceCount = domain.ClusterConfig?.InstanceCount; + if (instanceCount === undefined) { + throw new Error(`no opensearch:DescribeDomain answer for ${domainName}`); + } + new AnalyticsStack(props, false, instanceCount); + return; + } + + let count = 0; + for (const pathPrefix of pathPrefixes) { + count += (await props.ctx.synthReads.listServiceLinkedRoles(pathPrefix)).length; + } + new AnalyticsStack(props, count > 0); +} diff --git a/source/idea/ideactl/src/cdk/stacks/bastion-host.ts b/source/idea/ideactl/src/cdk/stacks/bastion-host.ts new file mode 100644 index 00000000..bc561db5 --- /dev/null +++ b/source/idea/ideactl/src/cdk/stacks/bastion-host.ts @@ -0,0 +1,254 @@ +/** + * The bastion is a single EC2 host, built as an L1 `CfnInstance` next to an L2 `LaunchTemplate`. + * The launch template carries the encrypted root volume and IMDSv2; the instance repeats the + * block device mapping, the AMI, the instance type and the key name, and references the launch + * template by id and latest version. Both carry the same user data, so the script is emitted + * twice on purpose. + * + * The instance takes the instance profile by its literal **name**, not a `Ref`, so the only + * ordering edge is the explicit `DependsOn` on the instance profile (and the profile's own + * explicit `DependsOn` on the role). + */ + +import { Duration, Fn, Tags } from 'aws-cdk-lib'; +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as route53 from 'aws-cdk-lib/aws-route53'; + +import type { StackBuildProps } from '../app.ts'; +import { IdeaBaseStack } from '../base-stack.ts'; +import { IDEA_TAG_NODE_TYPE } from '../constructs/base.ts'; +import { InstanceProfile, Policy, Role } from '../constructs/common.ts'; +import { + ExistingSocaCluster, + lookupClusterDns, + lookupEbsKmsKey, + lookupKeyPair, +} from '../constructs/existing-resources.ts'; +import { buildBootstrapUserData } from '../userdata.ts'; + +export const MODULE_BASTION_HOST = 'bastion-host'; +const NODE_TYPE_INFRA = 'infra'; + +/** `Utils.get_ec2_block_device_name`. */ +export function ec2BlockDeviceName(baseOs: string): string { + return baseOs === 'amazonlinux2' || baseOs === 'amazonlinux2023' ? '/dev/xvda' : '/dev/sda1'; +} + +export class BastionHostStack extends IdeaBaseStack { + readonly cluster: ExistingSocaCluster; + readonly bootstrapPackageUri: string; + bastionHostRole!: Role; + bastionHostInstanceProfile!: InstanceProfile; + ec2Instance!: ec2.CfnInstance; + clusterDnsRecordSet!: route53.RecordSet; + + constructor(props: StackBuildProps) { + super({ + scope: props.app, + ctx: props.ctx, + moduleName: props.moduleName, + deploymentId: props.deploymentId, + terminationProtection: props.terminationProtection, + env: props.env, + }); + + this.bootstrapPackageUri = this.getBootstrapPackageUri(); + this.cluster = new ExistingSocaCluster(this.context, this.stack); + + this.buildIamRoles(); + this.buildEc2Instance(); + this.buildRoute53RecordSet(); + this.buildClusterSettings(); + } + + /** + * The bootstrap package is not a CDK asset: the CLI uploads it and passes its location as the + * `bootstrap_package_uri` context parameter. When absent, the deployment identifier derives + * the package name for a standalone synth, the same rule `bootstrapPackageBasenames` uses. + * + * The naming rule is local so stack synthesis does not load an S3 client. + */ + getBootstrapPackageUri(): string { + const fromContext: unknown = this.stack.node.tryGetContext('bootstrap_package_uri'); + if (typeof fromContext === 'string' && fromContext !== '') return fromContext; + const bucket = this.context.config.getString('cluster.cluster_s3_bucket', undefined, { + required: true, + }) as string; + return `s3://${bucket}/idea/bootstrap/bootstrap-${this.moduleId}-${this.deploymentId}.tar.gz`; + } + + buildIamRoles(): void { + this.bastionHostRole = new Role(this.context, `${this.moduleId}-role`, this.stack, { + description: 'IAM role assigned to the bastion-host', + assumedBy: ['ssm', 'ec2'], + managedPolicies: this.getEc2InstanceManagedPolicies(), + }); + this.bastionHostRole.attachInlinePolicy( + new Policy(this.context, 'bastion-host-policy', this.stack, { + policyTemplateName: 'bastion-host.yml', + }), + ); + this.bastionHostInstanceProfile = new InstanceProfile( + this.context, + `${this.moduleId}-instance-profile`, + this.stack, + [this.bastionHostRole], + ); + this.bastionHostInstanceProfile.node.addDependency(this.bastionHostRole); + } + + buildEc2Instance(): void { + const config = this.context.config; + const isPublic = config.getBool('bastion-host.public', false); + const baseOs = config.getString('bastion-host.base_os', undefined, { required: true }) as string; + const instanceAmi = config.getString('bastion-host.instance_ami', undefined, { required: true }) as string; + const instanceType = config.getString('bastion-host.instance_type', undefined, { required: true }) as string; + const volumeSize = config.getInt('bastion-host.volume_size', 200); + const keyPairName = config.getString('cluster.network.ssh_key_pair', undefined, { required: true }) as string; + const enableDetailedMonitoring = config.getBool('bastion-host.ec2.enable_detailed_monitoring', false); + const enableTerminationProtection = config.getBool('bastion-host.ec2.enable_termination_protection', false); + const metadataHttpTokens = config.getString('bastion-host.ec2.metadata_http_tokens', undefined, { + required: true, + }) as string; + + const httpsProxy = config.getString('cluster.network.https_proxy', ''); + const noProxy = config.getString('cluster.network.no_proxy', ''); + const proxyConfig: Record = + httpsProxy === '' ? {} : { http_proxy: httpsProxy, https_proxy: httpsProxy, no_proxy: noProxy }; + + const ebsKmsKey = lookupEbsKmsKey(this.context, this.stack); + const instanceProfileName = this.bastionHostInstanceProfile.instanceProfileName as string; + const securityGroup = this.cluster.getSecurityGroup(MODULE_BASTION_HOST) as ec2.ISecurityGroup; + + const subnetIds = + isPublic && this.cluster.publicSubnets.length > 0 + ? this.cluster.existingVpc.getPublicSubnetIds() + : this.cluster.existingVpc.getPrivateSubnetIds(); + + const blockDeviceName = ec2BlockDeviceName(baseOs); + const blockDeviceTypeString = config.getString('bastion-host.volume_type', 'gp3'); + // Anything other than gp3 becomes gp2 in the launch template, including a typo. + const blockDeviceVolumeType = + blockDeviceTypeString === 'gp3' ? ec2.EbsDeviceVolumeType.GP3 : ec2.EbsDeviceVolumeType.GP2; + + const userData = buildBootstrapUserData({ + awsRegion: this.awsRegion, + bootstrapPackageUri: this.bootstrapPackageUri, + installCommands: ['/bin/bash bastion-host/setup.sh'], + proxyConfig, + baseOs, + }); + + const launchTemplate = new ec2.LaunchTemplate(this.stack, `${this.moduleId}-lt`, { + instanceType: new ec2.InstanceType(instanceType), + machineImage: ec2.MachineImage.genericLinux({ [this.awsRegion]: instanceAmi }), + userData: ec2.UserData.custom(Fn.sub(userData)), + keyPair: lookupKeyPair(this.context, this.stack), + blockDevices: [ + { + deviceName: blockDeviceName, + volume: ec2.BlockDeviceVolume.ebs(volumeSize, { + encrypted: true, + kmsKey: ebsKmsKey, + volumeType: blockDeviceVolumeType, + }), + }, + ], + requireImdsv2: metadataHttpTokens === 'required', + }); + + this.ec2Instance = new ec2.CfnInstance(this.stack, `${this.moduleId}-instance`, { + blockDeviceMappings: [ + { + deviceName: blockDeviceName, + ebs: { encrypted: true, volumeSize, volumeType: blockDeviceTypeString }, + }, + ], + disableApiTermination: enableTerminationProtection, + iamInstanceProfile: instanceProfileName, + instanceType, + imageId: instanceAmi, + keyName: keyPairName, + launchTemplate: { + version: launchTemplate.latestVersionNumber, + launchTemplateId: launchTemplate.launchTemplateId, + }, + networkInterfaces: [ + { + deviceIndex: '0', + associatePublicIpAddress: isPublic, + groupSet: [securityGroup.securityGroupId], + subnetId: subnetIds[0] as string, + }, + ], + userData: Fn.base64(Fn.sub(userData)), + monitoring: enableDetailedMonitoring, + }); + Tags.of(this.ec2Instance).add('Name', this.buildResourceName(this.moduleId)); + Tags.of(this.ec2Instance).add(IDEA_TAG_NODE_TYPE, NODE_TYPE_INFRA); + this.addBackupTags(this.ec2Instance); + this.ec2Instance.node.addDependency(this.bastionHostInstanceProfile); + + this.addNagSuppression( + [{ rule_id: 'AwsSolutions-EC26', reason: 'EBS Encryption is enforced via Launch Template' }], + this.ec2Instance, + ); + + if (!enableDetailedMonitoring) { + this.addNagSuppression( + [ + { + rule_id: 'AwsSolutions-EC28', + reason: 'Detailed monitoring is a configurable option to save costs.', + }, + ], + this.ec2Instance, + ); + } + + if (!enableTerminationProtection) { + this.addNagSuppression( + [ + { + rule_id: 'AwsSolutions-EC29', + reason: + 'termination protection is a configurable option. Enable termination protection via AWS EC2 console after deploying the cluster if required.', + }, + ], + this.ec2Instance, + ); + } + } + + buildRoute53RecordSet(): void { + const hostname = this.context.config.getString('bastion-host.hostname', undefined, { + required: true, + }) as string; + this.clusterDnsRecordSet = new route53.RecordSet(this.stack, `${this.moduleId}-dns-record`, { + recordType: route53.RecordType.A, + target: route53.RecordTarget.fromIpAddresses(this.ec2Instance.attrPrivateIp), + ttl: Duration.minutes(5), + recordName: hostname, + zone: lookupClusterDns(this.context, this.stack), + }); + } + + buildClusterSettings(): void { + const clusterSettings: Record = { + deployment_id: this.deploymentId, + private_ip: this.ec2Instance.attrPrivateIp, + private_dns_name: this.ec2Instance.attrPrivateDnsName, + }; + if (this.context.config.getBool('bastion-host.public', false)) { + clusterSettings['public_ip'] = this.ec2Instance.attrPublicIp; + } + clusterSettings['instance_id'] = this.ec2Instance.ref; + clusterSettings['iam_role_arn'] = this.bastionHostRole.roleArn; + clusterSettings['instance_profile_arn'] = this.bastionHostInstanceProfile.ref; + this.updateClusterSettings(clusterSettings); + } +} + +export function buildStack(props: StackBuildProps): void { + new BastionHostStack(props); +} diff --git a/source/idea/ideactl/src/cdk/stacks/bootstrap.ts b/source/idea/ideactl/src/cdk/stacks/bootstrap.ts new file mode 100644 index 00000000..49068e3b --- /dev/null +++ b/source/idea/ideactl/src/cdk/stacks/bootstrap.ts @@ -0,0 +1,99 @@ +/** + * Rendering for the deployed `-bootstrap` stack. + * + * The static toolkit template is rendered and passed to `cdk bootstrap --template`. It is not a + * CDK construct. The template uses `cluster_name`, `aws_dns_suffix`, `aws_elb_account_id`, and + * `input_permissions_boundary`. + */ + +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import { Stack } from "aws-cdk-lib"; + +import { jinjaEnv, renderTemplate } from '../../config/jinja.ts'; +import { resourcesDir } from '../policy.ts'; +import type { StackBuildProps } from "../app.ts"; + +export interface BootstrapStackVars { + clusterName: string; + awsDnsSuffix: string; + /** `Utils.get_value_as_string(aws_region, region_elb_account_id_config)`: undefined if the region has no entry. */ + awsElbAccountId?: string; + /** `CdkInvoker.custom_permissions_boundary`; Python default `None`, which Jinja/nunjucks print as `''`. */ + inputPermissionsBoundary?: string; +} + +/** `resources/cdk/cdk_toolkit_stack.yml`'s directory, for the `FileSystemLoader` root. */ +export function bootstrapTemplateDir(): string { + return join(resourcesDir(), 'cdk'); +} + +/** `resources/config/region_elb_account_id.yml`. */ +export function regionElbAccountIdPath(): string { + return join(resourcesDir(), 'config', 'region_elb_account_id.yml'); +} + +/** + * `region_elb_account_id.yml` is a flat `region: account-id` mapping, one entry per line, `#` + * comments allowed. Values with leading-zero octal syntax normalize to decimal. Other values, + * including leading-zero values containing an 8 or 9, remain strings. + */ +function pyyamlOctalNormalize(value: string): string { + if (/^[-+]?0[0-7_]+$/.test(value)) { + const negative = value.startsWith('-'); + const digits = value.replace(/^[-+]/, '').replace(/_/g, ''); + return String(parseInt(digits, 8) * (negative ? -1 : 1)); + } + return value; +} + +function parseRegionElbAccountIdFile(text: string): Record { + const result: Record = {}; + for (const rawLine of text.split('\n')) { + const line = rawLine.trim(); + if (line === '' || line.startsWith('#')) continue; + const separator = line.indexOf(':'); + if (separator === -1) continue; + const key = line.slice(0, separator).trim(); + const value = line.slice(separator + 1).trim(); + if (key !== '') result[key] = pyyamlOctalNormalize(value); + } + return result; +} + +/** + * `Utils.get_value_as_string(aws_region, region_elb_account_id_config)`. Returns `undefined` + * (`None`) for a region absent from the file, which makes the bucket-policy + * `{% if aws_elb_account_id %}` branch fall through to the `logdelivery.elasticloadbalancing.*` + * service-principal form. + */ +export function elbAccountIdForRegion(region: string, path: string = regionElbAccountIdPath()): string | undefined { + const config = parseRegionElbAccountIdFile(readFileSync(path, 'utf-8')); + return config[region]; +} + +/** Renders `cdk_toolkit_stack.yml`. */ +export function renderBootstrapStack(vars: BootstrapStackVars, templateDir: string = bootstrapTemplateDir()): string { + const env = jinjaEnv(templateDir); + return renderTemplate(env, 'cdk_toolkit_stack.yml', { + cluster_name: vars.clusterName, + aws_dns_suffix: vars.awsDnsSuffix, + aws_elb_account_id: vars.awsElbAccountId, + input_permissions_boundary: vars.inputPermissionsBoundary, + }); +} + +/** + * Creates the empty app target required by `cdk bootstrap --app`. + * + * The deployed bootstrap template is rendered separately by `renderBootstrapStack`; adding it to + * this construct would change the CDK app contract. + */ +export function buildStack(props: StackBuildProps): void { + const stackName = `${props.ctx.clusterName}-bootstrap`; + new Stack(props.app, stackName, { + env: props.env, + stackName, + }); +} diff --git a/source/idea/ideactl/src/cdk/stacks/cluster-manager.ts b/source/idea/ideactl/src/cdk/stacks/cluster-manager.ts new file mode 100644 index 00000000..3bbc279c --- /dev/null +++ b/source/idea/ideactl/src/cdk/stacks/cluster-manager.ts @@ -0,0 +1,723 @@ +/** + * Nothing is exported and nothing is imported through CloudFormation: every cross-stack value is + * a literal read out of the cluster config at synth time, and the stack hands its own values back + * through the `Custom::ClusterSettings` resource at the end. The four `AWS::IAM::Policy` + * resources take CDK's default `PolicyName`, which is the logical id, so the construct ids are + * load bearing twice over. + * + * The bedrock block (managed policy, two custom resources with their lambdas, the delivery role) + * is gated on `cluster-manager.bedrock.enabled`. + */ + +import { CustomResource, Duration, Fn, Tags } from 'aws-cdk-lib'; +import * as asg from 'aws-cdk-lib/aws-autoscaling'; +import * as cognito from 'aws-cdk-lib/aws-cognito'; +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as elbv2 from 'aws-cdk-lib/aws-elasticloadbalancingv2'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager'; + +import { ArnBuilder } from '../../config/arn-builder.ts'; +import type { StackBuildProps } from '../app.ts'; +import { IdeaBaseStack } from '../base-stack.ts'; +import { IdeaCodeAsset } from '../code-asset.ts'; +import { IDEA_TAG_NAME, IDEA_TAG_NODE_TYPE, kmsKeyArn } from '../constructs/base.ts'; +import { + CustomResourceProvider, + LOG_RETENTION_DAYS, + ManagedPolicy, + Policy, + Role, + SQSQueue, +} from '../constructs/common.ts'; +import { OAuthClientIdAndSecret } from '../constructs/directory-service.ts'; +import { + ExistingSocaCluster, + lookupEbsKmsKey, + lookupKeyPair, +} from '../constructs/existing-resources.ts'; +import { WebPortalSecurityGroup } from '../constructs/network.ts'; +import { buildBootstrapUserData } from '../userdata.ts'; + +const MODULE_CLUSTER_MANAGER = 'cluster-manager'; +const NODE_TYPE_APP = 'app'; +const OS_AMAZONLINUX2 = 'amazonlinux2'; +const OS_AMAZONLINUX2023 = 'amazonlinux2023'; +/** `constants.SQS_MAX_RECEIVE_COUNT_CLUSTER_TASKS` / `SQS_MAX_RECEIVE_COUNT_NOTIFICATIONS`. */ +const SQS_MAX_RECEIVE_COUNT_CLUSTER_TASKS = 16; +const SQS_MAX_RECEIVE_COUNT_NOTIFICATIONS = 3; +/** `constants.SQS_VISIBILITY_TASKS` / `SQS_VISIBILITY_NOTIFICATIONS`. */ +const SQS_VISIBILITY_SECONDS = 30; + +/** `Utils.get_ec2_block_device_name`. */ +export function ec2BlockDeviceName(baseOs: string): string { + return baseOs === OS_AMAZONLINUX2 || baseOs === OS_AMAZONLINUX2023 ? '/dev/xvda' : '/dev/sda1'; +} + +export class ClusterManagerStack extends IdeaBaseStack { + readonly cluster: ExistingSocaCluster; + readonly arnBuilder: ArnBuilder; + readonly bootstrapPackageUri: string; + readonly userPool: cognito.IUserPool; + private readonly ecsEnabled: boolean; + private readonly hostsPresent: boolean; + + oauth2ClientSecret!: OAuthClientIdAndSecret; + jwtSigningSecret!: secretsmanager.CfnSecret; + clusterTasksSqsQueue!: SQSQueue; + notificationsSqsQueue!: SQSQueue; + clusterManagerRole!: Role; + projectRoleBoundary: ManagedPolicy | undefined; + bedrockInvocationLogGroupName: string | undefined; + bedrockInvocationLogRole: Role | undefined; + clusterManagerSecurityGroup!: WebPortalSecurityGroup; + autoScalingGroup!: asg.AutoScalingGroup; + webPortalEndpoint!: CustomResource; + externalEndpoint!: CustomResource; + internalEndpoint!: CustomResource; + + constructor(props: StackBuildProps) { + super({ + scope: props.app, + ctx: props.ctx, + moduleName: props.moduleName, + deploymentId: props.deploymentId, + terminationProtection: props.terminationProtection, + env: props.env, + }); + + this.bootstrapPackageUri = this.lookupBootstrapPackageUri(); + this.cluster = new ExistingSocaCluster(this.context, this.stack); + this.arnBuilder = new ArnBuilder(this.context.config); + this.ecsEnabled = this.context.config.getBool("ecs.enabled", false); + // `ecs.enabled` alone routes the endpoints to the container services and deletes the hosts that + // serve them in one change set, so nothing proves the new target before the old one is gone. + // With `ecs.retain_existing_hosts` the same deploy routes the endpoints and keeps the hosts, + // idle and unregistered, so a later deploy removes them once the containers are serving and + // turning the flag off puts the hosts back in service. + this.hostsPresent = + !this.ecsEnabled || this.context.config.getBool("ecs.retain_existing_hosts", false); + + this.userPool = this.lookupUserPool(); + + this.buildOauth2Client(); + this.buildJwtSigningSecret(); + this.buildAccessControlGroups(this.userPool); + this.buildSqsQueues(); + this.buildIamRoles(); + this.buildProjectRoleBoundary(); + this.buildBedrockInvocationLogging(); + this.buildSecurityGroups(); + if (this.hostsPresent) this.buildAutoScalingGroup(); + this.buildEndpoints(); + this.buildClusterSettings(); + } + + /** + * The bootstrap package is not a CDK asset: the CLI uploads it and passes its location as the + * `bootstrap_package_uri` context parameter. When absent, the deployment identifier derives + * the package name for a standalone synth. + */ + private lookupBootstrapPackageUri(): string { + const fromContext: unknown = this.stack.node.tryGetContext('bootstrap_package_uri'); + if (typeof fromContext === 'string' && fromContext !== '') return fromContext; + const bucket = this.context.config.getString('cluster.cluster_s3_bucket', undefined, { + required: true, + }) as string; + return `s3://${bucket}/idea/bootstrap/bootstrap-${this.moduleId}-${this.deploymentId}.tar.gz`; + } + + buildOauth2Client(): void { + const resourceServer = this.userPool.addResourceServer('resource-server', { + identifier: this.moduleId, + scopes: [ + new cognito.ResourceServerScope({ scopeName: 'read', scopeDescription: 'Allow Read Access' }), + new cognito.ResourceServerScope({ scopeName: 'write', scopeDescription: 'Allow Write Access' }), + ], + }); + + const refreshTokenValidityHours = this.context.config.getInt( + 'cluster-manager.oauth2_client.refresh_token_validity_hours', + 24, + ); + const client = this.userPool.addClient(`${this.moduleId}-client`, { + accessTokenValidity: Duration.hours(1), + authFlows: { adminUserPassword: true }, + generateSecret: true, + idTokenValidity: Duration.hours(1), + oAuth: { + flows: { clientCredentials: true }, + scopes: [ + cognito.OAuthScope.custom(`${this.moduleId}/read`), + cognito.OAuthScope.custom(`${this.moduleId}/write`), + ], + }, + refreshTokenValidity: Duration.hours(refreshTokenValidityHours), + userPoolClientName: this.moduleId, + }); + client.node.addDependency(resourceServer); + + // the lambda that reads the generated secret back lives in the identity-provider stack + const oauthCredentialsLambdaArn = this.context.config.getString( + 'identity-provider.cognito.oauth_credentials_lambda_arn', + undefined, + { required: true }, + ) as string; + const clientSecret = new CustomResource(this.stack, `${this.moduleId}-creds`, { + serviceToken: oauthCredentialsLambdaArn, + properties: { + UserPoolId: this.userPool.userPoolId, + ClientId: client.userPoolClientId, + }, + resourceType: 'Custom::GetOAuthCredentials', + }); + + this.oauth2ClientSecret = new OAuthClientIdAndSecret( + this.context, + this.moduleId, + MODULE_CLUSTER_MANAGER, + this.stack, + client.userPoolClientId, + clientSecret.getAttString('ClientSecret'), + ); + } + + /** + * Signs the temporary file-download tokens. An L1 with no removal policy: regenerating it + * invalidates every outstanding signed download URL. + */ + buildJwtSigningSecret(): void { + const kmsKeyId = this.context.config.getString('cluster.secretsmanager.kms_key_id'); + this.jwtSigningSecret = new secretsmanager.CfnSecret( + this.stack, + `${this.moduleId}-jwt-signing-secret`, + { + name: `${this.clusterName}-${this.moduleId}-jwt-signing-secret`, + description: `JWT signing secret for ${this.moduleId} secure file downloads`, + generateSecretString: { + secretStringTemplate: '{}', + generateStringKey: 'secret', + excludeCharacters: ' "\'\\/`', + includeSpace: false, + passwordLength: 64, + requireEachIncludedType: false, + }, + kmsKeyId: kmsKeyId === undefined ? undefined : kmsKeyArn(this.context, kmsKeyId), + tags: [ + { key: 'idea:ClusterName', value: this.clusterName }, + { key: 'idea:ModuleName', value: this.moduleId }, + { key: 'idea:SecretType', value: 'jwt-signing' }, + { key: 'idea:Purpose', value: 'file-download-authentication' }, + ], + }, + ); + } + + buildSqsQueues(): void { + const kmsKeyId = this.context.config.getString('cluster.sqs.kms_key_id'); + + const clusterTasksDlq = new SQSQueue(this.context, 'cluster-tasks-sqs-queue-dlq', this.stack, { + queueName: `${this.clusterName}-${this.moduleId}-tasks-dlq.fifo`, + fifo: true, + contentBasedDeduplication: true, + encryptionMasterKey: kmsKeyId, + isDeadLetterQueue: true, + }); + this.clusterTasksSqsQueue = new SQSQueue(this.context, 'cluster-tasks-sqs-queue', this.stack, { + queueName: `${this.clusterName}-${this.moduleId}-tasks.fifo`, + fifo: true, + contentBasedDeduplication: true, + encryptionMasterKey: kmsKeyId, + visibilityTimeout: Duration.seconds(SQS_VISIBILITY_SECONDS), + deadLetterQueue: { maxReceiveCount: SQS_MAX_RECEIVE_COUNT_CLUSTER_TASKS, queue: clusterTasksDlq }, + }); + // Both queues use the module name for their `Name` tag. + this.addCommonTags(this.clusterTasksSqsQueue); + this.addCommonTags(clusterTasksDlq); + + const notificationsDlq = new SQSQueue(this.context, 'notifications-sqs-queue-dlq', this.stack, { + queueName: `${this.clusterName}-${this.moduleId}-notifications-dlq.fifo`, + fifo: true, + contentBasedDeduplication: true, + encryptionMasterKey: kmsKeyId, + isDeadLetterQueue: true, + }); + this.notificationsSqsQueue = new SQSQueue(this.context, 'notifications-sqs-queue', this.stack, { + queueName: `${this.clusterName}-${this.moduleId}-notifications.fifo`, + fifo: true, + contentBasedDeduplication: true, + encryptionMasterKey: kmsKeyId, + visibilityTimeout: Duration.seconds(SQS_VISIBILITY_SECONDS), + deadLetterQueue: { maxReceiveCount: SQS_MAX_RECEIVE_COUNT_NOTIFICATIONS, queue: notificationsDlq }, + }); + this.addCommonTags(this.notificationsSqsQueue); + this.addCommonTags(notificationsDlq); + } + + buildIamRoles(): void { + this.clusterManagerRole = new Role(this.context, `${this.moduleId}-role`, this.stack, { + description: 'IAM role assigned to the cluster-manager', + assumedBy: ['ssm', 'ec2'], + managedPolicies: this.getEc2InstanceManagedPolicies(), + }); + this.clusterManagerRole.attachInlinePolicy( + new Policy(this.context, 'cluster-manager-policy', this.stack, { + policyTemplateName: 'cluster-manager.yml', + moduleId: this.moduleId, + }), + ); + } + + isBedrockEnabled(): boolean { + return this.context.config.getBool(`${this.moduleId}.bedrock.enabled`, false); + } + + /** Permissions ceiling for the per-project instance roles cluster-manager creates at runtime. */ + buildProjectRoleBoundary(): void { + if (!this.isBedrockEnabled()) return; + + this.projectRoleBoundary = new ManagedPolicy(this.context, 'project-role-boundary', this.stack, { + managedPolicyName: `${this.clusterName}-${this.awsRegion}-${this.moduleId}-project-boundary`, + description: 'Permissions boundary for IDEA per-project instance roles', + policyTemplateName: 'project-role-boundary.yml', + moduleId: this.moduleId, + }); + + // iam will not delete the boundary while runtime-created roles still reference it, so this + // resource depends on it, forcing cloudformation to run the lambda that clears those refs + // before the policy goes. + const detachBoundaries = new CustomResourceProvider( + this.context, + 'detach-project-boundaries', + this.stack, + { + ideaCodeAsset: new IdeaCodeAsset('idea_custom_resource_detach_project_boundaries'), + lambdaTimeoutSeconds: 300, + policyTemplateName: 'custom-resource-detach-project-boundaries.yml', + resourceType: 'ProjectRoleBoundaries', + }, + ).invoke('project-role-boundaries', { + RolePath: this.arnBuilder.projectRolePath, + BoundaryPolicyArn: this.arnBuilder.getProjectPermissionsBoundaryArn(), + }); + detachBoundaries.node.addDependency(this.projectRoleBoundary); + } + + /** + * Destination for bedrock invocation logs. The log group name is fixed, so a retain-then- + * recreate cycle would fail on "already exists": the custom resource creates it when absent, + * adopts it when present and never deletes it. + */ + buildBedrockInvocationLogging(): void { + if (!this.isBedrockEnabled()) return; + + const retentionInDays = this.context.config.getInt( + `${this.moduleId}.bedrock.invocation_logging.log_retention_in_days`, + 30, + ); + const validRetention = retentionInDays in LOG_RETENTION_DAYS; + if (!validRetention) { + console.warn( + `invalid bedrock.invocation_logging.log_retention_in_days: ${retentionInDays}. ` + + `valid values: ${Object.keys(LOG_RETENTION_DAYS).join(', ')}. ` + + 'leaving the retention of the log group unchanged.', + ); + } + this.bedrockInvocationLogGroupName = this.arnBuilder.bedrockInvocationLogGroupName; + const ensureProperties: Record = { + LogGroupName: this.bedrockInvocationLogGroupName, + }; + // a string, not a number: the custom resource's properties are compared as written + if (validRetention) ensureProperties['RetentionInDays'] = String(retentionInDays); + + new CustomResourceProvider(this.context, 'ensure-bedrock-log-group', this.stack, { + ideaCodeAsset: new IdeaCodeAsset('idea_custom_resource_ensure_log_group'), + lambdaTimeoutSeconds: 60, + policyTemplateName: 'custom-resource-ensure-log-group.yml', + resourceType: 'BedrockInvocationLogGroup', + }).invoke('bedrock-invocation-log-group', ensureProperties); + + this.bedrockInvocationLogRole = new Role(this.context, 'bedrock-invocation-logging', this.stack, { + description: 'IAM role assumed by Amazon Bedrock to deliver model invocation logs', + assumedBy: ['bedrock'], + }); + // confused deputy guard: only this account's bedrock may assume the role. one service + // principal means one trust statement, at index 0. the L2 does not model the condition. + (this.bedrockInvocationLogRole.node.defaultChild as iam.CfnRole).addOverride( + 'Properties.AssumeRolePolicyDocument.Statement.0.Condition', + { StringEquals: { 'aws:SourceAccount': { Ref: 'AWS::AccountId' } } }, + ); + this.bedrockInvocationLogRole.attachInlinePolicy( + new Policy(this.context, 'bedrock-invocation-logging-policy', this.stack, { + policyTemplateName: 'bedrock-invocation-logging.yml', + moduleId: this.moduleId, + }), + ); + } + + buildSecurityGroups(): void { + this.clusterManagerSecurityGroup = new WebPortalSecurityGroup( + this.context, + `${this.moduleId}-security-group`, + this.stack, + this.cluster.vpc, + this.cluster.getSecurityGroup('bastion-host') as ec2.ISecurityGroup, + this.cluster.getSecurityGroup('external-load-balancer') as ec2.ISecurityGroup, + ); + // The rule exists to reach a host. It goes when the last host does, not when routing moves. + if (!this.hostsPresent) this.removeBastionHostIngressRule(this.clusterManagerSecurityGroup); + } + + private removeBastionHostIngressRule(securityGroup: ec2.SecurityGroup): void { + const ingressRule = securityGroup.node.children.find( + (child) => + child instanceof ec2.CfnSecurityGroupIngress && + child.description === "Allow SSH from Bastion Host", + ); + if (ingressRule === undefined) { + throw new Error("Cluster-manager security group has no bastion SSH ingress rule"); + } + securityGroup.node.tryRemoveChild(ingressRule.node.id); + } + + private ecsTargetGroupArn(index: number): string { + const targetGroupArns = this.context.config.getList( + "ecs.cluster-manager.target_group_arns", + [], + { required: true }, + ); + const targetGroupArn = targetGroupArns[index]; + if (targetGroupArn === undefined || targetGroupArn === "") { + throw new Error(`ecs.cluster-manager.target_group_arns[${index}] is required when ecs.enabled is true`); + } + return targetGroupArn; + } + + private ecsServiceName(): string { + const serviceArn = this.context.config.getString( + "ecs.cluster-manager.service_arn", + undefined, + { required: true }, + ) as string; + const serviceName = serviceArn.split("/").at(-1); + if (serviceName === undefined || serviceName === "") { + throw new Error("ecs.cluster-manager.service_arn does not contain a service name"); + } + return serviceName; + } + + buildAutoScalingGroup(): void { + const config = this.context.config; + const keyPair = lookupKeyPair(this.context, this.stack); + const isPublic = + config.getBool('cluster-manager.ec2.autoscaling.public', false) && + this.cluster.publicSubnets.length > 0; + const baseOs = config.getString('cluster-manager.ec2.autoscaling.base_os', undefined, { + required: true, + }) as string; + const instanceAmi = config.getString('cluster-manager.ec2.autoscaling.instance_ami', undefined, { + required: true, + }) as string; + const instanceType = config.getString('cluster-manager.ec2.autoscaling.instance_type', undefined, { + required: true, + }) as string; + const volumeSize = config.getInt('cluster-manager.ec2.autoscaling.volume_size', 200); + const enableDetailedMonitoring = config.getBool( + 'cluster-manager.ec2.autoscaling.enable_detailed_monitoring', + false, + ); + const minCapacity = config.getInt('cluster-manager.ec2.autoscaling.min_capacity', 1); + const maxCapacity = config.getInt('cluster-manager.ec2.autoscaling.max_capacity', 3); + const cooldownMinutes = config.getInt('cluster-manager.ec2.autoscaling.cooldown_minutes', 5); + const newInstancesProtectedFromScaleIn = config.getBool( + 'cluster-manager.ec2.autoscaling.new_instances_protected_from_scale_in', + true, + ); + const elbHealthcheckGraceTimeMinutes = config.getInt( + 'cluster-manager.ec2.autoscaling.elb_healthcheck.grace_time_minutes', + 15, + ); + const scalingPolicyTargetUtilizationPercent = config.getInt( + 'cluster-manager.ec2.autoscaling.cpu_utilization_scaling_policy.target_utilization_percent', + 80, + ); + const scalingPolicyEstimatedInstanceWarmupMinutes = config.getInt( + 'cluster-manager.ec2.autoscaling.cpu_utilization_scaling_policy.estimated_instance_warmup_minutes', + 15, + ); + const rollingUpdateMaxBatchSize = config.getInt( + 'cluster-manager.ec2.autoscaling.rolling_update_policy.max_batch_size', + 1, + ); + const rollingUpdateMinInstancesInService = config.getInt( + 'cluster-manager.ec2.autoscaling.rolling_update_policy.min_instances_in_service', + 1, + ); + const rollingUpdatePauseTimeMinutes = config.getInt( + 'cluster-manager.ec2.autoscaling.rolling_update_policy.pause_time_minutes', + 15, + ); + const metadataHttpTokens = config.getString( + 'cluster-manager.ec2.autoscaling.metadata_http_tokens', + undefined, + { required: true }, + ) as string; + const httpsProxy = config.getString('cluster.network.https_proxy', ''); + const noProxy = config.getString('cluster.network.no_proxy', ''); + const proxyConfig: Record = + httpsProxy === '' ? {} : { http_proxy: httpsProxy, https_proxy: httpsProxy, no_proxy: noProxy }; + const ebsKmsKey = lookupEbsKmsKey(this.context, this.stack); + + const vpcSubnets: ec2.SubnetSelection = { + subnets: isPublic ? this.cluster.publicSubnets : this.cluster.privateSubnets, + }; + + const blockDeviceName = ec2BlockDeviceName(baseOs); + const blockDeviceTypeString = config.getString('cluster-manager.ec2.autoscaling.volume_type', 'gp3'); + const blockDeviceVolumeType = + blockDeviceTypeString === 'gp3' ? ec2.EbsDeviceVolumeType.GP3 : ec2.EbsDeviceVolumeType.GP2; + + const userData = buildBootstrapUserData({ + awsRegion: this.awsRegion, + bootstrapPackageUri: this.bootstrapPackageUri, + installCommands: ['/bin/bash cluster-manager/setup.sh'], + proxyConfig, + baseOs, + }); + + const launchTemplate = new ec2.LaunchTemplate(this.stack, `${this.moduleId}-lt`, { + instanceType: new ec2.InstanceType(instanceType), + machineImage: ec2.MachineImage.genericLinux({ [this.awsRegion]: instanceAmi }), + securityGroup: this.clusterManagerSecurityGroup, + userData: ec2.UserData.custom(Fn.sub(userData)), + keyPair, + blockDevices: [ + { + deviceName: blockDeviceName, + volume: ec2.BlockDeviceVolume.ebs(volumeSize, { + encrypted: true, + kmsKey: ebsKmsKey, + volumeType: blockDeviceVolumeType, + }), + }, + ], + role: this.clusterManagerRole, + requireImdsv2: metadataHttpTokens === 'required', + }); + + this.autoScalingGroup = new asg.AutoScalingGroup(this.stack, 'cluster-manager-asg', { + vpc: this.cluster.vpc, + vpcSubnets, + autoScalingGroupName: `${this.clusterName}-${this.moduleId}-asg`, + launchTemplate, + // Monitoring.BASIC is the zero value, which the L2 reads as "not set" and therefore + // accepts alongside a launch template. + instanceMonitoring: enableDetailedMonitoring ? asg.Monitoring.DETAILED : asg.Monitoring.BASIC, + groupMetrics: [asg.GroupMetrics.all()], + minCapacity, + maxCapacity, + newInstancesProtectedFromScaleIn, + cooldown: Duration.minutes(cooldownMinutes), + healthChecks: asg.HealthChecks.withAdditionalChecks({ + additionalTypes: [asg.AdditionalHealthCheckType.ELB], + gracePeriod: Duration.minutes(elbHealthcheckGraceTimeMinutes), + }), + updatePolicy: asg.UpdatePolicy.rollingUpdate({ + maxBatchSize: rollingUpdateMaxBatchSize, + minInstancesInService: rollingUpdateMinInstancesInService, + pauseTime: Duration.minutes(rollingUpdatePauseTimeMinutes), + }), + terminationPolicies: [asg.TerminationPolicy.DEFAULT], + }); + + this.autoScalingGroup.scaleOnCpuUtilization('cpu-utilization-scaling-policy', { + targetUtilizationPercent: scalingPolicyTargetUtilizationPercent, + estimatedInstanceWarmup: Duration.minutes(scalingPolicyEstimatedInstanceWarmupMinutes), + }); + + Tags.of(this.autoScalingGroup).add(IDEA_TAG_NODE_TYPE, NODE_TYPE_APP); + Tags.of(this.autoScalingGroup).add(IDEA_TAG_NAME, `${this.clusterName}-${this.moduleId}`); + this.autoScalingGroup.node.addDependency(this.clusterTasksSqsQueue); + this.autoScalingGroup.node.addDependency(this.notificationsSqsQueue); + + if (!enableDetailedMonitoring) { + this.addNagSuppression( + [ + { + rule_id: 'AwsSolutions-EC28', + reason: 'detailed monitoring is a configurable option to save costs', + }, + ], + this.autoScalingGroup, + true, + ); + } + + this.addNagSuppression( + [ + { + rule_id: 'AwsSolutions-AS3', + reason: 'ASG notifications scaling notifications can be managed via AWS Console', + }, + ], + this.autoScalingGroup, + ); + } + + buildEndpoints(): void { + const config = this.context.config; + const clusterEndpointsLambdaArn = config.getString('cluster.cluster_endpoints_lambda_arn', undefined, { + required: true, + }) as string; + const externalHttpsListenerArn = config.getString( + 'cluster.load_balancers.external_alb.https_listener_arn', + undefined, + { required: true }, + ) as string; + + // web portal endpoint: no conditions, it rewrites the external listener's default action + const defaultTargetGroupArn = this.ecsEnabled + ? this.ecsTargetGroupArn(2) + : new elbv2.CfnTargetGroup(this.stack, 'web-portal-target-group', { + port: 8443, + protocol: 'HTTPS', + targetType: 'instance', + vpcId: this.cluster.vpc.vpcId, + name: this.getTargetGroupName('web-portal'), + healthCheckPath: '/healthcheck', + }).ref; + + this.webPortalEndpoint = new CustomResource(this.stack, 'web-portal-endpoint', { + serviceToken: clusterEndpointsLambdaArn, + properties: { + endpoint_name: `${this.moduleId}-web-portal-endpoint`, + listener_arn: externalHttpsListenerArn, + priority: 0, + default_action: true, + actions: [{ Type: 'forward', TargetGroupArn: defaultTargetGroupArn }], + }, + resourceType: 'Custom::WebPortalEndpoint', + }); + + const externalEndpointPriority = config.getInt( + 'cluster-manager.endpoints.external.priority', + 0, + { required: true }, + ); + const externalEndpointPathPatterns = config.getList( + 'cluster-manager.endpoints.external.path_patterns', + [], + { required: true }, + ); + const externalTargetGroupArn = this.ecsEnabled + ? this.ecsTargetGroupArn(0) + : new elbv2.CfnTargetGroup( + this.stack, + `${this.moduleId}-external-target-group`, + { + port: 8443, + protocol: 'HTTPS', + targetType: 'instance', + vpcId: this.cluster.vpc.vpcId, + name: this.getTargetGroupName('cm-ext'), + healthCheckPath: '/healthcheck', + }, + ).ref; + this.externalEndpoint = new CustomResource(this.stack, 'external-endpoint', { + serviceToken: clusterEndpointsLambdaArn, + properties: { + endpoint_name: `${this.moduleId}-external-endpoint`, + listener_arn: externalHttpsListenerArn, + priority: externalEndpointPriority, + conditions: [{ Field: 'path-pattern', Values: externalEndpointPathPatterns }], + actions: [{ Type: 'forward', TargetGroupArn: externalTargetGroupArn }], + }, + resourceType: 'Custom::ClusterManagerEndpointExternal', + }); + + const internalHttpsListenerArn = config.getString( + 'cluster.load_balancers.internal_alb.https_listener_arn', + undefined, + { required: true }, + ) as string; + const internalEndpointPriority = config.getInt( + 'cluster-manager.endpoints.internal.priority', + 0, + { required: true }, + ); + const internalEndpointPathPatterns = config.getList( + 'cluster-manager.endpoints.internal.path_patterns', + [], + { required: true }, + ); + const internalTargetGroupArn = this.ecsEnabled + ? this.ecsTargetGroupArn(1) + : new elbv2.CfnTargetGroup( + this.stack, + `${this.moduleId}-internal-target-group`, + { + port: 8443, + protocol: 'HTTPS', + targetType: 'instance', + vpcId: this.cluster.vpc.vpcId, + name: this.getTargetGroupName('cm-int'), + healthCheckPath: '/healthcheck', + }, + ).ref; + this.internalEndpoint = new CustomResource(this.stack, 'internal-endpoint', { + serviceToken: clusterEndpointsLambdaArn, + properties: { + endpoint_name: `${this.moduleId}-internal-endpoint`, + listener_arn: internalHttpsListenerArn, + priority: internalEndpointPriority, + conditions: [{ Field: 'path-pattern', Values: internalEndpointPathPatterns }], + actions: [{ Type: 'forward', TargetGroupArn: internalTargetGroupArn }], + }, + resourceType: 'Custom::ClusterManagerEndpointInternal', + }); + + // Under the container flag these ARNs are the container target groups, which take IP targets, + // so a retained group registers with nothing and stands idle until a rollback recreates its own. + if (!this.ecsEnabled) { + // registered on the L1, and in an order of their own: web-portal, internal, external + (this.autoScalingGroup.node.defaultChild as asg.CfnAutoScalingGroup).targetGroupArns = [ + defaultTargetGroupArn, + internalTargetGroupArn, + externalTargetGroupArn, + ]; + } + } + + buildClusterSettings(): void { + const clusterSettings: Record = { + deployment_id: this.deploymentId, + client_id: this.oauth2ClientSecret.clientId.ref, + client_secret: this.oauth2ClientSecret.clientSecret.ref, + security_group_id: this.clusterManagerSecurityGroup.securityGroupId, + iam_role_arn: this.clusterManagerRole.roleArn, + task_queue_url: this.clusterTasksSqsQueue.queueUrl, + task_queue_arn: this.clusterTasksSqsQueue.queueArn, + notifications_queue_url: this.notificationsSqsQueue.queueUrl, + notifications_queue_arn: this.notificationsSqsQueue.queueArn, + asg_name: this.ecsEnabled ? this.ecsServiceName() : this.autoScalingGroup.autoScalingGroupName, + asg_arn: this.ecsEnabled + ? (this.context.config.getString("ecs.cluster-manager.service_arn", undefined, { + required: true, + }) as string) + : this.autoScalingGroup.autoScalingGroupArn, + }; + + if (this.bedrockInvocationLogGroupName !== undefined) { + clusterSettings['bedrock.invocation_log_group_name'] = this.bedrockInvocationLogGroupName; + clusterSettings['bedrock.invocation_log_role_arn'] = (this.bedrockInvocationLogRole as Role).roleArn; + } + + clusterSettings['jwt_signing_secret_arn'] = this.jwtSigningSecret.ref; + + this.updateClusterSettings(clusterSettings); + } +} + +export function buildStack(props: StackBuildProps): void { + new ClusterManagerStack(props); +} diff --git a/source/idea/ideactl/src/cdk/stacks/cluster.ts b/source/idea/ideactl/src/cdk/stacks/cluster.ts new file mode 100644 index 00000000..4234ca70 --- /dev/null +++ b/source/idea/ideactl/src/cdk/stacks/cluster.ts @@ -0,0 +1,1116 @@ +/** + * The base infrastructure every other module stack reads out of the cluster-settings table: VPC, + * both load balancers and their + * listeners, the WAF trio, the self-signed certificates, the prefix list, the shared IAM policies + * and roles, and the four custom-resource lambdas. + * + * Build order is load-bearing: it fixes template order, the + * `{IndirectPeer}` counters on the security groups, and the `settings` map insertion order. + * + * Two behaviours are load-bearing: + * + * - the three DCV broker listeners read `cluster.external_alb.dcv_broker_*_listener_arn`, keys + * nothing writes (the stack writes `cluster.load_balancers.internal_alb.dcv_broker_*`), so they + * always synthesize a fixed-response and the virtual-desktop-controller stack sets the real + * default action out of band through the cluster-endpoints lambda; + * - the private hosted zone's `Name` tag carries the cluster prefix twice. + */ + +import { CustomResource, Duration, RemovalPolicy } from 'aws-cdk-lib'; +import * as backup from 'aws-cdk-lib/aws-backup'; +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as elbv2 from 'aws-cdk-lib/aws-elasticloadbalancingv2'; +import * as events from 'aws-cdk-lib/aws-events'; +import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as kms from 'aws-cdk-lib/aws-kms'; +import * as route53 from 'aws-cdk-lib/aws-route53'; +import * as route53Targets from 'aws-cdk-lib/aws-route53-targets'; +import * as wafv2 from 'aws-cdk-lib/aws-wafv2'; + +import { ConfigKeyNotFound, isEmpty } from '../../config/cluster-config.ts'; +import type { ClusterConfig } from '../../config/cluster-config.ts'; +import type { StackBuildProps } from '../app.ts'; +import { IdeaBaseStack } from '../base-stack.ts'; +import { IdeaCodeAsset } from '../code-asset.ts'; +import { kmsKeyArn } from '../constructs/base.ts'; +import { BackupPlan } from '../constructs/backup.ts'; +import { + CreateTagsCustomResource, + LambdaFunction, + ManagedPolicy, + Policy, + Role, + SNSTopic, +} from '../constructs/common.ts'; +import { PrivateHostedZone } from '../constructs/dns.ts'; +import { ExistingVpc, lookupClusterS3Bucket } from '../constructs/existing-resources.ts'; +import { + BastionHostSecurityGroup, + DefaultClusterSecurityGroup, + ExternalLoadBalancerSecurityGroup, + InternalLoadBalancerSecurityGroup, + SecurityGroup, + Vpc, + VpcEndpointSecurityGroup, + VpcGatewayEndpoint, + VpcInterfaceEndpoint, + WebAcl, +} from '../constructs/network.ts'; + +const LOG_RETENTION_ROLE_NAME = 'log-retention'; +const MODULE_VIRTUAL_DESKTOP_CONTROLLER = 'virtual-desktop-controller'; +const DEFAULT_SSL_POLICY = 'ELBSecurityPolicy-FS-1-2-Res-2020-10'; +/** `constants.CAVEATS['ROUTE53_CROSS_ZONE_ALIAS_RESTRICTED_REGION_LIST']`. */ +const ROUTE53_CROSS_ZONE_ALIAS_RESTRICTED_REGIONS = ['us-gov-east-1', 'us-gov-west-1']; + +/** `cdk.RemovalPolicy(value)` takes the member name; the TS enum values are other strings. */ +function removalPolicyByName(name: string): RemovalPolicy { + if (!Object.prototype.hasOwnProperty.call(RemovalPolicy, name)) { + throw new Error(`'${name}' is not a valid RemovalPolicy`); + } + return RemovalPolicy[name as keyof typeof RemovalPolicy]; +} + +/** `config.get_int(key, required=True)` / `get_bool(key, required=True)`: missing or NULL raises. */ +function requiredInt(config: ClusterConfig, key: string): number { + const value = config.getInt(key); + if (value === undefined) throw new ConfigKeyNotFound(`'${key}', key: ${key}`); + return value; +} + +function requiredBool(config: ClusterConfig, key: string): boolean { + const value = config.getBool(key); + if (value === undefined) throw new ConfigKeyNotFound(`'${key}', key: ${key}`); + return value; +} + +interface DescribedListener { + DefaultActions?: { Type?: string; TargetGroupArn?: string }[]; +} + +/** + * The listener ARNs this stack reads back at synth time. Both keys are optional: on a first + * deploy neither listener exists yet, and nothing ever writes the three DCV broker keys. + */ +const LISTENER_ARN_KEYS = [ + 'cluster.load_balancers.external_alb.https_listener_arn', + 'cluster.external_alb.dcv_broker_client_listener_arn', + 'cluster.external_alb.dcv_broker_agent_listener_arn', + 'cluster.external_alb.dcv_broker_gateway_listener_arn', +]; + +/** + * `elbv2:DescribeListeners` for every listener ARN in the settings, before the tree is built. + * `SynthReads` is asynchronous and a stack constructor is not, so `buildStack` does the reads and + * hands the results in. A read that fails fails the synth: guessing here resets the external + * listener's default action and takes the web portal offline. + */ +async function describeListeners(ctx: StackBuildProps['ctx']): Promise> { + const listeners = new Map(); + for (const key of LISTENER_ARN_KEYS) { + const arn = ctx.config.getString(key); + if (isEmpty(arn) || listeners.has(arn as string)) continue; + listeners.set(arn as string, await ctx.synthReads.describeListener(arn as string)); + } + return listeners; +} + +export class ClusterStack extends IdeaBaseStack { + /** The listeners `buildStack` read back, by ARN. */ + private readonly describedListeners: Map; + private newVpc: Vpc | undefined; + private existingVpc: ExistingVpc | undefined; + + vpcInterfaceEndpoints: Record | undefined; + + selfSignedCertificateLambda: LambdaFunction | undefined; + externalCertificate: CustomResource | undefined; + internalCertificate: CustomResource | undefined; + + backupRole: Role | undefined; + backupVault: backup.BackupVault | undefined; + backupPlan: BackupPlan | undefined; + + clusterEndpointsLambda: LambdaFunction | undefined; + externalAlb: elbv2.ApplicationLoadBalancer | undefined; + externalAlbHttpsListener: elbv2.CfnListener | undefined; + externalAlbWafWebAcl: WebAcl | undefined; + internalAlb: elbv2.ApplicationLoadBalancer | undefined; + internalAlbHttpsListener: elbv2.CfnListener | undefined; + internalAlbDcvBrokerClientListener: elbv2.CfnListener | undefined; + internalAlbDcvBrokerAgentListener: elbv2.CfnListener | undefined; + internalAlbDcvBrokerGatewayListener: elbv2.CfnListener | undefined; + internalAlbDnsRecordSet: route53.RecordSet | route53.CnameRecord | undefined; + + privateHostedZone: PrivateHostedZone | undefined; + clusterPrefixList: ec2.CfnPrefixList | undefined; + readonly securityGroups: Record = {}; + readonly roles: Record = {}; + amazonSsmManagedInstanceCorePolicy: ManagedPolicy | undefined; + cloudWatchAgentServerPolicy: ManagedPolicy | undefined; + amazonPrometheusRemoteWritePolicy: ManagedPolicy | undefined; + + solutionMetricsLambda: LambdaFunction | undefined; + clusterSettingsLambda: LambdaFunction | undefined; + + ec2EventsSnsTopic: SNSTopic | undefined; + + constructor(props: StackBuildProps, describedListeners: Map = new Map()) { + super({ + scope: props.app, + ctx: props.ctx, + moduleName: props.moduleName, + deploymentId: props.deploymentId, + terminationProtection: props.terminationProtection, + env: props.env, + }); + + this.describedListeners = describedListeners; + this.buildBackups(); + this.buildPolicies(); + this.buildRoles(); + this.buildSelfSignedCertificatesLambda(); + this.buildSelfSignedCertificates(); + this.buildClusterSettingsLambda(); + this.buildVpc(); + this.buildClusterPrefixList(); + this.buildSecurityGroups(); + this.buildPrivateHostedZone(); + this.buildEc2NotificationModule(); + this.buildClusterEndpoints(); + this.buildVpcEndpoints(); + this.buildSolutionMetricsLambda(); + this.buildClusterSettings(); + } + + private get config() { + return this.context.config; + } + + private useExistingVpc(): boolean { + return this.config.getBool('cluster.network.use_existing_vpc', false); + } + + get vpc(): ec2.IVpc { + return this.useExistingVpc() ? (this.existingVpc as ExistingVpc).vpc : (this.newVpc as Vpc); + } + + publicSubnets(): ec2.ISubnet[] { + return this.useExistingVpc() ? (this.existingVpc as ExistingVpc).getPublicSubnets() : this.vpc.publicSubnets; + } + + privateSubnets(): ec2.ISubnet[] { + return this.useExistingVpc() ? (this.existingVpc as ExistingVpc).getPrivateSubnets() : this.vpc.privateSubnets; + } + + // --- backups --------------------------------------------------------------------------------- + + buildBackups(): void { + if (!this.config.getBool('cluster.backups.enabled', false)) return; + + const enableRestore = this.config.getBool('cluster.backups.enable_restore', true); + + // Backup policies have to be managed policies: as inline policies they exceed the 10240 byte + // limit on the role. + const backupCreatePolicy = new ManagedPolicy(this.context, 'backup-create-policy', this.stack, { + managedPolicyName: `${this.clusterName}-${this.awsRegion}-backup-create`, + description: + 'Provides AWS Backup permission to create backups on your behalf across AWS services', + policyTemplateName: 'backup-create.yml', + }); + const backupS3CreatePolicy = new ManagedPolicy(this.context, 'backup-s3-create-policy', this.stack, { + managedPolicyName: `${this.clusterName}-${this.awsRegion}-backup-s3-create`, + description: + 'Policy containing permissions necessary for AWS Backup to backup data in any S3 bucket. ' + + 'This includes read access to all S3 objects and any decrypt access for all KMS keys.', + policyTemplateName: 'backup-s3-create.yml', + }); + + let backupRestorePolicy: ManagedPolicy | undefined; + let backupS3RestorePolicy: ManagedPolicy | undefined; + if (enableRestore) { + backupRestorePolicy = new ManagedPolicy(this.context, 'backup-restore-policy', this.stack, { + managedPolicyName: `${this.clusterName}-${this.awsRegion}-backup-restore`, + description: + 'Provides AWS Backup permission to perform restores on your behalf across AWS services. ' + + 'This policy includes permissions to create and delete AWS resources, such as EBS volumes, RDS instances, and EFS file systems, which are part of the restore process.', + policyTemplateName: 'backup-restore.yml', + }); + backupS3RestorePolicy = new ManagedPolicy(this.context, 'backup-s3-restore-policy', this.stack, { + managedPolicyName: `${this.clusterName}-${this.awsRegion}-backup-s3-restore`, + description: + 'Policy containing permissions necessary for AWS Backup to restore a S3 backup to a bucket. ' + + 'This includes read/write permissions to all S3 buckets, and permissions to GenerateDataKey and DescribeKey for all KMS keys.', + policyTemplateName: 'backup-s3-restore.yml', + }); + } + + const backupRole = new Role(this.context, `${this.moduleId}-backup-role`, this.stack, { + description: 'Role used by AWS Backup to authenticate when backing or restoring the resources', + assumedBy: ['backup'], + }); + backupRole.addManagedPolicy(backupCreatePolicy); + backupRole.addManagedPolicy(backupS3CreatePolicy); + if (enableRestore) { + backupRole.addManagedPolicy(backupRestorePolicy as ManagedPolicy); + backupRole.addManagedPolicy(backupS3RestorePolicy as ManagedPolicy); + } + + const backupVaultRemovalPolicy = this.config.getString( + 'cluster.backups.backup_vault.removal_policy', + 'DESTROY', + ); + const backupVaultKmsKeyId = this.config.getString('cluster.backups.backup_vault.kms_key_id'); + const backupVaultEncryptionKey = isEmpty(backupVaultKmsKeyId) + ? undefined + : kms.Key.fromKeyArn(this.stack, 'backup-vault-kms-key', kmsKeyArn(this.context, backupVaultKmsKeyId as string)); + const backupVault = new backup.BackupVault(this.stack, 'backup-vault', { + backupVaultName: `${this.clusterName}-${this.moduleId}-backup-vault`, + encryptionKey: backupVaultEncryptionKey, + removalPolicy: removalPolicyByName(backupVaultRemovalPolicy), + }); + + // The immutable reference keeps BackupSelection from attaching the AWS Backup managed + // policies to the role; the copied policies above are what the role carries. + const backupPlan = new BackupPlan(this.stack, { + backupPlanName: `${this.clusterName}-${this.moduleId}`, + backupPlanConfig: this.config.getConfig('cluster.backups.backup_plan'), + backupVault, + backupRole: backupRole.withoutPolicyUpdates(), + }); + + backupPlan.backupSelection.node.addDependency(backupCreatePolicy); + backupPlan.backupSelection.node.addDependency(backupS3CreatePolicy); + if (enableRestore) { + backupPlan.backupSelection.node.addDependency(backupRestorePolicy as ManagedPolicy); + backupPlan.backupSelection.node.addDependency(backupS3RestorePolicy as ManagedPolicy); + } + backupPlan.backupSelection.node.addDependency(backupRole); + + this.backupRole = backupRole; + this.backupVault = backupVault; + this.backupPlan = backupPlan; + } + + // --- policies and roles ---------------------------------------------------------------------- + + buildPolicies(): void { + this.amazonSsmManagedInstanceCorePolicy = new ManagedPolicy( + this.context, + 'amazon-ssm-managed-instance-core', + this.stack, + { + managedPolicyName: `${this.clusterName}-${this.awsRegion}-amazon-ssm-managed-instance-core`, + description: + 'The policy for Amazon EC2 Role to enable AWS Systems Manager service core functionality.', + policyTemplateName: 'amazon-ssm-managed-instance-core.yml', + }, + ); + + this.cloudWatchAgentServerPolicy = new ManagedPolicy( + this.context, + 'cloud-watch-agent-server-policy', + this.stack, + { + managedPolicyName: `${this.clusterName}-${this.awsRegion}-cloud-watch-agent-server-policy`, + description: 'Permissions required to use AmazonCloudWatchAgent on servers', + policyTemplateName: 'cloud-watch-agent-server-policy.yml', + }, + ); + + if (this.isMetricsProviderAmazonManagedPrometheus()) { + this.amazonPrometheusRemoteWritePolicy = new ManagedPolicy( + this.context, + 'amazon-prometheus-remote-write-access', + this.stack, + { + managedPolicyName: `${this.clusterName}-${this.awsRegion}-amazon-prometheus-remote-write-access`, + description: 'Grants write only access to AWS Managed Prometheus workspaces', + policyTemplateName: 'amazon-prometheus-remote-write-access.yml', + }, + ); + } + } + + buildRoles(): void { + // The policy is a constructor argument, so it is created before the role and precedes it in + // the template. + this.roles[LOG_RETENTION_ROLE_NAME] = new Role(this.context, LOG_RETENTION_ROLE_NAME, this.stack, { + description: 'log retention role for CDK custom resources', + assumedBy: ['lambda'], + inlinePolicies: [ + new Policy(this.context, 'LogRetention', this.stack, { policyTemplateName: 'log-retention.yml' }), + ], + }); + } + + private get logRetentionRole(): iam.IRole { + return this.roles[LOG_RETENTION_ROLE_NAME] as Role; + } + + private addPythonRuntimeSuppression(lambdaFunction: LambdaFunction): void { + this.addNagSuppression( + [{ rule_id: 'AwsSolutions-L1', reason: 'Python Runtime is selected for stability.' }], + lambdaFunction, + ); + } + + // --- certificates ---------------------------------------------------------------------------- + + buildSelfSignedCertificatesLambda(): void { + const lambdaName = 'self-signed-certificate'; + + const policy = new Policy(this.context, `${lambdaName}-policy`, this.stack, { + policyTemplateName: 'custom-resource-self-signed-certificate.yml', + }); + const role = new Role(this.context, `${lambdaName}-role`, this.stack, { + description: `Role for generating self-signed certificates Lambda function for Cluster: ${this.clusterName}`, + assumedBy: ['lambda'], + }); + role.attachInlinePolicy(policy); + + this.selfSignedCertificateLambda = new LambdaFunction(this.context, lambdaName, this.stack, { + ideaCodeAsset: new IdeaCodeAsset('idea_custom_resource_self_signed_certificate'), + description: 'Manage self-signed certificates for IDEA cluster infrastructure', + timeoutSeconds: 180, + role, + logRetentionRole: this.logRetentionRole, + }); + // Without the explicit dependencies, stack deletion races the policy against the function. + this.selfSignedCertificateLambda.node.addDependency(policy); + this.selfSignedCertificateLambda.node.addDependency(role); + this.addPythonRuntimeSuppression(this.selfSignedCertificateLambda); + } + + buildSelfSignedCertificates(): void { + const serviceToken = (this.selfSignedCertificateLambda as LambdaFunction).functionArn; + const kmsKeyId = this.config.getString('cluster.secretsmanager.kms_key_id'); + + if (!this.config.getBool('cluster.load_balancers.external_alb.certificates.provided', false)) { + this.externalCertificate = new CustomResource( + this.stack, + `${this.clusterName}-${this.moduleId}-external-cert`, + { + serviceToken, + properties: { + domain_name: `${this.clusterName}.idea.default`, + certificate_name: `${this.clusterName}-external`, + create_acm_certificate: true, + kms_key_id: kmsKeyId, + tags: { + Name: `${this.clusterName} external alb certs`, + 'idea:ClusterName': this.clusterName, + }, + }, + resourceType: 'Custom::SelfSignedCertificateExternal', + }, + ); + this.externalCertificate.node.addDependency(this.selfSignedCertificateLambda as LambdaFunction); + } + + const privateHostedZoneName = this.config.getString('cluster.route53.private_hosted_zone_name', undefined, { + required: true, + }) as string; + this.internalCertificate = new CustomResource( + this.stack, + `${this.clusterName}-${this.moduleId}-internal-cert`, + { + serviceToken, + properties: { + domain_name: `*.${privateHostedZoneName}`, + certificate_name: `${this.clusterName}-internal`, + create_acm_certificate: true, + kms_key_id: kmsKeyId, + tags: { + Name: `${this.clusterName} internal alb certs`, + 'idea:ClusterName': this.clusterName, + }, + }, + resourceType: 'Custom::SelfSignedCertificateInternal', + }, + ); + this.internalCertificate.node.addDependency(this.selfSignedCertificateLambda as LambdaFunction); + } + + // --- lambdas --------------------------------------------------------------------------------- + + buildClusterSettingsLambda(): void { + const lambdaName = 'cluster-settings'; + + const role = new Role(this.context, `${lambdaName}-role`, this.stack, { + description: `Role for cluster-settings lambda function for Cluster: ${this.clusterName}`, + assumedBy: ['lambda'], + }); + const policy = new Policy(this.context, `${lambdaName}-policy`, this.stack, { + policyTemplateName: 'custom-resource-update-cluster-settings.yml', + }); + + this.clusterSettingsLambda = new LambdaFunction(this.context, lambdaName, this.stack, { + ideaCodeAsset: new IdeaCodeAsset('idea_custom_resource_update_cluster_settings'), + description: 'Update cluster settings during cluster module deployment', + timeoutSeconds: 180, + role, + logRetentionRole: this.logRetentionRole, + }); + role.attachInlinePolicy(policy); + this.clusterSettingsLambda.node.addDependency(policy); + this.clusterSettingsLambda.node.addDependency(role); + this.addPythonRuntimeSuppression(this.clusterSettingsLambda); + } + + buildSolutionMetricsLambda(): void { + const lambdaName = 'solution-metrics'; + + const role = new Role(this.context, `${lambdaName}-role`, this.stack, { + description: `Role for solution-metrics metrics Lambda function for Cluster: ${this.clusterName}`, + assumedBy: ['lambda'], + }); + const policy = new Policy(this.context, `${lambdaName}-policy`, this.stack, { + policyTemplateName: 'solution-metrics-lambda-function.yml', + }); + + this.solutionMetricsLambda = new LambdaFunction(this.context, lambdaName, this.stack, { + ideaCodeAsset: new IdeaCodeAsset('idea_solution_metrics'), + description: 'Send anonymous Metrics to AWS', + timeoutSeconds: 180, + role, + logRetentionRole: this.logRetentionRole, + }); + role.attachInlinePolicy(policy); + this.solutionMetricsLambda.node.addDependency(policy); + this.solutionMetricsLambda.node.addDependency(role); + this.addPythonRuntimeSuppression(this.solutionMetricsLambda); + } + + // --- network --------------------------------------------------------------------------------- + + buildVpc(): void { + // An existing VPC is looked up and never modified: no VPC resources are created and the + // administrator carries the same configuration into every upgrade. + if (this.useExistingVpc()) { + this.existingVpc = new ExistingVpc(this.context, 'existing-vpc', this.stack); + } else { + this.newVpc = new Vpc(this.context, 'vpc', this.stack); + } + } + + buildPrivateHostedZone(): void { + this.privateHostedZone = new PrivateHostedZone(this.context, this.stack, this.vpc); + } + + /** + * The cluster prefix list is the one place administrators manage external access from. Entries + * are added out of band by `idea-admin utils cluster-prefix-list`, so the stack only ever + * creates the list and adds the configured client IPs; it never removes an entry. + */ + buildClusterPrefixList(): void { + const maxEntries = this.config.getInt('cluster.network.cluster_prefix_list_max_entries', 10); + this.clusterPrefixList = new ec2.CfnPrefixList(this.stack, 'cluster-prefix-list', { + addressFamily: 'IPv4', + maxEntries, + prefixListName: `${this.clusterName}-prefix-list`, + }); + this.addCommonTags(this.clusterPrefixList); + + const clientIps = this.config.getList('cluster.network.client_ip', []); + if (isEmpty(clientIps)) return; + + const lambdaName = 'update-cluster-prefix-list'; + const policy = new Policy(this.context, `${lambdaName}-policy`, this.stack, { + policyTemplateName: 'custom-resource-update-cluster-prefix-list.yml', + }); + const role = new Role(this.context, `${lambdaName}-role`, this.stack, { + description: `Role to manage cluster prefix list Lambda function for Cluster: ${this.clusterName}`, + assumedBy: ['lambda'], + }); + role.attachInlinePolicy(policy); + + const lambdaFunction = new LambdaFunction(this.context, lambdaName, this.stack, { + ideaCodeAsset: new IdeaCodeAsset('idea_custom_resource_update_cluster_prefix_list'), + description: 'Manage Cluster Prefix List', + timeoutSeconds: 180, + role, + logRetentionRole: this.logRetentionRole, + }); + lambdaFunction.node.addDependency(policy); + lambdaFunction.node.addDependency(role); + lambdaFunction.node.addDependency(this.clusterPrefixList); + this.addPythonRuntimeSuppression(lambdaFunction); + + const entries = clientIps.map((clientIp) => ({ + Cidr: clientIp.includes('/') ? clientIp : `${clientIp}/32`, + Description: 'Allow access to cluster from Client IP', + })); + + const customResource = new CustomResource( + this.stack, + `${this.clusterName}-${this.moduleId}-cluster-prefix-list`, + { + serviceToken: lambdaFunction.functionArn, + properties: { + prefix_list_id: this.clusterPrefixList.attrPrefixListId, + add_entries: entries, + }, + resourceType: 'Custom::ClusterPrefixList', + }, + ); + customResource.node.addDependency(lambdaFunction); + } + + buildSecurityGroups(): void { + const prefixList = this.clusterPrefixList as ec2.CfnPrefixList; + + this.securityGroups['cluster'] = new DefaultClusterSecurityGroup( + this.context, + 'default-security-group', + this.stack, + this.vpc, + ); + + const bastionHostSecurityGroup = new BastionHostSecurityGroup( + this.context, + 'bastion-host-security-group', + this.stack, + this.vpc, + prefixList.attrPrefixListId, + ); + this.securityGroups['bastion-host'] = bastionHostSecurityGroup; + + const externalLoadBalancerSecurityGroup = new ExternalLoadBalancerSecurityGroup( + this.context, + 'external-load-balancer-security-group', + this.stack, + this.vpc, + prefixList.attrPrefixListId, + bastionHostSecurityGroup, + ); + this.securityGroups['external-load-balancer'] = externalLoadBalancerSecurityGroup; + + this.securityGroups['internal-load-balancer'] = new InternalLoadBalancerSecurityGroup( + this.context, + 'internal-load-balancer-security-group', + this.stack, + this.vpc, + ); + + const natEips: ec2.CfnEIP[] = []; + for (const subnet of this.vpc.publicSubnets) { + const eip = subnet.node.tryFindChild('EIP'); + if (eip !== undefined) natEips.push(eip as ec2.CfnEIP); + } + if (natEips.length > 0) { + externalLoadBalancerSecurityGroup.addNatGatewayIpsIngressRule(natEips); + } + + if (!this.useExistingVpc() && this.config.getBool('cluster.network.use_vpc_endpoints', false)) { + this.securityGroups['vpc-endpoint'] = new VpcEndpointSecurityGroup( + this.context, + 'vpc-endpoint-security-group', + this.stack, + this.vpc, + ); + } + } + + buildVpcEndpoints(): void { + if (!this.config.getBool('cluster.network.use_vpc_endpoints', false)) return; + if (this.useExistingVpc()) return; + + const gatewayEndpoints: Record = {}; + const interfaceEndpoints: Record = {}; + + const createTags = new CreateTagsCustomResource(this.context, this.stack, this.logRetentionRole); + + for (const service of this.config.getList('cluster.network.vpc_gateway_endpoints', [])) { + gatewayEndpoints[service] = new VpcGatewayEndpoint(this.context, this.stack, service, this.vpc, createTags); + } + + const configured = this.config.getConfig('cluster.network.vpc_interface_endpoints', {}) ?? {}; + for (const [service, endpointConfig] of Object.entries(configured)) { + const enabled = (endpointConfig as Record | undefined)?.['enabled']; + if (enabled !== true) continue; + interfaceEndpoints[service] = new VpcInterfaceEndpoint( + this.context, + this.stack, + service, + this.vpc, + this.securityGroups['vpc-endpoint'] as SecurityGroup, + createTags, + ); + } + + this.vpcInterfaceEndpoints = interfaceEndpoints; + } + + // --- ec2 state change notifications ------------------------------------------------------------ + + buildEc2NotificationModule(): void { + this.ec2EventsSnsTopic = new SNSTopic(this.context, 'cluster-ec2-state-change-sns-topic', this.stack, { + displayName: `${this.clusterName}-${this.moduleId}-ec2-state-change-sns-topic`, + topicName: `${this.clusterName}-${this.moduleId}-ec2-state-change-sns-topic`, + masterKey: this.config.getString('cluster.sns.kms_key_id'), + }); + this.addCommonTags(this.ec2EventsSnsTopic); + + const lambdaName = `${this.moduleId}-ec2state-event-transformer`; + const role = new Role(this.context, `${lambdaName}-role`, this.stack, { + description: `${lambdaName}-role`, + assumedBy: ['lambda'], + }); + role.attachInlinePolicy( + new Policy(this.context, `${lambdaName}-policy`, this.stack, { + policyTemplateName: 'ec2state-event-transformer.yml', + }), + ); + + const transformer = new LambdaFunction(this.context, lambdaName, this.stack, { + description: `${this.moduleId} lambda to intercept all ec2 state change events and transform to the required event object`, + environment: { + IDEA_EC2_STATE_SNS_TOPIC_ARN: this.ec2EventsSnsTopic.topicArn, + IDEA_CLUSTER_NAME_TAG_KEY: 'idea:ClusterName', + IDEA_CLUSTER_NAME_TAG_VALUE: this.clusterName, + IDEA_TAG_PREFIX: 'idea:', + }, + timeoutSeconds: 180, + role, + ideaCodeAsset: new IdeaCodeAsset('idea_ec2_state_event_transformation_lambda'), + }); + this.addPythonRuntimeSuppression(transformer); + + const rule = new events.Rule(this.stack, `${this.clusterName}-ec2-state-monitoring-rule`, { + enabled: true, + ruleName: `${this.clusterName}-${this.moduleId}-ec2-state-monitoring-rule`, + description: 'Event Rule to monitor state changes on EC2 Instances', + eventPattern: { + source: ['aws.ec2'], + detailType: ['EC2 Instance State-change Notification'], + region: [this.awsRegion], + }, + }); + rule.addTarget(new eventsTargets.LambdaFunction(transformer)); + } + + // --- load balancers -------------------------------------------------------------------------- + + /** + * A listener needs a default action at create time, but the cluster-manager and the + * virtual-desktop-controller stacks repoint it afterwards through the cluster-endpoints lambda. + * Re-reading the live listener keeps this stack from resetting their target group. Only + * `forward` is carried over. + */ + getAlbListenerDefaultActions(listenerArn?: string): elbv2.CfnListener.ActionProperty[] { + if (!isEmpty(listenerArn)) { + const listener = this.describedListeners.get(listenerArn as string); + if (listener === undefined) { + throw new Error( + `listener ${listenerArn as string} was not read at synth time: add its setting key to LISTENER_ARN_KEYS`, + ); + } + const existingAction = listener.DefaultActions?.[0]; + if (existingAction?.Type === 'forward') { + return [ + { + type: 'forward', + forwardConfig: { + targetGroups: [{ targetGroupArn: existingAction.TargetGroupArn }], + }, + }, + ]; + } + } + return [ + { + type: 'fixed-response', + fixedResponseConfig: { + statusCode: '200', + contentType: 'application/json', + messageBody: JSON.stringify({ success: true, message: 'OK' }), + }, + }, + ]; + } + + buildClusterEndpoints(): void { + const lambdaName = 'cluster-endpoints'; + + const clusterEndpointsPolicy = new Policy(this.context, `${lambdaName}-policy`, this.stack, { + policyTemplateName: 'custom-resource-cluster-endpoints.yml', + }); + const clusterEndpointsRole = new Role(this.context, `${lambdaName}-role`, this.stack, { + description: `Role for cluster endpoints lambda function for Cluster: ${this.clusterName}`, + assumedBy: ['lambda'], + }); + clusterEndpointsRole.attachInlinePolicy(clusterEndpointsPolicy); + + this.clusterEndpointsLambda = new LambdaFunction(this.context, lambdaName, this.stack, { + ideaCodeAsset: new IdeaCodeAsset('idea_custom_resource_cluster_endpoints'), + description: 'Manage cluster endpoints exposed via internal and external ALB', + timeoutSeconds: 600, + role: clusterEndpointsRole, + logRetentionRole: this.logRetentionRole, + }); + this.clusterEndpointsLambda.node.addDependency(clusterEndpointsPolicy); + this.clusterEndpointsLambda.node.addDependency(clusterEndpointsRole); + this.addPythonRuntimeSuppression(this.clusterEndpointsLambda); + + // external ALB: public or private subnets + const isPublic = this.config.getBool('cluster.load_balancers.external_alb.public', true); + this.externalAlb = new elbv2.ApplicationLoadBalancer(this.stack, `${this.clusterName}-external-alb`, { + loadBalancerName: `${this.clusterName}-external-alb`, + securityGroup: this.securityGroups['external-load-balancer'] as SecurityGroup, + http2Enabled: true, + vpc: this.vpc, + vpcSubnets: { subnets: isPublic ? this.publicSubnets() : this.privateSubnets() }, + internetFacing: isPublic, + dropInvalidHeaderFields: true, + }); + // large file downloads need more than the 60 second default + this.externalAlb.setAttribute( + 'idle_timeout.timeout_seconds', + String(this.config.getInt('cluster.load_balancers.external_alb.idle_timeout_seconds', 600)), + ); + if (this.externalCertificate !== undefined) { + this.externalAlb.node.addDependency(this.externalCertificate); + } + + if (this.config.getBool('cluster.load_balancers.external_alb.waf.enabled', false)) { + this.externalAlbWafWebAcl = new WebAcl(this.context, 'external-alb', this.stack); + + const wafAssociation = new wafv2.CfnWebACLAssociation( + this.stack, + `${this.clusterName}-external-alb-waf-association`, + { + resourceArn: this.externalAlb.loadBalancerArn, + webAclArn: this.externalAlbWafWebAcl.webAclArn, + }, + ); + wafAssociation.node.addDependency(this.externalAlb); + wafAssociation.node.addDependency(this.externalAlbWafWebAcl.webAcl); + } + + // internal ALB: always private subnets + this.internalAlb = new elbv2.ApplicationLoadBalancer(this.stack, `${this.clusterName}-internal-alb`, { + loadBalancerName: `${this.clusterName}-internal-alb`, + securityGroup: this.securityGroups['internal-load-balancer'] as SecurityGroup, + http2Enabled: true, + vpc: this.vpc, + vpcSubnets: { subnets: this.privateSubnets() }, + internetFacing: false, + dropInvalidHeaderFields: true, + }); + this.internalAlb.setAttribute( + 'idle_timeout.timeout_seconds', + String(this.config.getInt('cluster.load_balancers.internal_alb.idle_timeout_seconds', 600)), + ); + + const externalAccessLogs = this.config.getBool('cluster.load_balancers.external_alb.access_logs', false); + const internalAccessLogs = this.config.getBool('cluster.load_balancers.internal_alb.access_logs', false); + if (externalAccessLogs || internalAccessLogs) { + const accessLogDestination = lookupClusterS3Bucket(this.context, this.stack); + if (externalAccessLogs) { + this.externalAlb.logAccessLogs( + accessLogDestination, + `logs/${this.moduleId}/alb-access-logs/external-alb`, + ); + } + if (internalAccessLogs) { + this.internalAlb.logAccessLogs( + accessLogDestination, + `logs/${this.moduleId}/alb-access-logs/internal-alb`, + ); + } + } + + new elbv2.CfnListener(this.externalAlb, 'http-listener', { + port: 80, + loadBalancerArn: this.externalAlb.loadBalancerArn, + protocol: 'HTTP', + defaultActions: [ + { + type: 'redirect', + redirectConfig: { + host: '#{host}', + path: '/#{path}', + port: '443', + protocol: 'HTTPS', + query: '#{query}', + statusCode: 'HTTP_301', + }, + }, + ], + }); + + const externalAlbDefaultActions = this.getAlbListenerDefaultActions( + this.config.getString('cluster.load_balancers.external_alb.https_listener_arn'), + ); + + const externalAcmCertificateArn = + this.externalCertificate === undefined + ? (this.config.getString('cluster.load_balancers.external_alb.certificates.acm_certificate_arn', undefined, { + required: true, + }) as string) + : this.externalCertificate.getAttString('acm_certificate_arn'); + + this.externalAlbHttpsListener = new elbv2.CfnListener(this.externalAlb, 'https-listener', { + port: 443, + sslPolicy: this.config.getString('cluster.load_balancers.external_alb.ssl_policy', DEFAULT_SSL_POLICY), + loadBalancerArn: this.externalAlb.loadBalancerArn, + protocol: 'HTTPS', + certificates: [{ certificateArn: externalAcmCertificateArn }], + defaultActions: externalAlbDefaultActions, + }); + if (this.externalCertificate !== undefined) { + this.externalAlbHttpsListener.node.addDependency(this.externalCertificate); + } + + const internalCertificate = this.internalCertificate as CustomResource; + this.internalAlb.node.addDependency(internalCertificate); + + const internalAcmCertificateArn = internalCertificate.getAttString('acm_certificate_arn'); + this.internalAlbHttpsListener = new elbv2.CfnListener(this.internalAlb, 'https-listener', { + port: 443, + sslPolicy: this.config.getString('cluster.load_balancers.internal_alb.ssl_policy', DEFAULT_SSL_POLICY), + loadBalancerArn: this.internalAlb.loadBalancerArn, + protocol: 'HTTPS', + certificates: [{ certificateArn: internalAcmCertificateArn }], + defaultActions: [ + { + type: 'fixed-response', + fixedResponseConfig: { + statusCode: '200', + contentType: 'application/json', + messageBody: JSON.stringify({ success: true, message: 'OK' }), + }, + }, + ], + }); + this.internalAlbHttpsListener.node.addDependency(internalCertificate); + + const privateHostedZone = this.privateHostedZone as PrivateHostedZone; + const recordName = `internal-alb.${privateHostedZone.zoneName}`; + if (ROUTE53_CROSS_ZONE_ALIAS_RESTRICTED_REGIONS.includes(this.awsRegion)) { + this.internalAlbDnsRecordSet = new route53.CnameRecord(this.stack, 'internal-alb-dns-record', { + recordName, + zone: privateHostedZone, + domainName: this.internalAlb.loadBalancerDnsName, + ttl: Duration.minutes(5), + }); + } else { + this.internalAlbDnsRecordSet = new route53.RecordSet(this.stack, 'internal-alb-dns-record', { + recordType: route53.RecordType.A, + target: route53.RecordTarget.fromAlias(new route53Targets.LoadBalancerTarget(this.internalAlb)), + recordName, + zone: privateHostedZone, + }); + } + + if (this.config.isModuleEnabled(MODULE_VIRTUAL_DESKTOP_CONTROLLER)) { + this.internalAlbDcvBrokerClientListener = this.buildDcvBrokerListener( + 'dcv-broker-client-listener', + 'client', + internalAcmCertificateArn, + 'Allow HTTPS traffic from DCV Clients to DCV Broker', + ); + this.internalAlbDcvBrokerAgentListener = this.buildDcvBrokerListener( + 'dcv-broker-agent-listener', + 'agent', + internalAcmCertificateArn, + 'Allow HTTPS traffic from DCV Agents to DCV Broker', + ); + this.internalAlbDcvBrokerGatewayListener = this.buildDcvBrokerListener( + 'dcv-broker-gateway-listener', + 'gateway', + internalAcmCertificateArn, + 'Allow HTTPS traffic from DCV Connection Gateway to DCV Broker', + ); + } + } + + /** + * The listener ARN is read from `cluster.external_alb.dcv_broker__listener_arn`, which + * nothing writes, so the default action is always the fixed response. Reading the key the stack + * actually writes would embed the broker target group and change the property on every cluster. + */ + private buildDcvBrokerListener( + listenerId: string, + kind: 'client' | 'agent' | 'gateway', + certificateArn: string, + ingressDescription: string, + ): elbv2.CfnListener { + const defaultActions = this.getAlbListenerDefaultActions( + this.config.getString(`cluster.external_alb.dcv_broker_${kind}_listener_arn`), + ); + const port = requiredInt(this.config, `virtual-desktop-controller.dcv_broker.${kind}_communication_port`); + const internalAlb = this.internalAlb as elbv2.ApplicationLoadBalancer; + const listener = new elbv2.CfnListener(internalAlb, listenerId, { + port, + sslPolicy: this.config.getString('virtual-desktop-controller.dcv_broker.ssl_policy', DEFAULT_SSL_POLICY), + loadBalancerArn: internalAlb.loadBalancerArn, + protocol: 'HTTPS', + certificates: [{ certificateArn }], + defaultActions, + }); + listener.node.addDependency(this.internalCertificate as CustomResource); + (this.securityGroups['internal-load-balancer'] as SecurityGroup).addIngressRule( + ec2.Peer.ipv4(this.vpc.vpcCidrBlock), + ec2.Port.tcp(port), + ingressDescription, + ); + return listener; + } + + // --- cluster settings -------------------------------------------------------------------------- + + buildClusterSettings(): void { + // settings are written in this module's scope, so no key carries the module id + const clusterSettings: Record = { + deployment_id: this.deploymentId, + 'network.vpc_id': this.vpc.vpcId, + 'network.cluster_prefix_list_id': (this.clusterPrefixList as ec2.CfnPrefixList).attrPrefixListId, + }; + + const publicSubnets = this.config.getList('cluster.network.public_subnets', []); + const isExternalAlbPublic = this.config.getBool('cluster.load_balancers.external_alb.public', true); + if (isEmpty(publicSubnets) && isExternalAlbPublic) { + for (const subnet of this.vpc.publicSubnets) publicSubnets.push(subnet.subnetId); + } + clusterSettings['network.public_subnets'] = publicSubnets; + + const privateSubnets = this.config.getList('cluster.network.private_subnets', []); + if (isEmpty(privateSubnets)) { + for (const subnet of this.vpc.privateSubnets) privateSubnets.push(subnet.subnetId); + } + clusterSettings['network.private_subnets'] = privateSubnets; + + if (!this.useExistingVpc()) { + clusterSettings['network.nat_gateway_ips'] = (this.newVpc as Vpc).natGatewayIps.map((eip) => eip.ref); + } + + for (const [name, securityGroup] of Object.entries(this.securityGroups)) { + clusterSettings[`network.security_groups.${name}`] = securityGroup.securityGroupId; + } + + for (const [name, role] of Object.entries(this.roles)) { + clusterSettings[`iam.roles.${name}`] = role.roleArn; + } + clusterSettings['iam.policies.amazon_ssm_managed_instance_core_arn'] = ( + this.amazonSsmManagedInstanceCorePolicy as ManagedPolicy + ).managedPolicyArn; + clusterSettings['iam.policies.cloud_watch_agent_server_arn'] = ( + this.cloudWatchAgentServerPolicy as ManagedPolicy + ).managedPolicyArn; + if (this.amazonPrometheusRemoteWritePolicy !== undefined) { + clusterSettings['iam.policies.amazon_prometheus_remote_write_arn'] = + this.amazonPrometheusRemoteWritePolicy.managedPolicyArn; + } + + clusterSettings['solution.solution_metrics_lambda_arn'] = ( + this.solutionMetricsLambda as LambdaFunction + ).functionArn; + clusterSettings['cluster_settings_lambda_arn'] = (this.clusterSettingsLambda as LambdaFunction).functionArn; + clusterSettings['self_signed_certificate_lambda_arn'] = ( + this.selfSignedCertificateLambda as LambdaFunction + ).functionArn; + + const privateHostedZone = this.privateHostedZone as PrivateHostedZone; + clusterSettings['route53.private_hosted_zone_id'] = privateHostedZone.hostedZoneId; + clusterSettings['route53.private_hosted_zone_arn'] = privateHostedZone.hostedZoneArn; + + if (!requiredBool(this.config, 'cluster.load_balancers.external_alb.certificates.provided')) { + const externalCertificate = this.externalCertificate as CustomResource; + clusterSettings['load_balancers.external_alb.certificates.certificate_secret_arn'] = + externalCertificate.getAttString('certificate_secret_arn'); + clusterSettings['load_balancers.external_alb.certificates.private_key_secret_arn'] = + externalCertificate.getAttString('private_key_secret_arn'); + clusterSettings['load_balancers.external_alb.certificates.acm_certificate_arn'] = + externalCertificate.getAttString('acm_certificate_arn'); + } else { + clusterSettings['load_balancers.external_alb.certificates.provided'] = this.config.getString( + 'cluster.load_balancers.external_alb.certificates.provided', + undefined, + { required: true }, + ); + clusterSettings['load_balancers.external_alb.certificates.acm_certificate_arn'] = this.config.getString( + 'cluster.load_balancers.external_alb.certificates.acm_certificate_arn', + undefined, + { required: true }, + ); + } + + const internalCertificate = this.internalCertificate as CustomResource; + clusterSettings['load_balancers.internal_alb.certificates.certificate_secret_arn'] = + internalCertificate.getAttString('certificate_secret_arn'); + clusterSettings['load_balancers.internal_alb.certificates.private_key_secret_arn'] = + internalCertificate.getAttString('private_key_secret_arn'); + clusterSettings['load_balancers.internal_alb.certificates.acm_certificate_arn'] = + internalCertificate.getAttString('acm_certificate_arn'); + clusterSettings['load_balancers.internal_alb.certificates.custom_dns_name'] = + `internal-alb.${privateHostedZone.zoneName}`; + + const externalAlb = this.externalAlb as elbv2.ApplicationLoadBalancer; + const internalAlb = this.internalAlb as elbv2.ApplicationLoadBalancer; + clusterSettings['cluster_endpoints_lambda_arn'] = (this.clusterEndpointsLambda as LambdaFunction).functionArn; + clusterSettings['load_balancers.external_alb.load_balancer_arn'] = externalAlb.loadBalancerArn; + clusterSettings['load_balancers.external_alb.load_balancer_dns_name'] = externalAlb.loadBalancerDnsName; + clusterSettings['load_balancers.external_alb.https_listener_arn'] = ( + this.externalAlbHttpsListener as elbv2.CfnListener + ).attrListenerArn; + + clusterSettings['load_balancers.internal_alb.load_balancer_arn'] = internalAlb.loadBalancerArn; + clusterSettings['load_balancers.internal_alb.load_balancer_dns_name'] = internalAlb.loadBalancerDnsName; + clusterSettings['load_balancers.internal_alb.https_listener_arn'] = ( + this.internalAlbHttpsListener as elbv2.CfnListener + ).attrListenerArn; + + const ec2EventsSnsTopic = this.ec2EventsSnsTopic as SNSTopic; + clusterSettings['ec2.state_change_notifications_sns_topic_arn'] = ec2EventsSnsTopic.topicArn; + clusterSettings['ec2.state_change_notifications_sns_topic_name'] = ec2EventsSnsTopic.topicName; + + if (this.internalAlbDcvBrokerClientListener !== undefined) { + clusterSettings['load_balancers.internal_alb.dcv_broker_client_listener_arn'] = + this.internalAlbDcvBrokerClientListener.attrListenerArn; + } + if (this.internalAlbDcvBrokerAgentListener !== undefined) { + clusterSettings['load_balancers.internal_alb.dcv_broker_agent_listener_arn'] = + this.internalAlbDcvBrokerAgentListener.attrListenerArn; + } + if (this.internalAlbDcvBrokerGatewayListener !== undefined) { + clusterSettings['load_balancers.internal_alb.dcv_broker_gateway_listener_arn'] = + this.internalAlbDcvBrokerGatewayListener.attrListenerArn; + } + + // An interface endpoint's url is written once, at provisioning: an administrator who edited + // the configuration keeps their value. + for (const [service, endpoint] of Object.entries(this.vpcInterfaceEndpoints ?? {})) { + const endpointConfigKey = `network.vpc_interface_endpoints.${service}.endpoint_url`; + const existingEndpointUrl = this.config.getString(`cluster.${endpointConfigKey}`); + clusterSettings[endpointConfigKey] = isEmpty(existingEndpointUrl) + ? endpoint.getEndpointUrl() + : existingEndpointUrl; + } + + if (this.config.getBool('cluster.backups.enabled', false)) { + clusterSettings['backups.role_arn'] = (this.backupRole as Role).roleArn; + clusterSettings['backups.backup_vault.arn'] = (this.backupVault as backup.BackupVault).backupVaultArn; + clusterSettings['backups.backup_plan.arn'] = (this.backupPlan as BackupPlan).getBackupPlanArn(); + } + + // This stack owns the cluster-settings lambda, so the service token is a GetAtt rather than + // the literal ARN every other module stack reads from the configuration. + new CustomResource(this.stack, `${this.clusterName}-${this.moduleId}-settings`, { + serviceToken: (this.clusterSettingsLambda as LambdaFunction).functionArn, + properties: { + cluster_name: this.clusterName, + module_id: this.moduleId, + version: this.releaseVersion, + settings: clusterSettings, + }, + resourceType: 'Custom::ClusterSettings', + }); + } +} + +export async function buildStack(props: StackBuildProps): Promise { + new ClusterStack(props, await describeListeners(props.ctx)); +} diff --git a/source/idea/ideactl/src/cdk/stacks/directoryservice.ts b/source/idea/ideactl/src/cdk/stacks/directoryservice.ts new file mode 100644 index 00000000..b0e2f88a --- /dev/null +++ b/source/idea/ideactl/src/cdk/stacks/directoryservice.ts @@ -0,0 +1,435 @@ +/** + * `directoryservice.provider` picks one of three disjoint resource sets: + * + * - `openldap`: the credentials pair, an IAM role/profile, the OpenLDAP security group, a + * self-signed certificate custom resource, a launch template + `AWS::EC2::Instance`, and an A + * record. No AD-automation queue. + * - `aws_managed_activedirectory`: the `ActiveDirectory` construct (secrets, MicrosoftAD, + * `Custom::ADSecurityGroupId`, resolver endpoint/rule/association) unless + * `directoryservice.use_existing` is set, plus the AD-automation queue pair. + * - `activedirectory` (self managed): only the AD-automation queue pair; every credential is a + * config-supplied secret ARN, checked non-empty at synth. + * + * An unrecognised provider builds nothing and writes no cluster settings. + */ + +import { CustomResource, Duration, Fn, Tags } from 'aws-cdk-lib'; +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as route53 from 'aws-cdk-lib/aws-route53'; +import type * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager'; +import * as sqs from 'aws-cdk-lib/aws-sqs'; + +import type { StackBuildProps } from '../app.ts'; +import { IdeaBaseStack } from '../base-stack.ts'; +import { buildBootstrapUserData } from '../userdata.ts'; +import { isEmpty } from '../../config/cluster-config.ts'; +import { + DIRECTORYSERVICE_ACTIVE_DIRECTORY, + DIRECTORYSERVICE_AWS_MANAGED_ACTIVE_DIRECTORY, + DIRECTORYSERVICE_OPENLDAP, + IDEA_TAG_NAME, + IDEA_TAG_NODE_TYPE, +} from '../constructs/base.ts'; +import { InstanceProfile, Policy, Role, SQSQueue } from '../constructs/common.ts'; +import { ActiveDirectory, DirectoryServiceCredentials, MODULE_DIRECTORYSERVICE } from '../constructs/directory-service.ts'; +import { ExistingSocaCluster, lookupClusterDns, lookupEbsKmsKey } from '../constructs/existing-resources.ts'; +import { OpenLDAPServerSecurityGroup } from '../constructs/network.ts'; + +/** `constants.NODE_TYPE_INFRA`. */ +const NODE_TYPE_INFRA = 'infra'; +/** `constants.SQS_VISIBILITY_TIMEOUT_AD_AUTOMATION`. */ +const SQS_VISIBILITY_TIMEOUT_AD_AUTOMATION = 30; +/** `constants.SQS_MAX_RECEIVE_COUNT_AD_AUTOMATION`. */ +const SQS_MAX_RECEIVE_COUNT_AD_AUTOMATION = 16; + +/** `Utils.get_ec2_block_device_name`. */ +export function ec2BlockDeviceName(baseOs: string): string { + return baseOs === 'amazonlinux2' || baseOs === 'amazonlinux2023' ? '/dev/xvda' : '/dev/sda1'; +} + +export class DirectoryServiceStack extends IdeaBaseStack { + readonly cluster: ExistingSocaCluster; + + bootstrapPackageUri: string | undefined; + openldapRole: Role | undefined; + openldapInstanceProfile: InstanceProfile | undefined; + openldapSecurityGroup: OpenLDAPServerSecurityGroup | undefined; + openldapCerts: CustomResource | undefined; + openldapEc2Instance: ec2.CfnInstance | undefined; + openldapClusterDnsRecordSet: route53.RecordSet | undefined; + openldapCredentials: DirectoryServiceCredentials | undefined; + + activedirectory: ActiveDirectory | undefined; + + adAutomationSqsQueue: SQSQueue | undefined; + + constructor(props: StackBuildProps) { + super({ + scope: props.app, + ctx: props.ctx, + moduleName: props.moduleName, + deploymentId: props.deploymentId, + terminationProtection: props.terminationProtection, + env: props.env, + }); + + this.cluster = new ExistingSocaCluster(this.context, this.stack); + + const provider = this.context.config.getString('directoryservice.provider', undefined, { + required: true, + }) as string; + + if (provider === DIRECTORYSERVICE_OPENLDAP) { + this.assertRootCredentialsWhenProvided(); + + // Missing `bootstrap_package_uri` is represented by the literal `None`. + const contextUri: unknown = this.stack.node.tryGetContext('bootstrap_package_uri'); + this.bootstrapPackageUri = contextUri === undefined || contextUri === null ? 'None' : String(contextUri); + + this.openldapCredentials = new DirectoryServiceCredentials( + this.context, + `${this.moduleId}-openldap-credentials`, + this.stack, + 'Admin', + ); + this.buildIamRoles(); + this.buildSecurityGroups(); + this.buildOpenldapCerts(); + this.buildEc2Instance(); + this.buildRoute53RecordSet(); + this.buildOpenldapClusterSettings(); + } else if (provider === DIRECTORYSERVICE_AWS_MANAGED_ACTIVE_DIRECTORY) { + this.assertRootCredentialsWhenProvided(); + + if (this.context.config.getBool('directoryservice.use_existing', false)) { + assertNotEmpty( + this.context.config.getString('directoryservice.directory_id'), + 'directoryservice.directory_id', + ); + } else { + this.activedirectory = new ActiveDirectory(this.context, 'active-directory', this.stack, { + cluster: this.cluster, + enableSso: false, + }); + } + this.buildAdAutomationSqsQueue(); + this.buildAwsManagedAdClusterSettings(); + } else if (provider === DIRECTORYSERVICE_ACTIVE_DIRECTORY) { + // Self managed AD: IDEA has no write access, so a clusteradmin is bootstrapped from + // config-supplied secrets by the directoryservice helper after installation. + if (this.context.config.getBool('directoryservice.root_credentials_provided', false) !== true) { + throw new Error('directoryservice.root_credentials_provided must be true'); + } + for (const key of [ + 'directoryservice.root_username_secret_arn', + 'directoryservice.root_password_secret_arn', + 'directoryservice.clusteradmin.clusteradmin_username_secret_arn', + 'directoryservice.clusteradmin.clusteradmin_password_secret_arn', + ]) { + assertNotEmpty(this.context.config.getString(key), key); + } + + this.buildAdAutomationSqsQueue(); + this.buildActivedirectoryClusterSettings(); + } + } + + /** Both AD-backed providers and openldap check the pair only when the flag is set. */ + private assertRootCredentialsWhenProvided(): void { + if (!this.context.config.getBool('directoryservice.root_credentials_provided', false)) return; + for (const key of ['directoryservice.root_username_secret_arn', 'directoryservice.root_password_secret_arn']) { + assertNotEmpty(this.context.config.getString(key), key); + } + } + + buildIamRoles(): void { + this.openldapRole = new Role(this.context, `${this.moduleId}-openldap-role`, this.stack, { + description: 'IAM role assigned to the OpenLDAP server', + assumedBy: ['ssm', 'ec2'], + managedPolicies: this.getEc2InstanceManagedPolicies(), + }); + this.openldapRole.attachInlinePolicy( + new Policy(this.context, 'openldap-server-policy', this.stack, { + policyTemplateName: 'openldap-server.yml', + }), + ); + this.openldapInstanceProfile = new InstanceProfile( + this.context, + `${this.moduleId}-openldap-instance-profile`, + this.stack, + [this.openldapRole], + ); + } + + buildSecurityGroups(): void { + this.openldapSecurityGroup = new OpenLDAPServerSecurityGroup( + this.context, + `${this.moduleId}-security-group`, + this.stack, + this.cluster.vpc, + this.cluster.getSecurityGroup('bastion-host') as ec2.ISecurityGroup, + ); + } + + /** + * OpenLDAP TLS certificates, saved to Secrets Manager. Only the server can read the private key; + * every cluster node reads the certificate so it can join the directory. + */ + buildOpenldapCerts(): void { + const hostname = this.context.config.getString('directoryservice.hostname', undefined, { + required: true, + }) as string; + const serviceToken = this.context.config.getString('cluster.self_signed_certificate_lambda_arn', undefined, { + required: true, + }) as string; + this.openldapCerts = new CustomResource(this.stack, 'openldap-server-certs', { + serviceToken, + properties: { + domain_name: hostname, + certificate_name: `${this.clusterName}-${this.moduleId}`, + create_acm_certificate: false, + kms_key_id: this.context.config.getString('cluster.secretsmanager.kms_key_id'), + tags: { + Name: `${this.clusterName}-${this.moduleId}`, + 'idea:ClusterName': this.clusterName, + 'idea:ModuleName': MODULE_DIRECTORYSERVICE, + }, + }, + resourceType: 'Custom::SelfSignedCertificateOpenLDAPServer', + }); + } + + buildEc2Instance(): void { + const config = this.context.config; + const isPublic = config.getBool('directoryservice.public', false); + const baseOs = config.getString('directoryservice.base_os', undefined, { required: true }) as string; + const instanceAmi = config.getString('directoryservice.instance_ami', undefined, { required: true }) as string; + const instanceType = config.getString('directoryservice.instance_type', undefined, { required: true }) as string; + const volumeSize = config.getInt('directoryservice.volume_size', 200); + const keyPairName = config.getString('cluster.network.ssh_key_pair', undefined, { required: true }) as string; + const enableDetailedMonitoring = config.getBool('directoryservice.ec2.enable_detailed_monitoring', false); + const enableTerminationProtection = config.getBool('directoryservice.ec2.enable_termination_protection', false); + const metadataHttpTokens = config.getString('directoryservice.ec2.metadata_http_tokens', undefined, { + required: true, + }) as string; + const httpsProxy = config.getString('cluster.network.https_proxy', ''); + const noProxy = config.getString('cluster.network.no_proxy', ''); + const proxyConfig: Record = isEmpty(httpsProxy) + ? {} + : { http_proxy: httpsProxy, https_proxy: httpsProxy, no_proxy: noProxy }; + + const ebsKmsKey = lookupEbsKmsKey(this.context, this.stack); + + const subnetIds = + isPublic && this.cluster.publicSubnets.length > 0 + ? this.cluster.existingVpc.getPublicSubnetIds() + : this.cluster.existingVpc.getPrivateSubnetIds(); + + const blockDeviceName = ec2BlockDeviceName(baseOs); + const blockDeviceTypeString = config.getString('directoryservice.volume_type', 'gp3'); + const blockDeviceVolumeType = + blockDeviceTypeString === 'gp3' ? ec2.EbsDeviceVolumeType.GP3 : ec2.EbsDeviceVolumeType.GP2; + + const userData = buildBootstrapUserData({ + awsRegion: this.awsRegion, + bootstrapPackageUri: this.bootstrapPackageUri as string, + installCommands: ['/bin/bash openldap-server/setup.sh'], + baseOs, + infraConfig: { + LDAP_ROOT_USERNAME_SECRET_ARN: '${__LDAP_ROOT_USERNAME_SECRET_ARN__}', + LDAP_ROOT_PASSWORD_SECRET_ARN: '${__LDAP_ROOT_PASSWORD_SECRET_ARN__}', + LDAP_TLS_CERTIFICATE_SECRET_ARN: '${__LDAP_TLS_CERTIFICATE_SECRET_ARN__}', + LDAP_TLS_PRIVATE_KEY_SECRET_ARN: '${__LDAP_TLS_PRIVATE_KEY_SECRET_ARN__}', + }, + proxyConfig, + }); + + const credentials = this.openldapCredentials as DirectoryServiceCredentials; + const certs = this.openldapCerts as CustomResource; + const substitutedUserdata = Fn.sub(userData, { + __LDAP_ROOT_USERNAME_SECRET_ARN__: credentials.getUsernameSecretArn(), + __LDAP_ROOT_PASSWORD_SECRET_ARN__: credentials.getPasswordSecretArn(), + __LDAP_TLS_CERTIFICATE_SECRET_ARN__: certs.getAttString('certificate_secret_arn'), + __LDAP_TLS_PRIVATE_KEY_SECRET_ARN__: certs.getAttString('private_key_secret_arn'), + }); + + const launchTemplate = new ec2.LaunchTemplate(this.stack, `${this.moduleId}-lt`, { + instanceType: new ec2.InstanceType(instanceType), + machineImage: ec2.MachineImage.genericLinux({ [this.awsRegion]: instanceAmi }), + userData: ec2.UserData.custom(substitutedUserdata), + keyName: keyPairName, + blockDevices: [ + { + deviceName: blockDeviceName, + volume: { + ebsDevice: { + encrypted: true, + kmsKey: ebsKmsKey, + volumeSize, + volumeType: blockDeviceVolumeType, + }, + }, + }, + ], + requireImdsv2: metadataHttpTokens === 'required', + }); + + this.openldapEc2Instance = new ec2.CfnInstance(this.stack, `${this.moduleId}-instance`, { + blockDeviceMappings: [ + { deviceName: blockDeviceName, ebs: { volumeSize, volumeType: blockDeviceTypeString } }, + ], + disableApiTermination: enableTerminationProtection, + iamInstanceProfile: (this.openldapInstanceProfile as InstanceProfile).instanceProfileName, + instanceType, + imageId: instanceAmi, + keyName: keyPairName, + launchTemplate: { + version: launchTemplate.latestVersionNumber, + launchTemplateId: launchTemplate.launchTemplateId, + }, + networkInterfaces: [ + { + deviceIndex: '0', + associatePublicIpAddress: isPublic, + groupSet: [(this.openldapSecurityGroup as OpenLDAPServerSecurityGroup).securityGroupId], + subnetId: subnetIds[0] as string, + }, + ], + userData: Fn.base64(substitutedUserdata), + monitoring: enableDetailedMonitoring, + }); + Tags.of(this.openldapEc2Instance).add(IDEA_TAG_NAME, this.buildResourceName(this.moduleId)); + Tags.of(this.openldapEc2Instance).add(IDEA_TAG_NODE_TYPE, NODE_TYPE_INFRA); + this.addBackupTags(this.openldapEc2Instance); + + this.addNagSuppression( + [{ rule_id: 'AwsSolutions-EC26', reason: 'EBS Encryption is enforced via Launch Template' }], + this.openldapEc2Instance, + ); + + if (!enableDetailedMonitoring) { + this.addNagSuppression( + [ + { + rule_id: 'AwsSolutions-EC28', + reason: 'detailed monitoring is a configurable option to save costs', + }, + ], + this.openldapEc2Instance, + ); + } + + if (!enableTerminationProtection) { + this.addNagSuppression( + [ + { + rule_id: 'AwsSolutions-EC29', + reason: + 'termination protection not supported in CDK L2 construct. enable termination protection via AWS EC2 console after deploying the cluster.', + }, + ], + this.openldapEc2Instance, + ); + } + } + + buildRoute53RecordSet(): void { + const hostname = this.context.config.getString('directoryservice.hostname', undefined, { + required: true, + }) as string; + this.openldapClusterDnsRecordSet = new route53.RecordSet(this.stack, `${this.moduleId}-dns-record`, { + recordType: route53.RecordType.A, + target: route53.RecordTarget.fromIpAddresses( + (this.openldapEc2Instance as ec2.CfnInstance).attrPrivateIp, + ), + ttl: Duration.minutes(5), + recordName: hostname, + zone: lookupClusterDns(this.context, this.stack), + }); + } + + buildAdAutomationSqsQueue(): void { + const kmsKeyId = this.context.config.getString('cluster.sqs.kms_key_id'); + + this.adAutomationSqsQueue = new SQSQueue(this.context, 'ad-automation-sqs-queue', this.stack, { + queueName: `${this.clusterName}-${this.moduleId}-ad-automation.fifo`, + fifo: true, + contentBasedDeduplication: true, + encryptionMasterKey: kmsKeyId, + visibilityTimeout: Duration.seconds(SQS_VISIBILITY_TIMEOUT_AD_AUTOMATION), + deadLetterQueue: { + maxReceiveCount: SQS_MAX_RECEIVE_COUNT_AD_AUTOMATION, + queue: new SQSQueue(this.context, 'ad-automation-sqs-queue-dlq', this.stack, { + queueName: `${this.clusterName}-${this.moduleId}-ad-automation-dlq.fifo`, + fifo: true, + contentBasedDeduplication: true, + encryptionMasterKey: kmsKeyId, + isDeadLetterQueue: true, + }), + }, + }); + // Both queues use `Name=-`. + this.addCommonTags(this.adAutomationSqsQueue); + this.addCommonTags((this.adAutomationSqsQueue.deadLetterQueue as sqs.DeadLetterQueue).queue); + } + + buildOpenldapClusterSettings(): void { + const instance = this.openldapEc2Instance as ec2.CfnInstance; + const credentials = this.openldapCredentials as DirectoryServiceCredentials; + const certs = this.openldapCerts as CustomResource; + const clusterSettings: Record = { + deployment_id: this.deploymentId, + private_ip: instance.attrPrivateIp, + private_dns_name: instance.attrPrivateDnsName, + instance_id: instance.ref, + security_group_id: (this.openldapSecurityGroup as OpenLDAPServerSecurityGroup).securityGroupId, + iam_role_arn: (this.openldapRole as Role).roleArn, + instance_profile_arn: (this.openldapInstanceProfile as InstanceProfile).ref, + // This path reads raw secret resources. With supplied root credentials those resources are + // absent, so the dereference fails. + root_username_secret_arn: (credentials.adminUsername as secretsmanager.CfnSecret).ref, + root_password_secret_arn: (credentials.adminPassword as secretsmanager.CfnSecret).ref, + tls_certificate_secret_arn: certs.getAttString('certificate_secret_arn'), + tls_private_key_secret_arn: certs.getAttString('private_key_secret_arn'), + }; + + if (this.context.config.getBool('directoryservice.public', false)) { + clusterSettings['public_ip'] = instance.attrPublicIp; + } + + this.updateClusterSettings(clusterSettings); + } + + buildAwsManagedAdClusterSettings(): void { + const queue = this.adAutomationSqsQueue as SQSQueue; + const clusterSettings: Record = { + deployment_id: this.deploymentId, + 'ad_automation.sqs_queue_url': queue.queueUrl, + 'ad_automation.sqs_queue_arn': queue.queueArn, + }; + if (this.activedirectory !== undefined) { + clusterSettings['directory_id'] = this.activedirectory.ad.ref; + clusterSettings['root_username_secret_arn'] = this.activedirectory.credentials.getUsernameSecretArn(); + clusterSettings['root_password_secret_arn'] = this.activedirectory.credentials.getPasswordSecretArn(); + } + this.updateClusterSettings(clusterSettings); + } + + buildActivedirectoryClusterSettings(): void { + const queue = this.adAutomationSqsQueue as SQSQueue; + this.updateClusterSettings({ + deployment_id: this.deploymentId, + 'ad_automation.sqs_queue_url': queue.queueUrl, + 'ad_automation.sqs_queue_arn': queue.queueArn, + }); + } +} + +/** Throws with the key when a required value is empty. */ +function assertNotEmpty(value: string | undefined, key: string): void { + if (isEmpty(value)) throw new Error(`${key} is required`); +} + +export function buildStack(props: StackBuildProps): void { + new DirectoryServiceStack(props); +} diff --git a/source/idea/ideactl/src/cdk/stacks/ecs.ts b/source/idea/ideactl/src/cdk/stacks/ecs.ts new file mode 100644 index 00000000..eac290a1 --- /dev/null +++ b/source/idea/ideactl/src/cdk/stacks/ecs.ts @@ -0,0 +1,1683 @@ +/** + * Container control-plane stack. + * + * This stack supplies shared ECS capacity and runs the five host-module roles + * as arm64 tasks. It owns the task ENI security groups, while the host security + * group is limited to the container instances. + */ + +import { Annotations, Aws, CustomResource, Duration, Fn, RemovalPolicy } from "aws-cdk-lib"; +import type { StackBuildProps } from "../app.ts"; +import { IdeaBaseStack } from "../base-stack.ts"; +import { isDsActivedirectory } from "../constructs/base.ts"; +import { LOG_RETENTION_DAYS, Policy } from "../constructs/common.ts"; +import { ExistingSocaCluster } from "../constructs/existing-resources.ts"; +import { buildTrimmedResourceName } from "../../util/names.ts"; +import * as autoscaling from "aws-cdk-lib/aws-autoscaling"; +import * as ec2 from "aws-cdk-lib/aws-ec2"; +import * as ecs from "aws-cdk-lib/aws-ecs"; +import * as efs from "aws-cdk-lib/aws-efs"; +import * as elbv2 from "aws-cdk-lib/aws-elasticloadbalancingv2"; +import * as iam from "aws-cdk-lib/aws-iam"; +import * as lambda from "aws-cdk-lib/aws-lambda"; +import * as logs from "aws-cdk-lib/aws-logs"; +import * as servicediscovery from "aws-cdk-lib/aws-servicediscovery"; +import * as secretsmanager from "aws-cdk-lib/aws-secretsmanager"; +import { Provider } from "aws-cdk-lib/custom-resources"; + +const CONTAINER_ROLES = ["cluster-manager", "vdc", "scheduler", "dcv-broker", "dcv-gateway"] as const; + +type ContainerRole = (typeof CONTAINER_ROLES)[number]; + +/** Stream family preserved from the agent `application_{ip}` prefix, minus the address. */ +const STREAM_PREFIX_APPLICATION = "application"; +/** Stream family for OpenPBS files that used `server_logs_`, `sched_logs_`, and `accounting_logs_`. */ +const STREAM_PREFIX_OPENPBS = "openpbs"; +/** Stream family preserved from the agent `dcv-session-manager-broker_{ip}` prefix. */ +const STREAM_PREFIX_BROKER = "dcv-session-manager-broker"; +/** Stream family preserved from the agent `dcv-connection-gateway_{ip}` prefix. */ +const STREAM_PREFIX_GATEWAY = "dcv-connection-gateway"; +/** Stream family for the optional host daemon. */ +const STREAM_PREFIX_DATADOG = "datadog"; +/** Agent-created groups use `cluster.cloudwatch_logs.retention_in_days`, which defaults to 90. */ +const DEFAULT_AGENT_LOG_RETENTION_DAYS = 90; + +/** + * Start allowances, used for the load balancer health check grace and for the container health + * check that runs alongside it. + * + * The library default grace is 60 seconds, which is shorter than every role's start: the first + * deployment would then fail its target health checks while the process being checked is still + * starting, and the circuit breaker would roll it back. The two wait ceilings below are the ones + * the image's role scripts impose on themselves, so they are facts about the start rather than + * estimates: `roles/scheduler.sh` waits up to 30 attempts 5 seconds apart for the batch server to + * answer, and `roles/broker.sh` waits up to 30 attempts 2 seconds apart for service discovery to + * resolve. The application allowance is the time a role needs after its own wait to answer a health + * request, and on a scheduler first start it also covers creating a new state directory and its + * datastore. + */ +const PBS_SERVER_WAIT_SECONDS = 150; +const BROKER_DISCOVERY_WAIT_SECONDS = 60; +const APPLICATION_START_SECONDS = 120; +/** + * The load balancer grace has to cover registration as well as the start, because a target that has + * not yet passed its consecutive successful checks is not healthy yet and the grace is what keeps the + * platform from acting on that. These target groups use the library health check, which is five + * checks thirty seconds apart for the application load balancer and three for the network one, so the + * application figure is the longer of the two. + */ +const TARGET_REGISTRATION_SECONDS = 150; + +/** Task policy template per role: the same template the module's instance role renders. */ +const TASK_POLICY_TEMPLATES: Readonly> = { + "cluster-manager": "cluster-manager.yml", + vdc: "virtual-desktop-controller.yml", + scheduler: "scheduler.yml", + "dcv-broker": "virtual-desktop-dcv-broker.yml", + "dcv-gateway": "virtual-desktop-dcv-connection-gateway.yml", +}; + +/** Egress every module host security group carries, for IPv4 and IPv6. */ +const TCP_EGRESS_DESCRIPTION = "Allow all egress for TCP"; +/** The ingress and egress pair the host groups add on an Active Directory cluster. */ +const DIRECTORY_SERVICE_INGRESS_DESCRIPTION = "Allow UDP Traffic from VPC. Required for Directory Service"; +const DIRECTORY_SERVICE_EGRESS_DESCRIPTION = "Allow UDP Traffic. Required for Directory Service"; +/** + * The container-optimised host image, resolved at launch. The x86_64 image has no architecture + * segment in its parameter path and the arm64 one does, which is why this is a function of the + * architecture rather than a string with a slot in it. + */ +function hostImageParameter(architecture: ec2.InstanceArchitecture): string { + const prefix = "/aws/service/ecs/optimized-ami/amazon-linux-2023"; + return architecture === ec2.InstanceArchitecture.ARM_64 + ? `${prefix}/arm64/recommended/image_id` + : `${prefix}/recommended/image_id`; +} + +/** The task definition value for one host architecture. */ +function cpuArchitecture(architecture: ec2.InstanceArchitecture): string { + return architecture === ec2.InstanceArchitecture.ARM_64 ? "ARM64" : "X86_64"; +} +/** PBS state and logs live on the scheduler task volume. */ +const SCHEDULER_PBS_HOME = "/var/spool/pbs"; +const APPLICATION_LOG_DIRECTORY = "/opt/idea/app/logs"; +const BROKER_LOG_DIRECTORY = "/var/log/dcv-session-manager-broker"; +const GATEWAY_LOG_DIRECTORY = "/var/log/dcv-connection-gateway"; + +/** + * Create-or-adopt handler used by the ECS stack. Delete is a no-op so history + * survives a stack rollback. Retention is applied only when the property is set. + */ +const ENSURE_AGENT_LOG_GROUP_HANDLER = ` +import boto3 + +def handler(event, context): + name = event["ResourceProperties"]["LogGroupName"] + if event["RequestType"] != "Delete": + logs = boto3.client("logs") + try: + logs.create_log_group(logGroupName=name) + except logs.exceptions.ResourceAlreadyExistsException: + pass + retention = event["ResourceProperties"].get("RetentionInDays") + if retention: + logs.put_retention_policy(logGroupName=name, retentionInDays=int(retention)) + return {"PhysicalResourceId": name, "Data": {"LogGroupName": name}} +`.trim(); + +/** + * Releases the host group's scale-in protection when the group is being deleted. + * + * Managed termination protection is what stops a scale-in killing a task mid-flight, and enabling + * it requires scale-in protection on the group. Nothing removes that protection when the group is + * meant to go away, so the group sits at desired zero with every instance still in service and + * CloudFormation waits out its own timeout. This runs on Delete, which is the only moment our code + * is in the loop during a rollback, and it clears both the group default and the instances that + * carry the flag already. A group that has gone, or that never launched, is not an error. + */ +const RELEASE_SCALE_IN_PROTECTION_HANDLER = ` +import boto3 + +def handler(event, context): + name = event["ResourceProperties"]["AutoScalingGroupName"] + if event["RequestType"] == "Delete": + autoscaling = boto3.client("autoscaling") + groups = autoscaling.describe_auto_scaling_groups(AutoScalingGroupNames=[name]) + for group in groups.get("AutoScalingGroups", []): + autoscaling.update_auto_scaling_group( + AutoScalingGroupName=name, NewInstancesProtectedFromScaleIn=False + ) + instances = [ + instance["InstanceId"] + for instance in group.get("Instances", []) + if instance.get("ProtectedFromScaleIn") + ] + if instances: + autoscaling.set_instance_protection( + AutoScalingGroupName=name, + InstanceIds=instances, + ProtectedFromScaleIn=False, + ) + return {"PhysicalResourceId": "scale-in-protection-" + name} +`.trim(); + +interface RoleSizing { + cpu: number; + memory: number; + desired: number; +} + +interface StorageMount { + readonly hostPath?: string; + readonly fileSystemId?: string; + readonly mountPath: string; + readonly name: string; +} + +interface RoleResources { + readonly service: ecs.Ec2Service; + readonly targetGroups: Array; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export class EcsStack extends IdeaBaseStack { + readonly cluster: ExistingSocaCluster; + readonly ecsCluster: ecs.Cluster; + readonly namespace: servicediscovery.PrivateDnsNamespace; + readonly hostSecurityGroup: ec2.SecurityGroup; + readonly hostRole: iam.Role; + readonly hostAutoScalingGroup: autoscaling.AutoScalingGroup; + readonly capacityProvider: ecs.AsgCapacityProvider; + /** Clears the host group's scale-in protection when the group is deleted. */ + private scaleInRelease!: CustomResource; + readonly executionRole: iam.Role; + readonly gatewayExecutionRole: iam.Role; + + private readonly roleResources: Partial> = {}; + /** Resolved once from the configured host family: the host image and every task follow it. */ + private readonly hostArchitecture: ec2.InstanceArchitecture; + /** + * Task identities this stack creates. The module instance roles stay where they are: one set + * belongs to the hosts until they retire, the other to the tasks. + */ + private readonly taskRoles: Partial> = {}; + private readonly taskSecurityGroups: Partial> = {}; + private instanceManagedPolicies: iam.IManagedPolicy[] | undefined; + private readonly gatewayCertificateSecretArn: string; + private readonly gatewayPrivateKeySecretArn: string; + private readonly ensuredLogGroups = new Map(); + private logGroupEnsureProvider: Provider | undefined; + + constructor(props: StackBuildProps) { + super({ + scope: props.app, + ctx: props.ctx, + moduleName: props.moduleName, + deploymentId: props.deploymentId, + terminationProtection: props.terminationProtection, + env: props.env, + }); + + this.cluster = new ExistingSocaCluster(this.context, this.stack); + const execLogGroupName = this.execCommandLogGroupName(); + const execLogGroup = this.ensureAgentLogGroup("exec-log-group-ensure", execLogGroupName); + this.ecsCluster = new ecs.Cluster(this.stack, "ecs-cluster", { + clusterName: `${this.clusterName}-ecs`, + containerInsightsV2: ecs.ContainerInsights.ENABLED, + executeCommandConfiguration: { + logConfiguration: { + cloudWatchLogGroup: logs.LogGroup.fromLogGroupName(this.stack, "exec-log-group-ref", execLogGroupName), + }, + logging: ecs.ExecuteCommandLogging.OVERRIDE, + }, + vpc: this.cluster.vpc, + }); + // The group has to exist before a session opens: a cluster naming a group that is not there + // runs sessions that are simply never recorded, and being attributable is the whole reason + // this path is preferred to reaching a container host and using the runtime directly. + this.ecsCluster.node.addDependency(execLogGroup); + this.namespace = new servicediscovery.PrivateDnsNamespace(this.stack, "service-discovery-namespace", { + name: `${this.clusterName}.ecs.local`, + vpc: this.cluster.vpc, + }); + this.hostArchitecture = this.hostInstanceType().architecture; + const gatewaySecrets = this.loadGatewayCertificateSecrets(); + this.gatewayCertificateSecretArn = gatewaySecrets.certificateSecretArn; + this.gatewayPrivateKeySecretArn = gatewaySecrets.privateKeySecretArn; + + this.hostSecurityGroup = this.buildHostSecurityGroup(); + this.hostRole = this.buildHostRole(); + this.hostAutoScalingGroup = this.buildHostAutoScalingGroup(); + this.capacityProvider = new ecs.AsgCapacityProvider(this.stack, "host-capacity-provider", { + autoScalingGroup: this.hostAutoScalingGroup, + capacityProviderName: `${this.clusterName}-ecs-capacity`, + enableManagedScaling: true, + enableManagedTerminationProtection: true, + targetCapacityPercent: 100, + }); + this.ecsCluster.addAsgCapacityProvider(this.capacityProvider); + this.releaseScaleInProtectionOnDelete(); + + this.executionRole = this.buildExecutionRole("ecs-task-execution-role"); + this.gatewayExecutionRole = this.buildExecutionRole("gateway-task-execution-role"); + this.buildApplicationServices(); + this.buildEndpoints(); + this.buildDatadogService(); + this.buildClusterSettings(); + } + + /** Returns a required string setting under the ECS module. */ + private requiredString(key: string): string { + return this.context.config.getString(key, undefined, { required: true }) as string; + } + + /** Returns a required integer setting under the ECS module. */ + private requiredInt(key: string): number { + const value = this.context.config.getInt(key); + if (value === undefined) throw new Error(`${key} is required for ECS`); + return value; + } + + /** Module id the application process reads from IDEA_MODULE_ID and from log group paths. */ + private ideaModuleId(role: ContainerRole): string { + if (role === "cluster-manager") return this.context.config.moduleId("cluster-manager"); + if (role === "scheduler") return this.context.config.moduleId("scheduler"); + return this.context.config.moduleId("virtual-desktop-controller"); + } + + /** Module name the application process reads from IDEA_MODULE_NAME. */ + private ideaModuleName(role: ContainerRole): string { + if (role === "cluster-manager") return "cluster-manager"; + if (role === "scheduler") return "scheduler"; + return "virtual-desktop-controller"; + } + + /** + * Log group the CloudWatch agent writes to. Cluster-manager and scheduler use `/{cluster}/{module-id}`. + * VDC components append `/controller`, `/dcv-broker`, or `/dcv-connection-gateway`. + */ + private agentLogGroupName(role: ContainerRole, component?: string): string { + const moduleId = this.ideaModuleId(role); + return component === undefined ? `/${this.clusterName}/${moduleId}` : `/${this.clusterName}/${moduleId}/${component}`; + } + + /** + * Where command-execution sessions are recorded. Sits under the same `/{cluster}` prefix the + * adopted agent groups use, so it is covered by the ensure provider's policy and by the cluster + * retention, and is never deleted by a rollback. + */ + private execCommandLogGroupName(): string { + return `/${this.clusterName}/${this.moduleId}/exec`; + } + + /** Retention copied from the CloudWatch agent setting. Invalid values leave an adopted group unchanged. */ + private agentLogRetentionDays(): number | undefined { + const retentionInDays = this.context.config.getInt( + "cluster.cloudwatch_logs.retention_in_days", + DEFAULT_AGENT_LOG_RETENTION_DAYS, + ); + return retentionInDays in LOG_RETENTION_DAYS ? retentionInDays : undefined; + } + + /** + * Provider that creates a missing group, adopts an existing one, sets retention, + * and never deletes. CloudFormation does not own the group, so an upgrade cannot + * fail with already-exists and a rollback cannot wipe history. + */ + private agentLogGroupProvider(): Provider { + if (this.logGroupEnsureProvider !== undefined) return this.logGroupEnsureProvider; + + const onEvent = new lambda.Function(this.stack, "agent-log-group-fn", { + code: lambda.Code.fromInline(ENSURE_AGENT_LOG_GROUP_HANDLER), + handler: "index.handler", + runtime: lambda.Runtime.PYTHON_3_13, + timeout: Duration.seconds(60), + }); + onEvent.addToRolePolicy( + new iam.PolicyStatement({ + actions: ["logs:CreateLogGroup", "logs:PutRetentionPolicy"], + resources: [ + `arn:${Aws.PARTITION}:logs:${Aws.REGION}:${Aws.ACCOUNT_ID}:log-group:/${this.clusterName}`, + `arn:${Aws.PARTITION}:logs:${Aws.REGION}:${Aws.ACCOUNT_ID}:log-group:/${this.clusterName}/*`, + ], + }), + ); + this.logGroupEnsureProvider = new Provider(this.stack, "agent-log-group-provider", { + onEventHandler: onEvent, + }); + return this.logGroupEnsureProvider; + } + + /** Ensures one agent log group exists with the cluster retention, then returns it. */ + private ensureAgentLogGroup(constructId: string, logGroupName: string): CustomResource { + const existing = this.ensuredLogGroups.get(logGroupName); + if (existing !== undefined) return existing; + + const properties: Record = { LogGroupName: logGroupName }; + const retentionInDays = this.agentLogRetentionDays(); + if (retentionInDays !== undefined) properties["RetentionInDays"] = String(retentionInDays); + + const resource = new CustomResource(this.stack, constructId, { + properties, + serviceToken: this.agentLogGroupProvider().serviceToken, + }); + this.ensuredLogGroups.set(logGroupName, resource); + return resource; + } + + /** + * Writes to a preserved group without emitting AWS::Logs::LogGroup. + * Stream names become `{prefix}/{container}/{task-id}`. + */ + private adoptedLogDriver(constructId: string, logGroupName: string, streamPrefix: string): ecs.LogDriver { + this.ensureAgentLogGroup(`${constructId}-ensure`, logGroupName); + const logGroup = logs.LogGroup.fromLogGroupName(this.stack, `${constructId}-ref`, logGroupName); + return ecs.LogDrivers.awsLogs({ logGroup, streamPrefix }); + } + + /** Keeps the task from starting before its log group has been created or adopted. */ + private bindContainerToLogGroup(container: ecs.ContainerDefinition, logGroupName: string): void { + const resource = this.ensuredLogGroups.get(logGroupName); + if (resource !== undefined) container.node.addDependency(resource); + } + + /** Follows log files that the awslogs driver cannot tail. */ + private fileTailScript(directories: string[]): string { + const mkdirLines = directories.map((directory) => `install -d -m 0755 "${directory}"`).join("\n"); + const globList = directories.map((directory) => `"${directory}"/*.log`).join(" "); + return [ + "set -euo pipefail", + mkdirLines, + "while true; do", + " shopt -s nullglob", + ` files=(${globList})`, + " if ((${#files[@]})); then exec tail -F -- \"${files[@]}\"; fi", + " sleep 5", + "done", + ].join("\n"); + } + + /** + * Sidecar that tails the role's log files into a preserved group. + * The main container still has its own awslogs driver for stdout. + */ + private addLogTailContainer( + taskDefinition: ecs.Ec2TaskDefinition, + input: { + constructId: string; + containerId: string; + directories: string[]; + logGroupName: string; + streamPrefix: string; + sourceVolume: string; + containerPath: string; + readOnly: boolean; + }, + ): void { + const container = taskDefinition.addContainer(input.containerId, { + command: [this.fileTailScript(input.directories)], + entryPoint: ["/bin/bash", "-lc"], + essential: false, + image: ecs.ContainerImage.fromRegistry(this.requiredString("ecs.image")), + logging: this.adoptedLogDriver(input.constructId, input.logGroupName, input.streamPrefix), + memoryReservationMiB: 32, + }); + container.addMountPoints({ + containerPath: input.containerPath, + readOnly: input.readOnly, + sourceVolume: input.sourceVolume, + }); + this.bindContainerToLogGroup(container, input.logGroupName); + } + + /** Shared volume plus sidecar so application.log reaches the preserved group. */ + private attachApplicationFileLogs( + role: "cluster-manager" | "vdc" | "scheduler", + taskDefinition: ecs.Ec2TaskDefinition, + container: ecs.ContainerDefinition, + ): void { + const logGroupName = this.agentLogGroupName(role, role === "vdc" ? "controller" : undefined); + taskDefinition.addVolume({ name: "application-logs" }); + container.addMountPoints({ + containerPath: APPLICATION_LOG_DIRECTORY, + readOnly: false, + sourceVolume: "application-logs", + }); + this.addLogTailContainer(taskDefinition, { + constructId: `${role}-application-logs`, + containerId: `${role}-application-logs`, + directories: [APPLICATION_LOG_DIRECTORY], + logGroupName, + streamPrefix: STREAM_PREFIX_APPLICATION, + sourceVolume: "application-logs", + containerPath: APPLICATION_LOG_DIRECTORY, + readOnly: true, + }); + } + + /** + * Creates the task ENI group for one role. + * + * The group re-establishes the network position the module's host security group holds, rule by + * rule, using only the VPC the earlier cluster stack created. Nothing here reads a group an + * application stack publishes later. + * + * Two host rules have no task counterpart. SSH from the bastion group is one: a task runs no ssh + * daemon, and the module stacks already remove that rule when the container flag is on. The + * separate `8443` allowance for the external load balancer group is the other: it is a subset of + * the API rule below, because every load balancer sits in this VPC. + * + * Where a host rule opens every port and protocol from the VPC, the task rule names the ports the + * task actually listens on instead. The controller and the gateway are the two host groups with + * such a rule, and their listeners are `8443` for the API, `8443` over UDP for the gateway's QUIC + * transport, and `8989` for the gateway health check. + */ + private taskSecurityGroup(role: ContainerRole): ec2.SecurityGroup { + const existing = this.taskSecurityGroups[role]; + if (existing !== undefined) return existing; + + const securityGroup = new ec2.SecurityGroup(this.stack, `${role}-task-security-group`, { + allowAllOutbound: false, + description: `Security group for the ${role} ECS task`, + securityGroupName: this.buildResourceName(`ecs-${role}-task-security-group`), + vpc: this.cluster.vpc, + }); + const vpcPeer = ec2.Peer.ipv4(this.cluster.vpc.vpcCidrBlock); + const apiIngress = (): void => { + securityGroup.addIngressRule( + vpcPeer, + ec2.Port.tcp(8443), + "Allow HTTP traffic from all VPC nodes for API access", + ); + }; + + if (role === "dcv-broker") { + // The broker group is the one host group with no API rule: it is reached on the three broker + // ports, and the brokers reach each other on the three fixed discovery ports. + securityGroup.addIngressRule(vpcPeer, ec2.Port.tcpRange(8444, 8446), "Allow VPC to broker ports 8444-8446"); + for (const port of [47100, 47200, 47500]) { + securityGroup.addIngressRule(securityGroup, ec2.Port.tcp(port), `Allow broker to broker port ${port}`); + } + } else { + apiIngress(); + } + if (role === "scheduler") { + // The batch server and its execution hosts use reserved and ephemeral ports in both + // directions, which is why the host group allows every TCP port from the VPC. + securityGroup.addIngressRule( + vpcPeer, + ec2.Port.tcpRange(0, 65535), + "Allow all TCP traffic from VPC to scheduler", + ); + } + if (role === "dcv-gateway") { + securityGroup.addIngressRule( + vpcPeer, + ec2.Port.udp(8443), + "Allow UDP traffic from all VPC nodes for the QUIC transport", + ); + securityGroup.addIngressRule( + vpcPeer, + ec2.Port.tcp(8989), + "Allow TCP traffic access for HealthCheck to DCV Connection Gateway", + ); + this.addGatewayClientIngress(securityGroup); + } + + securityGroup.addEgressRule(ec2.Peer.ipv4("0.0.0.0/0"), ec2.Port.tcpRange(0, 65535), TCP_EGRESS_DESCRIPTION); + securityGroup.addEgressRule(ec2.Peer.ipv6("::/0"), ec2.Port.tcpRange(0, 65535), TCP_EGRESS_DESCRIPTION); + if (role === "dcv-gateway" && this.quicSupported()) { + // The gateway group carries this pair on a cluster with the QUIC transport on, because the + // gateway then reaches each desktop over UDP. + securityGroup.addEgressRule( + ec2.Peer.ipv4("0.0.0.0/0"), + ec2.Port.udpRange(0, 65535), + "Allow all egress for UDP for QUIC Support on DCV Connection Gateway", + ); + securityGroup.addEgressRule( + ec2.Peer.ipv6("::/0"), + ec2.Port.udpRange(0, 65535), + "Allow all egress for UDP for QUIC Support on DCV Connection Gateway", + ); + } + // The host groups of the roles that join the directory carry this pair, and the task joins the + // same way. The broker and the gateway host groups do not, so neither do their tasks. + if (role !== "dcv-broker" && role !== "dcv-gateway" && isDsActivedirectory(this.context)) { + securityGroup.addIngressRule(vpcPeer, ec2.Port.udpRange(0, 1024), DIRECTORY_SERVICE_INGRESS_DESCRIPTION); + securityGroup.addEgressRule( + ec2.Peer.ipv4("0.0.0.0/0"), + ec2.Port.udpRange(0, 1024), + DIRECTORY_SERVICE_EGRESS_DESCRIPTION, + ); + securityGroup.addEgressRule( + ec2.Peer.ipv6("::/0"), + ec2.Port.udpRange(0, 1024), + DIRECTORY_SERVICE_EGRESS_DESCRIPTION, + ); + } + + this.addCommonTags(securityGroup); + this.taskSecurityGroups[role] = securityGroup; + return securityGroup; + } + + /** True when the desktop module serves sessions over the QUIC transport. */ + private quicSupported(): boolean { + return this.context.config.getBool("virtual-desktop-controller.dcv_session.quic_support", false); + } + + /** + * Allows desktop clients to reach the gateway task from the prefix lists the host group allows. + * + * A network load balancer target group of type TCP and UDP preserves the client address and cannot + * be told not to, so the client address, not a load balancer address, is what the task sees. The + * peers are the cluster prefix list the cluster stack maintains plus any prefix list the operator + * added, which is exactly the set the host group allows all traffic from. The ports are the + * gateway's two listener ports rather than all traffic. + */ + private addGatewayClientIngress(securityGroup: ec2.SecurityGroup): void { + const clusterPrefixListId = this.context.config.getString("cluster.network.cluster_prefix_list_id"); + const operatorPrefixListIds = this.context.config.getList("cluster.network.prefix_list_ids", []); + if (clusterPrefixListId === undefined || clusterPrefixListId === "") { + Annotations.of(this.stack).addWarning( + "cluster.network.cluster_prefix_list_id is not set, so the gateway task allows desktop clients from inside the VPC only. A cluster deployed by this tool always has the setting; a synthesis without it is not a deployable cluster.", + ); + } + const prefixListIds = clusterPrefixListId === undefined || clusterPrefixListId === "" + ? operatorPrefixListIds + : [clusterPrefixListId, ...operatorPrefixListIds]; + for (const prefixListId of prefixListIds) { + securityGroup.addIngressRule( + ec2.Peer.prefixList(prefixListId), + ec2.Port.tcp(8443), + "Allow TCP traffic access from Prefix List to DCV Connection Gateway", + ); + securityGroup.addIngressRule( + ec2.Peer.prefixList(prefixListId), + ec2.Port.udp(8443), + "Allow UDP traffic access from Prefix List to DCV Connection Gateway", + ); + } + } + + /** Builds the egress-only group attached to container-instance ENIs. */ + private buildHostSecurityGroup(): ec2.SecurityGroup { + const securityGroup = new ec2.SecurityGroup(this.stack, "ecs-host-security-group", { + allowAllOutbound: true, + description: "Security group for ECS container hosts", + securityGroupName: this.buildResourceName("ecs-host-security-group"), + vpc: this.cluster.vpc, + }); + this.addCommonTags(securityGroup); + return securityGroup; + } + + /** Builds the role used by ECS container instances. */ + private buildHostRole(): iam.Role { + const role = new iam.Role(this.stack, "ecs-host-role", { + assumedBy: new iam.ServicePrincipal("ec2.amazonaws.com"), + roleName: this.buildResourceName("ecs-host-role", true), + }); + role.addManagedPolicy( + iam.ManagedPolicy.fromAwsManagedPolicyName("service-role/AmazonEC2ContainerServiceforEC2Role"), + ); + for (const policyArn of this.getEc2InstanceManagedPolicies()) { + role.addManagedPolicy(iam.ManagedPolicy.fromManagedPolicyArn(this.stack, `ecs-host-policy-${policyArn}`, policyArn)); + } + return role; + } + + /** + * The configured host family. Its architecture selects the host image and the processor + * architecture of every task definition, so the two cannot disagree by configuration. A family + * this code cannot resolve is refused here: the alternative is a launch template with an image for + * the wrong architecture, whose hosts boot and never register, and tasks that are never placed. + * + * This assumes `ecs.image` is a manifest with both architectures. A single-architecture image on + * the other family is a task that pulls and fails to run, which the stack cannot see at synthesis. + */ + private hostInstanceType(): ec2.InstanceType { + const configured = this.requiredString("ecs.hosts.instance_type"); + const instanceType = new ec2.InstanceType(configured); + try { + instanceType.architecture; + } catch (cause) { + throw new Error( + `ecs.hosts.instance_type ${configured} is not an instance type whose architecture this stack can resolve. Set a family with a size, for example m7g.large.`, + { cause }, + ); + } + return instanceType; + } + + /** + * Makes the host group deletable by the platform on its own. + * + * This depends on the group, so CloudFormation creates it straight after the group and deletes + * it straight before, which is the ordering that matters: on a rollback nothing of ours is + * running, and this is the last thing to execute while the group still exists. Without it a + * container stack that fails for any reason cannot roll back, because the group cannot remove + * instances that are protected from scale-in and no code clears the flag. + */ + private releaseScaleInProtectionOnDelete(): void { + const onEvent = new lambda.Function(this.stack, "host-scale-in-release-fn", { + code: lambda.Code.fromInline(RELEASE_SCALE_IN_PROTECTION_HANDLER), + handler: "index.handler", + runtime: lambda.Runtime.PYTHON_3_13, + timeout: Duration.minutes(5), + }); + onEvent.addToRolePolicy( + new iam.PolicyStatement({ + actions: ["autoscaling:DescribeAutoScalingGroups"], + resources: ["*"], + }), + ); + onEvent.addToRolePolicy( + new iam.PolicyStatement({ + actions: ["autoscaling:SetInstanceProtection", "autoscaling:UpdateAutoScalingGroup"], + resources: [ + `arn:${Aws.PARTITION}:autoscaling:${Aws.REGION}:${Aws.ACCOUNT_ID}:autoScalingGroup:*:autoScalingGroupName/${this.hostAutoScalingGroup.autoScalingGroupName}`, + ], + }), + ); + this.scaleInRelease = new CustomResource(this.stack, "host-scale-in-release", { + properties: { AutoScalingGroupName: this.hostAutoScalingGroup.autoScalingGroupName }, + resourceType: "Custom::ReleaseScaleInProtection", + serviceToken: new Provider(this.stack, "host-scale-in-release-provider", { + onEventHandler: onEvent, + }).serviceToken, + }); + this.scaleInRelease.node.addDependency(this.hostAutoScalingGroup); + } + + /** Builds the ECS host group and its metadata-isolating launch template. */ + private buildHostAutoScalingGroup(): autoscaling.AutoScalingGroup { + const userData = ec2.UserData.forLinux(); + userData.addCommands( + "mkdir -p /etc/ecs", + `echo ECS_CLUSTER=${this.ecsCluster.clusterName} >> /etc/ecs/ecs.config`, + "echo ECS_AWSVPC_BLOCK_IMDS=true >> /etc/ecs/ecs.config", + "install -d -o root -g root -m 0755 /var/run/datadog", + ...this.hostStorageCommands(), + ); + const launchTemplate = new ec2.LaunchTemplate(this.stack, "ecs-host-launch-template", { + blockDevices: [ + { + deviceName: "/dev/xvda", + volume: ec2.BlockDeviceVolume.ebs(this.context.config.getInt("ecs.hosts.volume_size", 60), { + encrypted: true, + volumeType: ec2.EbsDeviceVolumeType.GP3, + }), + }, + ], + instanceType: this.hostInstanceType(), + machineImage: ec2.MachineImage.resolveSsmParameterAtLaunch(hostImageParameter(this.hostArchitecture)), + httpPutResponseHopLimit: 1, + requireImdsv2: true, + role: this.hostRole, + securityGroup: this.hostSecurityGroup, + userData, + }); + const autoScalingGroup = new autoscaling.AutoScalingGroup(this.stack, "ecs-host-auto-scaling-group", { + autoScalingGroupName: this.buildResourceName("ecs-hosts"), + launchTemplate, + maxCapacity: this.context.config.getInt("ecs.hosts.max", 4), + minCapacity: this.context.config.getInt("ecs.hosts.min", 3), + newInstancesProtectedFromScaleIn: true, + vpc: this.cluster.vpc, + vpcSubnets: { subnets: this.cluster.privateSubnets }, + }); + this.addCommonTags(autoScalingGroup); + return autoScalingGroup; + } + + /** + * Mount commands are emitted only for ONTAP and Lustre, whose host-mounted + * paths are subsequently bind-mounted into the tasks. + */ + private hostStorageCommands(): string[] { + const commands: string[] = []; + for (const mount of this.storageMounts()) { + if (mount.hostPath === undefined) continue; + commands.push(`mkdir -p ${mount.hostPath}`); + const storage = this.context.config.getConfig(`shared-storage.${mount.name}`, {}); + if (storage === undefined) continue; + const provider = storage["provider"]; + if (provider === "fsx_lustre") { + const lustre = storage["fsx_lustre"]; + if (isRecord(lustre) && typeof lustre["dns"] === "string" && typeof lustre["mount_name"] === "string") { + commands.push( + `mount -t lustre ${lustre["dns"]}@tcp:/${lustre["mount_name"]} ${mount.hostPath}`, + ); + } + } + if (provider === "fsx_netapp_ontap") { + const ontap = storage["fsx_netapp_ontap"]; + const svm = isRecord(ontap) ? ontap["svm"] : undefined; + if (isRecord(svm) && typeof svm["nfs_dns"] === "string") { + commands.push(`mount -t nfs ${svm["nfs_dns"]} ${mount.hostPath}`); + } + } + } + return commands; + } + + /** Applies the physical-name rule used by application role constructs. */ + private applicationRoleName(name: string): string { + const fullName = this.buildResourceName(name, true); + return fullName.length <= 64 + ? fullName + : buildTrimmedResourceName(this.clusterName, name, this.awsRegion, 64); + } + + /** An IAM role ARN built from a name this account applies the same rule to. */ + private roleArnByName(name: string): string { + return Fn.join("", ["arn:", Aws.PARTITION, ":iam::", Aws.ACCOUNT_ID, ":role/", this.applicationRoleName(name)]); + } + + /** + * The managed policies the module instance roles carry. A task role gets the same set, so the + * permissions a task holds do not change when it stops borrowing the host role. + */ + private taskManagedPolicies(): iam.IManagedPolicy[] { + if (this.instanceManagedPolicies !== undefined) return this.instanceManagedPolicies; + this.instanceManagedPolicies = this.getEc2InstanceManagedPolicies().map((policyArn, index) => + iam.ManagedPolicy.fromManagedPolicyArn(this.stack, `ecs-task-managed-policy-${index}`, policyArn), + ); + return this.instanceManagedPolicies; + } + + /** + * The task identity for one role. + * + * Each role gets its own, so one task cannot use another's permissions, and the policy renders + * from the template the module's instance role renders from rather than being written again by + * hand. The trust is the account-scoped task principal. + */ + private taskRole(role: ContainerRole): iam.Role { + const existing = this.taskRoles[role]; + if (existing !== undefined) return existing; + + const taskRole = new iam.Role(this.stack, `${role}-task-role`, { + assumedBy: this.ecsTasksPrincipal(), + description: `IAM role assigned to the ${role} ECS task`, + managedPolicies: this.taskManagedPolicies(), + roleName: this.applicationRoleName(`ecs-${role}-task-role`), + }); + taskRole.attachInlinePolicy(this.buildTaskPolicy(role, taskRole)); + this.addCommonTags(taskRole); + this.taskRoles[role] = taskRole; + return taskRole; + } + + /** + * The inline policy for one task role. + * + * The scheduler template names the roles it may pass to a compute node or a spot fleet request. + * Those roles belong to the scheduler stack, which deploys after this one, so their ARNs come from + * the naming rule the account applies rather than from a setting that does not exist yet. + */ + private buildTaskPolicy(role: ContainerRole, taskRole: iam.Role): Policy { + const schedulerModuleId = this.context.config.moduleId("scheduler"); + const vars: Record = role === "scheduler" + ? { + compute_node_role_arn: this.roleArnByName(`${schedulerModuleId}-compute-node-role`), + scheduler_role_arn: taskRole.roleArn, + spot_fleet_request_role_arn: this.roleArnByName(`${schedulerModuleId}-spot-fleet-request-role`), + } + : { role_arn: taskRole.roleArn }; + return new Policy(this.context, `ecs-${role}-task-policy`, this.stack, { + moduleId: this.ideaModuleId(role), + policyTemplateName: TASK_POLICY_TEMPLATES[role], + vars, + }); + } + + /** + * Uses operator-provided certificate inputs when present. Otherwise this + * stack creates the self-signed gateway secrets before the gateway task. + */ + private loadGatewayCertificateSecrets(): { + certificateSecretArn: string; + privateKeySecretArn: string; + } { + const certificatePrefix = "virtual-desktop-controller.dcv_connection_gateway.certificate"; + const certificateSecretArn = this.context.config.getString(`${certificatePrefix}.certificate_secret_arn`); + const privateKeySecretArn = this.context.config.getString(`${certificatePrefix}.private_key_secret_arn`); + if (this.context.config.getBool(`${certificatePrefix}.provided`, false)) { + return { + certificateSecretArn: this.requiredString(`${certificatePrefix}.certificate_secret_arn`), + privateKeySecretArn: this.requiredString(`${certificatePrefix}.private_key_secret_arn`), + }; + } + if ( + certificateSecretArn !== undefined && + certificateSecretArn !== "" && + privateKeySecretArn !== undefined && + privateKeySecretArn !== "" + ) { + return { certificateSecretArn, privateKeySecretArn }; + } + + const vdcModuleId = this.context.config.moduleId("virtual-desktop-controller"); + const certificateName = `${this.clusterName}-${vdcModuleId}-gateway-certificate`; + const properties: Record = { + certificate_name: certificateName, + create_acm_certificate: false, + domain_name: `${vdcModuleId}.${this.clusterName}.idea.default`, + tags: { + Name: `${this.clusterName}-${vdcModuleId}-gateway Self Signed Certificate`, + "idea:ClusterName": this.clusterName, + "idea:ModuleName": "virtual-desktop-controller", + }, + }; + const kmsKeyId = this.context.config.getString("cluster.secretsmanager.kms_key_id"); + if (kmsKeyId !== undefined && kmsKeyId !== "") properties["kms_key_id"] = kmsKeyId; + + const certificate = new CustomResource(this.stack, "gateway-self-signed-certificate", { + properties, + resourceType: "Custom::SelfSignedCertificateConnectionGateway", + serviceToken: this.requiredString("cluster.self_signed_certificate_lambda_arn"), + }); + return { + certificateSecretArn: certificate.getAttString("certificate_secret_arn"), + privateKeySecretArn: certificate.getAttString("private_key_secret_arn"), + }; + } + + /** Builds a secret-free execution role with an account-scoped trust policy. */ + /** The task service, trusted only from this account. */ + private ecsTasksPrincipal(): iam.ServicePrincipal { + return new iam.ServicePrincipal("ecs-tasks.amazonaws.com", { + conditions: { + StringEquals: { "aws:SourceAccount": this.stack.account }, + }, + }); + } + + private buildExecutionRole(constructId: string): iam.Role { + const role = new iam.Role(this.stack, constructId, { + assumedBy: this.ecsTasksPrincipal(), + roleName: this.buildResourceName(constructId, true), + }); + role.addManagedPolicy(iam.ManagedPolicy.fromAwsManagedPolicyName("service-role/AmazonECSTaskExecutionRolePolicy")); + return role; + } + + /** Creates the five awsvpc application task definitions and services. */ + private buildApplicationServices(): void { + this.roleResources["cluster-manager"] = this.buildApiService("cluster-manager", { + component: undefined, + targetGroups: [ + this.applicationTargetGroup("cm-ecs-e", 8443, "/healthcheck"), + this.applicationTargetGroup("cm-ecs-i", 8443, "/healthcheck"), + this.applicationTargetGroup("cm-ecs-w", 8443, "/healthcheck"), + ], + }); + this.roleResources.vdc = this.buildApiService("vdc", { + component: "controller", + targetGroups: [ + this.applicationTargetGroup("vdc-ecs-e", 8443, "/healthcheck"), + this.applicationTargetGroup("vdc-ecs-i", 8443, "/healthcheck"), + ], + }); + this.roleResources.scheduler = this.buildSchedulerService(); + this.roleResources["dcv-broker"] = this.buildBrokerService(); + this.roleResources["dcv-gateway"] = this.buildGatewayService(); + } + + /** + * Attaches every target group this stack creates to the listener that serves it. + * + * A service may not name a target group that has no load balancer, and this module deploys + * before the modules whose stacks create those listener rules today, so without this the + * services are refused and the stack rolls back. The listeners themselves are cluster-stack + * resources and exist well before this module runs. + * + * The endpoint names are the module stacks' own, deliberately. The handler keys a rule by its + * `idea:EndpointName` tag and adopts one that is already there, so the module stack's later + * resource converges on this same rule rather than making a second, and on a cluster that + * already has the rule this modifies it where it stands, preserving its identity and priority. + */ + private buildEndpoints(): void { + const config = this.context.config; + const serviceToken = this.requiredString("cluster.cluster_endpoints_lambda_arn"); + const externalListener = this.requiredString( + "cluster.load_balancers.external_alb.https_listener_arn", + ); + const internalListener = this.requiredString( + "cluster.load_balancers.internal_alb.https_listener_arn", + ); + const clusterManagerId = config.moduleId("cluster-manager"); + const schedulerId = config.moduleId("scheduler"); + const vdcId = config.moduleId("virtual-desktop-controller"); + + /** + * The service may not be created before the endpoint that gives its target group a load + * balancer. Both only reference the target group, so without this they are siblings and + * CloudFormation creates them in parallel: the service wins the race and is refused, which is + * the failure this whole method exists to prevent. + */ + const attachBefore = (role: ContainerRole, endpoint: CustomResource): void => { + this.roleResources[role]?.service.node.addDependency(endpoint); + }; + + /** One routed endpoint: a rule at the module's own priority and path patterns. */ + const rule = ( + role: ContainerRole, + constructId: string, + endpointName: string, + listenerArn: string, + prefix: string, + targetGroup: elbv2.IApplicationTargetGroup, + ): void => { + const endpoint = new CustomResource(this.stack, constructId, { + properties: { + endpoint_name: endpointName, + listener_arn: listenerArn, + priority: config.getInt(`${prefix}.priority`, undefined, { required: true }), + conditions: [ + { + Field: "path-pattern", + Values: config.getList(`${prefix}.path_patterns`, [], { required: true }), + }, + ], + actions: [{ Type: "forward", TargetGroupArn: targetGroup.targetGroupArn }], + }, + resourceType: "Custom::EcsEndpoint", + serviceToken, + }); + attachBefore(role, endpoint); + }; + + /** One listener whose default action is this target group. No rule, so no priority. */ + const defaultAction = ( + role: ContainerRole, + constructId: string, + endpointName: string, + listenerArn: string, + targetGroup: elbv2.IApplicationTargetGroup, + ): void => { + const endpoint = new CustomResource(this.stack, constructId, { + properties: { + endpoint_name: endpointName, + listener_arn: listenerArn, + priority: 0, + default_action: true, + actions: [{ Type: "forward", TargetGroupArn: targetGroup.targetGroupArn }], + }, + resourceType: "Custom::EcsDefaultEndpoint", + serviceToken, + }); + attachBefore(role, endpoint); + }; + + const groups = (role: ContainerRole): elbv2.IApplicationTargetGroup[] => + this.roleResources[role]?.targetGroups as elbv2.IApplicationTargetGroup[]; + + const clusterManager = groups("cluster-manager"); + rule("cluster-manager", "cm-external-endpoint", `${clusterManagerId}-external-endpoint`, externalListener, "cluster-manager.endpoints.external", clusterManager[0]); + rule("cluster-manager", "cm-internal-endpoint", `${clusterManagerId}-internal-endpoint`, internalListener, "cluster-manager.endpoints.internal", clusterManager[1]); + defaultAction("cluster-manager", "cm-web-portal-endpoint", `${clusterManagerId}-web-portal-endpoint`, externalListener, clusterManager[2]); + + const scheduler = groups("scheduler"); + rule("scheduler", "scheduler-external-endpoint", `${schedulerId}-external-endpoint`, externalListener, "scheduler.endpoints.external", scheduler[0]); + rule("scheduler", "scheduler-internal-endpoint", `${schedulerId}-internal-endpoint`, internalListener, "scheduler.endpoints.internal", scheduler[1]); + + const vdc = groups("vdc"); + rule("vdc", "vdc-external-endpoint", `${vdcId}-controller-endpoint-ext`, externalListener, "virtual-desktop-controller.controller.endpoints.external", vdc[0]); + rule("vdc", "vdc-internal-endpoint", `${vdcId}-controller-endpoint-int`, internalListener, "virtual-desktop-controller.controller.endpoints.internal", vdc[1]); + + // The broker's three listeners each forward everything to one target group, so each is a + // default action. The agent endpoint registers under the client endpoint's name, which is how + // it is deployed today and is the name the module stack uses. + const broker = groups("dcv-broker"); + defaultAction("dcv-broker", "broker-client-endpoint", "broker-client-endpoint", this.requiredString("cluster.load_balancers.internal_alb.dcv_broker_client_listener_arn"), broker[0]); + defaultAction("dcv-broker", "broker-agent-endpoint", "broker-client-endpoint", this.requiredString("cluster.load_balancers.internal_alb.dcv_broker_agent_listener_arn"), broker[1]); + defaultAction("dcv-broker", "broker-gateway-endpoint", "broker-gateway-endpoint", this.requiredString("cluster.load_balancers.internal_alb.dcv_broker_gateway_listener_arn"), broker[2]); + } + + /** Creates a cluster-manager or VDC service with HTTPS API target groups. */ + private buildApiService( + role: "cluster-manager" | "vdc", + input: { component: string | undefined; targetGroups: elbv2.ApplicationTargetGroup[] }, + ): RoleResources { + const taskDefinition = this.buildTaskDefinition(role); + const logGroupName = this.agentLogGroupName(role, input.component); + const container = taskDefinition.addContainer(`${role}-container`, { + cpu: this.roleSizing(role).cpu, + dockerLabels: this.dockerLabels(role), + environment: this.commonEnvironment(role), + image: ecs.ContainerImage.fromRegistry(this.requiredString("ecs.image")), + logging: this.adoptedLogDriver(`${role}-logs`, logGroupName, STREAM_PREFIX_APPLICATION), + memoryLimitMiB: this.roleSizing(role).memory, + }); + container.addPortMappings({ containerPort: 8443, protocol: ecs.Protocol.TCP }); + this.bindContainerToLogGroup(container, logGroupName); + this.addStorageMounts(taskDefinition, container); + this.attachApplicationFileLogs(role, taskDefinition, container); + const service = this.buildEc2Service(role, taskDefinition, this.roleSizing(role).desired, { + minHealthyPercent: 50, + maxHealthyPercent: 200, + }); + for (const targetGroup of input.targetGroups) service.attachToApplicationTargetGroup(targetGroup); + return { service, targetGroups: input.targetGroups }; + } + + /** Creates the scheduler service and its deep container health check. */ + private buildSchedulerService(): RoleResources { + const role: ContainerRole = "scheduler"; + const taskDefinition = this.buildTaskDefinition(role); + const logGroupName = this.agentLogGroupName(role); + const openPbsLogGroupName = `${logGroupName}/openpbs`; + const container = taskDefinition.addContainer("scheduler-container", { + cpu: this.roleSizing(role).cpu, + dockerLabels: this.dockerLabels(role), + environment: { + ...this.commonEnvironment(role), + IDEA_ROUTE53_ZONE_ID: this.requiredString("cluster.route53.private_hosted_zone_id"), + IDEA_SCHEDULER_DNS_NAME: `scheduler.${this.clusterName}.${this.awsRegion}.local`, + PBS_HOME: SCHEDULER_PBS_HOME, + PBS_NODE_FAIL_REQUEUE: "600", + }, + healthCheck: { + command: [ + "CMD-SHELL", + "qstat -B && curl --fail --silent --show-error --unix-socket /run/idea.sock --max-time 4 --header 'Content-Type: application/json' --data '{\"header\":{\"namespace\":\"Scheduler.ListActiveJobs\"}}' http://localhost/scheduler/api/v1", + ], + interval: Duration.seconds(30), + retries: 3, + // The container check has no allowance but this one, on a first start and on every + // replacement, so it matches the load balancer grace. A shorter period would let the + // platform kill the container while `roles/scheduler.sh` is still inside its own wait for + // the batch server. + startPeriod: this.taskStartAllowance("scheduler"), + }, + image: ecs.ContainerImage.fromRegistry(this.requiredString("ecs.image")), + logging: this.adoptedLogDriver("scheduler-logs", logGroupName, STREAM_PREFIX_APPLICATION), + memoryLimitMiB: this.roleSizing(role).memory, + }); + container.addPortMappings({ containerPort: 8443, protocol: ecs.Protocol.TCP }); + this.bindContainerToLogGroup(container, logGroupName); + this.addSchedulerStorage(taskDefinition, container); + this.addStorageMounts(taskDefinition, container); + this.attachApplicationFileLogs(role, taskDefinition, container); + this.addLogTailContainer(taskDefinition, { + constructId: "scheduler-openpbs-logs", + containerId: "scheduler-openpbs-logs", + directories: [ + `${SCHEDULER_PBS_HOME}/server_logs`, + `${SCHEDULER_PBS_HOME}/sched_logs`, + `${SCHEDULER_PBS_HOME}/server_priv/accounting`, + ], + logGroupName: openPbsLogGroupName, + streamPrefix: STREAM_PREFIX_OPENPBS, + sourceVolume: "scheduler-pbs", + containerPath: SCHEDULER_PBS_HOME, + readOnly: true, + }); + const targetGroups = [ + this.applicationTargetGroup("sched-ecs-e", 8443, "/healthcheck", 15), + this.applicationTargetGroup("sched-ecs-i", 8443, "/healthcheck", 15), + ]; + // The batch server holds a single-writer lock on its state directory, so two scheduler tasks + // cannot run at once: the second to start fails to take the lock. A maximum of one hundred + // leaves no room for a replacement to start before the running task stops, which is why this + // one service differs from the other four. The cost is that a replacement is a short batch + // server outage rather than a handover: running jobs survive, submissions fail while it is down. + const service = this.buildEc2Service(role, taskDefinition, this.roleSizing(role).desired, { + minHealthyPercent: 0, + maxHealthyPercent: 100, + }); + for (const targetGroup of targetGroups) service.attachToApplicationTargetGroup(targetGroup); + return { service, targetGroups }; + } + + /** Creates the service-discoverable DCV broker and its three listeners. */ + private buildBrokerService(): RoleResources { + const role: ContainerRole = "dcv-broker"; + const taskDefinition = this.buildTaskDefinition(role); + const logGroupName = this.agentLogGroupName(role, "dcv-broker"); + const container = taskDefinition.addContainer("dcv-broker-container", { + cpu: this.roleSizing(role).cpu, + dockerLabels: this.dockerLabels(role), + environment: { + ...this.commonEnvironment(role), + IDEA_COGNITO_PROVIDER_URL: this.requiredString("identity-provider.cognito.provider_url"), + IDEA_SERVICE_DISCOVERY_NAME: `vdc-broker.${this.clusterName}.ecs.local`, + // The task network namespace has no second address family, so a dual-stack JVM fails to + // create its sockets. The virtual machine reads this variable itself at startup, which is + // why it is this name and not one the vendor launcher would have to pass on. + JAVA_TOOL_OPTIONS: "-Djava.net.preferIPv4Stack=true", + }, + image: ecs.ContainerImage.fromRegistry(this.requiredString("ecs.image")), + logging: this.adoptedLogDriver("dcv-broker-logs", logGroupName, STREAM_PREFIX_BROKER), + memoryLimitMiB: this.roleSizing(role).memory, + }); + for (const port of [8444, 8445, 8446]) { + container.addPortMappings({ containerPort: port, protocol: ecs.Protocol.TCP }); + } + taskDefinition.addVolume({ name: "broker-logs" }); + container.addMountPoints({ + containerPath: BROKER_LOG_DIRECTORY, + readOnly: false, + sourceVolume: "broker-logs", + }); + this.bindContainerToLogGroup(container, logGroupName); + this.addStorageMounts(taskDefinition, container); + this.addLogTailContainer(taskDefinition, { + constructId: "dcv-broker-file-logs", + containerId: "dcv-broker-file-logs", + directories: [BROKER_LOG_DIRECTORY], + logGroupName, + streamPrefix: STREAM_PREFIX_BROKER, + sourceVolume: "broker-logs", + containerPath: BROKER_LOG_DIRECTORY, + readOnly: true, + }); + const targetGroups = [ + this.applicationTargetGroup("brk-ecs-c", 8444, "/health"), + this.applicationTargetGroup("brk-ecs-a", 8445, "/health"), + this.applicationTargetGroup("brk-ecs-g", 8446, "/health"), + ]; + const service = this.buildEc2Service(role, taskDefinition, this.roleSizing(role).desired, { + cloudMapOptions: { + cloudMapNamespace: this.namespace, + dnsRecordType: servicediscovery.DnsRecordType.A, + dnsTtl: Duration.seconds(10), + failureThreshold: 1, + name: "vdc-broker", + }, + minHealthyPercent: 50, + maxHealthyPercent: 200, + }); + for (const targetGroup of targetGroups) service.attachToApplicationTargetGroup(targetGroup); + return { service, targetGroups }; + } + + /** Creates the DCV gateway service and target groups for TCP and TCP/UDP listeners. */ + private buildGatewayService(): RoleResources { + const role: ContainerRole = "dcv-gateway"; + const taskDefinition = this.buildTaskDefinition(role, this.gatewayExecutionRole); + const logGroupName = this.agentLogGroupName(role, "dcv-connection-gateway"); + const certificate = secretsmanager.Secret.fromSecretCompleteArn( + this.stack, + "gateway-certificate-secret", + this.gatewayCertificateSecretArn, + ); + const privateKey = secretsmanager.Secret.fromSecretCompleteArn( + this.stack, + "gateway-private-key-secret", + this.gatewayPrivateKeySecretArn, + ); + const container = taskDefinition.addContainer("dcv-gateway-container", { + cpu: this.roleSizing(role).cpu, + dockerLabels: this.dockerLabels(role), + environment: { + ...this.commonEnvironment(role), + IDEA_INTERNAL_ALB_ENDPOINT: `https://${this.requiredString("cluster.load_balancers.internal_alb.load_balancer_dns_name")}`, + }, + image: ecs.ContainerImage.fromRegistry(this.requiredString("ecs.image")), + logging: this.adoptedLogDriver("dcv-gateway-logs", logGroupName, STREAM_PREFIX_GATEWAY), + memoryLimitMiB: this.roleSizing(role).memory, + secrets: { + DCV_GATEWAY_CERT_PEM: ecs.Secret.fromSecretsManager(certificate), + DCV_GATEWAY_KEY_PEM: ecs.Secret.fromSecretsManager(privateKey), + }, + }); + // The gateway serves both stream protocols on 8443, but a container port may appear in only + // one mapping: the service API refuses a second. Under `awsvpc` the task owns its network + // interface and the mapping does not filter traffic, so one mapping publishes the port and the + // task security group is what admits each protocol. The health port 8989 is separate and + // unaffected. + container.addPortMappings({ containerPort: 8443, protocol: ecs.Protocol.TCP }); + taskDefinition.addVolume({ name: "gateway-logs" }); + container.addMountPoints({ + containerPath: GATEWAY_LOG_DIRECTORY, + readOnly: false, + sourceVolume: "gateway-logs", + }); + this.bindContainerToLogGroup(container, logGroupName); + this.addStorageMounts(taskDefinition, container); + this.addLogTailContainer(taskDefinition, { + constructId: "dcv-gateway-file-logs", + containerId: "dcv-gateway-file-logs", + directories: [GATEWAY_LOG_DIRECTORY], + logGroupName, + streamPrefix: STREAM_PREFIX_GATEWAY, + sourceVolume: "gateway-logs", + containerPath: GATEWAY_LOG_DIRECTORY, + readOnly: true, + }); + // The desktop stack attaches exactly one of these to its network load balancer, chosen by the + // same setting. Creating both leaves the other with no load balancer for ever, and a service + // may not name a target group that has none, so the service could never be created. + const quicSupported = this.context.config.getBool( + "virtual-desktop-controller.dcv_session.quic_support", + false, + ); + const targetGroups = [ + quicSupported + ? this.networkTargetGroup("gw-ecs-TUN", elbv2.Protocol.TCP_UDP) + : this.networkTargetGroup("gw-ecs-TN", elbv2.Protocol.TCP), + ]; + const service = this.buildEc2Service(role, taskDefinition, this.roleSizing(role).desired, { + minHealthyPercent: 50, + maxHealthyPercent: 200, + }); + for (const targetGroup of targetGroups) service.attachToNetworkTargetGroup(targetGroup); + return { service, targetGroups }; + } + + /** Creates a task definition with isolated roles and retained revisions. */ + private buildTaskDefinition( + role: ContainerRole, + executionRole: iam.IRole = this.executionRole, + ): ecs.Ec2TaskDefinition { + const taskDefinition = new ecs.Ec2TaskDefinition(this.stack, `${role}-task-definition`, { + executionRole, + networkMode: ecs.NetworkMode.AWS_VPC, + taskRole: this.taskRole(role), + }); + const resource = taskDefinition.node.defaultChild; + if (resource instanceof ecs.CfnTaskDefinition) { + resource.runtimePlatform = { + cpuArchitecture: cpuArchitecture(this.hostArchitecture), + operatingSystemFamily: "LINUX", + }; + } + taskDefinition.applyRemovalPolicy(RemovalPolicy.RETAIN); + return taskDefinition; + } + + /** + * How long this role's container takes to answer a health request. + * + * A role waits only for what its own start does: the batch server wait for the scheduler, the + * service discovery wait for the broker, and the application allowance for every role. + */ + private taskStartAllowance(role: ContainerRole): Duration { + if (role === "scheduler") return Duration.seconds(PBS_SERVER_WAIT_SECONDS + APPLICATION_START_SECONDS); + if (role === "dcv-broker") return Duration.seconds(BROKER_DISCOVERY_WAIT_SECONDS + APPLICATION_START_SECONDS); + return Duration.seconds(APPLICATION_START_SECONDS); + } + + /** The task start plus the time its target needs to register as healthy behind a load balancer. */ + private healthCheckGrace(role: ContainerRole): Duration { + return Duration.seconds(this.taskStartAllowance(role).toSeconds() + TARGET_REGISTRATION_SECONDS); + } + + /** Creates an EC2 service using the shared capacity provider. */ + private buildEc2Service( + role: ContainerRole, + taskDefinition: ecs.Ec2TaskDefinition, + desiredCount: number, + input: { + cloudMapOptions?: ecs.CloudMapOptions; + minHealthyPercent: number; + maxHealthyPercent: number; + }, + ): ecs.Ec2Service { + const service = new ecs.Ec2Service(this.stack, `${role}-service`, { + capacityProviderStrategies: [{ capacityProvider: this.capacityProvider.capacityProviderName, weight: 1 }], + circuitBreaker: { rollback: true }, + cloudMapOptions: input.cloudMapOptions, + cluster: this.ecsCluster, + desiredCount, + // An operator's diagnosis path into a running control-plane task, per task and per session + // and logged to the cluster's exec group. The permission it needs is already on every task + // role through the instance managed policies, so this uses a standing grant rather than + // adding one. It stays out of this tool's own identity: the tool observes and refuses. + enableExecuteCommand: true, + healthCheckGracePeriod: this.healthCheckGrace(role), + maxHealthyPercent: input.maxHealthyPercent, + minHealthyPercent: input.minHealthyPercent, + placementConstraints: role === "scheduler" ? undefined : [ecs.PlacementConstraint.distinctInstances()], + securityGroups: [this.taskSecurityGroup(role)], + taskDefinition, + vpcSubnets: { subnets: this.cluster.privateSubnets }, + }); + // A service is the likeliest thing in this stack to fail, and the release has to already + // exist when it does. Depending on the host group alone is not enough: the two are created in + // parallel, and a failing service cancels the release before it finishes, so the rollback has + // nothing to clear the protection with. + service.node.addDependency(this.scaleInRelease); + return service; + } + + /** Creates an HTTPS application target group for an awsvpc service. */ + private applicationTargetGroup( + identifier: string, + port: number, + healthCheckPath: string, + deregistrationDelaySeconds?: number, + ): elbv2.ApplicationTargetGroup { + return new elbv2.ApplicationTargetGroup(this.stack, `${identifier}-target-group`, { + deregistrationDelay: + deregistrationDelaySeconds === undefined ? undefined : Duration.seconds(deregistrationDelaySeconds), + healthCheck: { path: healthCheckPath, protocol: elbv2.Protocol.HTTPS }, + port, + protocol: elbv2.ApplicationProtocol.HTTPS, + targetGroupName: this.getTargetGroupName(identifier), + targetType: elbv2.TargetType.IP, + vpc: this.cluster.vpc, + }); + } + + /** Creates an IP target group for the existing network load balancer. */ + private networkTargetGroup(identifier: string, protocol: elbv2.Protocol): elbv2.NetworkTargetGroup { + const targetGroup = new elbv2.NetworkTargetGroup(this.stack, `${identifier}-target-group`, { + connectionTermination: true, + healthCheck: { + port: "8989", + protocol: elbv2.Protocol.TCP, + }, + port: 8443, + protocol, + targetGroupName: this.getTargetGroupName(identifier), + targetType: elbv2.TargetType.IP, + vpc: this.cluster.vpc, + }); + targetGroup.setAttribute("stickiness.enabled", "true"); + targetGroup.setAttribute("stickiness.type", "source_ip"); + return targetGroup; + } + + /** Returns the configured CPU, memory and desired count for a role. */ + private roleSizing(role: ContainerRole): RoleSizing { + return { + cpu: this.requiredInt(`ecs.tasks.${role}.cpu`), + memory: this.requiredInt(`ecs.tasks.${role}.memory`), + desired: this.requiredInt(`ecs.tasks.${role}.desired`), + }; + } + + /** Common task environment values. */ + private commonEnvironment(role: ContainerRole): Record { + return { + AWS_DEFAULT_REGION: this.awsRegion, + DD_DOGSTATSD_URL: "unix:///var/run/datadog/dsd.socket", + IDEA_CLUSTER_NAME: this.clusterName, + IDEA_CONTAINER_ROLE: role, + IDEA_MODULE_ID: this.ideaModuleId(role), + IDEA_MODULE_NAME: this.ideaModuleName(role), + IDEA_MODULE_SET: this.context.config.moduleSet, + }; + } + + /** Labels consumed by the observability agent. */ + private dockerLabels(role: ContainerRole): Record { + return { + "com.datadoghq.tags.env": this.clusterName, + "com.datadoghq.tags.service": role, + }; + } + + /** Returns EFS and host bind mounts declared by shared-storage settings. */ + private storageMounts(): StorageMount[] { + const mounts: StorageMount[] = []; + const storageRoot = this.context.config.getConfig("shared-storage", {}) ?? {}; + for (const [name, storage] of Object.entries(storageRoot)) { + if (!isRecord(storage) || typeof storage["mount_dir"] !== "string" || typeof storage["provider"] !== "string") { + continue; + } + const mountPath = storage["mount_dir"]; + if (storage["provider"] === "efs") { + const efs = storage["efs"]; + if (isRecord(efs) && typeof efs["file_system_id"] === "string") { + mounts.push({ fileSystemId: efs["file_system_id"], mountPath, name }); + } + } + if (storage["provider"] === "fsx_lustre" || storage["provider"] === "fsx_netapp_ontap") { + mounts.push({ hostPath: mountPath, mountPath, name }); + } + } + return mounts; + } + + /** Adds storage and the optional DogStatsD host volume to an app task. */ + private addStorageMounts(taskDefinition: ecs.Ec2TaskDefinition, container: ecs.ContainerDefinition): void { + for (const mount of this.storageMounts()) { + const volumeName = `storage-${mount.name}`; + if (mount.fileSystemId !== undefined) { + taskDefinition.addVolume({ + efsVolumeConfiguration: { fileSystemId: mount.fileSystemId }, + name: volumeName, + }); + } else if (mount.hostPath !== undefined) { + taskDefinition.addVolume({ host: { sourcePath: mount.hostPath }, name: volumeName }); + } else { + continue; + } + container.addMountPoints({ containerPath: mount.mountPath, readOnly: false, sourceVolume: volumeName }); + } + if (this.context.config.getBool("ecs.datadog.enabled", false)) { + taskDefinition.addVolume({ host: { sourcePath: "/var/run/datadog" }, name: "datadog" }); + container.addMountPoints({ + containerPath: "/var/run/datadog", + readOnly: true, + sourceVolume: "datadog", + }); + } + } + + /** Adds a scheduler-only EFS file system for persistent PBS state. */ + private addSchedulerStorage( + taskDefinition: ecs.Ec2TaskDefinition, + container: ecs.ContainerDefinition, + ): void { + const schedulerSecurityGroup = this.taskSecurityGroup("scheduler"); + const fileSystemSecurityGroup = new ec2.SecurityGroup( + this.stack, + "scheduler-pbs-file-system-security-group", + { + allowAllOutbound: false, + description: "Allows NFS only from the scheduler task and its host", + vpc: this.cluster.vpc, + }, + ); + fileSystemSecurityGroup.addIngressRule( + schedulerSecurityGroup, + ec2.Port.tcp(2049), + "Allow NFS from the scheduler task", + ); + // Which interface carries the mount, the task's or the container host's, is a property of the + // container agent rather than of this template, and the mount fails silently from the wrong one. + // Both peers are groups this stack owns, and the file system policy below is what actually + // limits access: only the scheduler task role, and only through its access point. + fileSystemSecurityGroup.addIngressRule( + this.hostSecurityGroup, + ec2.Port.tcp(2049), + "Allow NFS from the container host that mounts for the scheduler task", + ); + + const fileSystem = new efs.FileSystem(this.stack, "scheduler-pbs-file-system", { + encrypted: true, + securityGroup: fileSystemSecurityGroup, + vpc: this.cluster.vpc, + vpcSubnets: { subnets: this.cluster.privateSubnets }, + }); + fileSystem.applyRemovalPolicy(RemovalPolicy.RETAIN); + const accessPoint = fileSystem.addAccessPoint("scheduler-pbs-access-point", { + createAcl: { + ownerGid: "0", + ownerUid: "0", + permissions: "0700", + }, + path: "/pbs", + posixUser: { + gid: "0", + uid: "0", + }, + }); + accessPoint.applyRemovalPolicy(RemovalPolicy.RETAIN); + + const clientActions = [ + "elasticfilesystem:ClientMount", + "elasticfilesystem:ClientWrite", + "elasticfilesystem:ClientRootAccess", + ]; + // The policy is a property of the file system and the access point refers to the file system, + // so naming the access point here is a resource cycle. CloudFormation refuses the whole + // template for it at change-set creation, and synthesis cannot see it. A mount can only + // present an access point of the file system it is mounting, so requiring the shape of one + // restricts exactly as naming this one did while this file system has the single access point + // created below. + // The policy is a property of the file system, so naming the file system in it resolves an + // attribute of the resource the policy belongs to. CloudFormation counts that self reference + // as a circular dependency and refuses the template. A file system policy applies only to the + // file system carrying it, so the resource element does not have to name it. + const OWN_FILE_SYSTEM = "*"; + const accessPointOfThisFileSystem = + `arn:${Aws.PARTITION}:elasticfilesystem:${Aws.REGION}:${Aws.ACCOUNT_ID}:access-point/*`; + fileSystem.addToResourcePolicy( + new iam.PolicyStatement({ + actions: clientActions, + conditions: { + Bool: { "elasticfilesystem:AccessedViaMountTarget": "true" }, + }, + principals: [new iam.ArnPrincipal(this.taskRole("scheduler").roleArn)], + resources: [OWN_FILE_SYSTEM], + }), + ); + fileSystem.addToResourcePolicy( + new iam.PolicyStatement({ + actions: clientActions, + conditions: { + ArnNotEquals: { "aws:PrincipalArn": this.taskRole("scheduler").roleArn }, + }, + effect: iam.Effect.DENY, + principals: [new iam.AnyPrincipal()], + resources: [OWN_FILE_SYSTEM], + }), + ); + fileSystem.addToResourcePolicy( + new iam.PolicyStatement({ + actions: clientActions, + // A negated string condition is true when the key is absent, so a mount that presents no + // access point at all is denied by this statement as well. + conditions: { + StringNotLike: { + "elasticfilesystem:AccessPointArn": accessPointOfThisFileSystem, + }, + }, + effect: iam.Effect.DENY, + principals: [new iam.AnyPrincipal()], + resources: [OWN_FILE_SYSTEM], + }), + ); + + taskDefinition.addVolume({ + efsVolumeConfiguration: { + authorizationConfig: { + accessPointId: accessPoint.accessPointId, + iam: "ENABLED", + }, + fileSystemId: fileSystem.fileSystemId, + rootDirectory: "/", + transitEncryption: "ENABLED", + }, + name: "scheduler-pbs", + }); + container.addMountPoints({ + containerPath: SCHEDULER_PBS_HOME, + readOnly: false, + sourceVolume: "scheduler-pbs", + }); + } + + /** Returns a digest-pinned image hosted in a private ECR repository. */ + private datadogImage(): string { + const image = this.requiredString("ecs.datadog.image"); + const privateEcrDigest = + /^[0-9]{12}\.dkr\.ecr(?:-fips)?\.[a-z0-9-]+\.amazonaws\.com(?:\.cn)?\/[^@]+@sha256:[0-9a-f]{64}$/; + if (!privateEcrDigest.test(image)) { + throw new Error("ecs.datadog.image must be a digest-pinned private ECR image"); + } + return image; + } + + /** Creates the optional host-network observability daemon. */ + private buildDatadogService(): void { + if (!this.context.config.getBool("ecs.datadog.enabled", false)) return; + + const executionRole = this.buildExecutionRole("datadog-task-execution-role"); + // The agent needs no API calls of its own, so this role carries no policies. It exists because + // a task definition without one gets a generated role whose trust has no account condition. + const taskRole = new iam.Role(this.stack, "datadog-task-role", { + assumedBy: this.ecsTasksPrincipal(), + roleName: this.buildResourceName("datadog-task-role", true), + }); + const taskDefinition = new ecs.Ec2TaskDefinition(this.stack, "datadog-task-definition", { + executionRole, + networkMode: ecs.NetworkMode.HOST, + pidMode: ecs.PidMode.HOST, + taskRole, + }); + // Keeps the previous revision ACTIVE, so an agent image bump that fails has something to roll + // back to. + taskDefinition.applyRemovalPolicy(RemovalPolicy.RETAIN); + const apiKey = secretsmanager.Secret.fromSecretCompleteArn( + this.stack, + "datadog-api-key-secret", + this.requiredString("ecs.datadog.api_key_secret_arn"), + ); + const datadogLogGroupName = `/${this.clusterName}/${this.context.config.moduleId("ecs")}/datadog`; + const container = taskDefinition.addContainer("datadog-container", { + environment: { DD_TAGS: `idea_cluster:${this.clusterName}` }, + image: ecs.ContainerImage.fromRegistry(this.datadogImage()), + logging: this.adoptedLogDriver("datadog-logs", datadogLogGroupName, STREAM_PREFIX_DATADOG), + memoryReservationMiB: 512, + secrets: { DD_API_KEY: ecs.Secret.fromSecretsManager(apiKey) }, + }); + this.bindContainerToLogGroup(container, datadogLogGroupName); + const mounts: Array<{ name: string; path: string; readOnly: boolean }> = [ + { name: "docker-socket", path: "/var/run/docker.sock", readOnly: false }, + { name: "proc", path: "/proc", readOnly: true }, + { name: "cgroup", path: "/sys/fs/cgroup", readOnly: true }, + { name: "datadog", path: "/var/run/datadog", readOnly: false }, + ]; + for (const mount of mounts) { + taskDefinition.addVolume({ host: { sourcePath: mount.path }, name: mount.name }); + container.addMountPoints({ + containerPath: mount.path, + readOnly: mount.readOnly, + sourceVolume: mount.name, + }); + } + new ecs.Ec2Service(this.stack, "datadog-service", { + capacityProviderStrategies: [{ capacityProvider: this.capacityProvider.capacityProviderName, weight: 1 }], + cluster: this.ecsCluster, + daemon: true, + taskDefinition, + }); + } + + /** Publishes the ECS service and target-group identities for the cutover stacks. */ + private buildClusterSettings(): void { + const settings: Record = { + deployment_id: this.deploymentId, + image: this.requiredString("ecs.image"), + cluster_arn: this.ecsCluster.clusterArn, + cluster_name: this.ecsCluster.clusterName, + capacity_provider: this.capacityProvider.capacityProviderName, + namespace_id: this.namespace.namespaceId, + "dcv-gateway.certificate.certificate_secret_arn": this.gatewayCertificateSecretArn, + "dcv-gateway.certificate.private_key_secret_arn": this.gatewayPrivateKeySecretArn, + }; + for (const role of CONTAINER_ROLES) { + const resources = this.roleResources[role]; + if (resources === undefined) throw new Error(`ECS service for ${role} was not built`); + settings[`${role}.service_arn`] = resources.service.serviceArn; + settings[`${role}.target_group_arns`] = resources.targetGroups.map((targetGroup) => targetGroup.targetGroupArn); + } + this.updateClusterSettings(settings); + } +} + +export function buildStack(props: StackBuildProps): void { + new EcsStack(props); +} diff --git a/source/idea/ideactl/src/cdk/stacks/identity-provider.ts b/source/idea/ideactl/src/cdk/stacks/identity-provider.ts new file mode 100644 index 00000000..c7b40d97 --- /dev/null +++ b/source/idea/ideactl/src/cdk/stacks/identity-provider.ts @@ -0,0 +1,218 @@ +/** + * `identity-provider.provider` is the only branch: `cognito-idp` builds everything below, + * `keycloak` raises "not supported (yet)", anything else raises. The stack is twelve + * resources: two lambda role/policy/function trios, the user pool with its two cluster groups, + * its hosted domain, the pre-token-generation permission CDK adds for the trigger, and the + * `Custom::ClusterSettings` row every module stack ends with. + * + * The invitation email subject and body are read back rather than generated: `buildStack` calls + * `cognito-idp:DescribeUserPool` whenever `identity-provider.cognito.user_pool_id` is set, so an + * admin's edits in the console survive the next deploy. + * + * The Cognito domain prefix comes from `identity-provider.cognito.domain_url`, and with that key + * empty the shared `UserPool` construct mints `-`, replacing the deployed + * `AWS::Cognito::UserPoolDomain`. The `Custom::ClusterSettings` row writes both keys together. + */ + +import { RemovalPolicy } from 'aws-cdk-lib'; + +import type { StackBuildProps } from '../app.ts'; +import { IdeaBaseStack } from '../base-stack.ts'; +import { IdeaCodeAsset } from '../code-asset.ts'; +import { isEmpty } from '../../config/cluster-config.ts'; +import { LambdaFunction, Policy, Role } from '../constructs/common.ts'; +import { UserPool } from '../constructs/directory-service.ts'; +import { ExistingSocaCluster } from '../constructs/existing-resources.ts'; +import type { UserPool as UserPoolDescription } from '../synth-reads.ts'; + +/** `constants.IDENTITY_PROVIDER_COGNITO_IDP` / `..._KEYCLOAK`. */ +export const IDENTITY_PROVIDER_COGNITO_IDP = 'cognito-idp'; +export const IDENTITY_PROVIDER_KEYCLOAK = 'keycloak'; + +/** `app_constants.LOG_RETENTION_ROLE_NAME`. */ +const LOG_RETENTION_ROLE_NAME = 'log-retention'; + +/** `constants.CAVEATS['COGNITO_REQUIRE_FIPS_ENDPOINT_REGION_LIST']`. */ +export const COGNITO_REQUIRE_FIPS_ENDPOINT_REGION_LIST = ['us-gov-east-1', 'us-gov-west-1']; + +/** `RemovalPolicy` lookup uses member names, not enum values. */ +function removalPolicyByName(name: string): RemovalPolicy { + if (!Object.prototype.hasOwnProperty.call(RemovalPolicy, name)) { + throw new Error(`'${name}' is not a valid RemovalPolicy`); + } + return RemovalPolicy[name as keyof typeof RemovalPolicy]; +} + +/** The invitation email for a cluster that has no user pool yet, `os.linesep`-joined. */ +export function generatedInvitationEmailBody(clusterName: string, externalEndpoint: string): string { + return [ + '

Hello {username},

', + `

You have been invited to join the ${clusterName} cluster.

`, + '

Your temporary password is:

', + '

{####}

', + '

You can sign in to your account using the link below:
', + `${externalEndpoint}

`, + '

---
', + 'IDEA Cluster Admin

', + ].join('\n'); +} + +export class IdentityProviderStack extends IdeaBaseStack { + readonly cluster: ExistingSocaCluster; + idTokenClaimLambda: LambdaFunction | undefined; + userPool: UserPool | undefined; + oauthCredentialsLambda: LambdaFunction | undefined; + /** `cognito-idp:DescribeUserPool` of the deployed pool, read by `buildStack`. */ + private readonly describedUserPool: UserPoolDescription | undefined; + + constructor(props: StackBuildProps, describedUserPool?: UserPoolDescription) { + super({ + scope: props.app, + ctx: props.ctx, + moduleName: props.moduleName, + deploymentId: props.deploymentId, + terminationProtection: props.terminationProtection, + env: props.env, + }); + + this.describedUserPool = describedUserPool; + this.cluster = new ExistingSocaCluster(this.context, this.stack); + + const provider = this.context.config.getString('identity-provider.provider', undefined, { + required: true, + }) as string; + if (provider === IDENTITY_PROVIDER_KEYCLOAK) { + throw new Error(`identity provider: ${provider} not supported (yet).`); + } + if (provider !== IDENTITY_PROVIDER_COGNITO_IDP) { + throw new Error(`identity provider: ${provider} not supported`); + } + + this.buildCognitoIdp(); + this.buildCognitoClusterSettings(); + } + + /** The invitation email is generated for a new pool and read from an existing pool. */ + userInvitation(): { emailSubject: string; emailBody: string } { + const config = this.context.config; + // Not the custom dns name: it may not be configured yet when the cluster is first created. + const externalAlbDns = config.getString( + 'cluster.load_balancers.external_alb.load_balancer_dns_name', + undefined, + { required: true }, + ) as string; + const userPoolId = config.getString('identity-provider.cognito.user_pool_id'); + + if (isEmpty(userPoolId)) { + return { + emailSubject: `Invitation to Join IDEA Cluster: ${this.clusterName}`, + emailBody: generatedInvitationEmailBody(this.clusterName, `https://${externalAlbDns}`), + }; + } + + if (this.describedUserPool === undefined) { + throw new Error( + `identity-provider: cognito-idp:DescribeUserPool for user pool ${userPoolId as string} is required at ` + + 'synth time and was not read. Build this stack through buildStack(): synthesizing without the read ' + + "would rewrite the user pool's invitation email.", + ); + } + const template = this.describedUserPool.AdminCreateUserConfig?.InviteMessageTemplate; + if (template?.EmailSubject === undefined || template.EmailMessage === undefined) { + throw new Error( + `identity-provider: user pool ${userPoolId as string} has no AdminCreateUserConfig.InviteMessageTemplate`, + ); + } + return { emailSubject: template.EmailSubject, emailBody: template.EmailMessage }; + } + + buildCognitoIdp(): void { + const config = this.context.config; + const removalPolicy = removalPolicyByName( + config.getString('identity-provider.cognito.removal_policy', undefined, { required: true }) as string, + ); + + const userInvitation = this.userInvitation(); + + // Adds the custom claims to the Cognito ID token when SSO is enabled. + const claimLambdaName = 'id-token-claim'; + const idTokenClaimLambdaRole = new Role(this.context, `${claimLambdaName}-role`, this.stack, { + description: `Role for id token claim Lambda function for Cluster: ${this.clusterName}`, + assumedBy: ['lambda', 'cognito-idp'], + }); + idTokenClaimLambdaRole.attachInlinePolicy( + new Policy(this.context, `${claimLambdaName}-policy`, this.stack, { + policyTemplateName: 'custom_resource_sso_claim_modifier.yml', + }), + ); + this.idTokenClaimLambda = new LambdaFunction(this.context, claimLambdaName, this.stack, { + ideaCodeAsset: new IdeaCodeAsset('idea_custom_resource_sso_claim_modifier'), + description: 'Modify Cognito ID Token for SSO Enabled Clusters', + timeoutSeconds: 180, + role: idTokenClaimLambdaRole, + logRetentionRole: this.cluster.getRole(LOG_RETENTION_ROLE_NAME), + }); + this.addNagSuppression( + [{ rule_id: 'AwsSolutions-L1', reason: 'Python Runtime is selected for stability.' }], + this.idTokenClaimLambda, + ); + this.idTokenClaimLambda.node.addDependency(idTokenClaimLambdaRole); + + this.userPool = new UserPool(this.context, `${this.clusterName}-user-pool`, this.stack, { + removalPolicy, + userInvitation, + lambdaTriggers: { preTokenGeneration: this.idTokenClaimLambda }, + }); + + // One lambda for the whole cluster: every module stack invokes it as a custom resource to + // fetch its own OAuth2 client id and secret, through the arn in the cluster settings below. + const lambdaName = 'oauth-credentials'; + const oauthCredentialsLambdaRole = new Role(this.context, `${lambdaName}-role`, this.stack, { + description: `Role for auth credentials Lambda function for Cluster: ${this.clusterName}`, + assumedBy: ['lambda'], + }); + oauthCredentialsLambdaRole.attachInlinePolicy( + new Policy(this.context, `${lambdaName}-policy`, this.stack, { + policyTemplateName: 'custom-resource-get-user-pool-client-secret.yml', + }), + ); + this.oauthCredentialsLambda = new LambdaFunction(this.context, lambdaName, this.stack, { + ideaCodeAsset: new IdeaCodeAsset('idea_custom_resource_get_user_pool_client_secret'), + description: 'Get OAuth Credentials for a ClientId in UserPool', + timeoutSeconds: 180, + role: oauthCredentialsLambdaRole, + logRetentionRole: this.cluster.getRole(LOG_RETENTION_ROLE_NAME), + }); + this.addNagSuppression( + [{ rule_id: 'AwsSolutions-L1', reason: 'Python Runtime is selected for stability.' }], + this.oauthCredentialsLambda, + ); + this.oauthCredentialsLambda.node.addDependency(oauthCredentialsLambdaRole); + } + + buildCognitoClusterSettings(): void { + const userPool = this.userPool as UserPool; + this.updateClusterSettings({ + deployment_id: this.deploymentId, + 'cognito.user_pool_id': userPool.userPool.userPoolId, + 'cognito.provider_url': userPool.userPool.userPoolProviderUrl, + 'cognito.domain_url': userPool.domain.baseUrl({ + fips: COGNITO_REQUIRE_FIPS_ENDPOINT_REGION_LIST.includes(this.awsRegion), + }), + 'cognito.oauth_credentials_lambda_arn': (this.oauthCredentialsLambda as LambdaFunction).functionArn, + }); + } +} + +/** + * Reads the deployed user pool before the tree is built. `SynthReads` is asynchronous and a stack + * constructor is not, so the one synth-time read this stack needs happens here. A cluster with no + * pool yet skips it and the invitation email is generated instead. + */ +export async function buildStack(props: StackBuildProps): Promise { + const userPoolId = props.ctx.config.getString('identity-provider.cognito.user_pool_id'); + const describedUserPool = isEmpty(userPoolId) + ? undefined + : await props.ctx.synthReads.describeUserPool(userPoolId as string); + new IdentityProviderStack(props, describedUserPool); +} diff --git a/source/idea/ideactl/src/cdk/stacks/metrics.ts b/source/idea/ideactl/src/cdk/stacks/metrics.ts new file mode 100644 index 00000000..319f16bb --- /dev/null +++ b/source/idea/ideactl/src/cdk/stacks/metrics.ts @@ -0,0 +1,104 @@ +/** + * Three of the four providers build nothing: `prometheus` only asserts two config keys exist, + * `dogstatsd` does not even do that (the agent ships with the modules), and an unknown provider + * raises at synth. Only `cloudwatch` and `amazon_managed_prometheus` emit a resource, and the + * CloudWatch dashboard is deliberately empty, so `DashboardBody` is `{"widgets":[]}`. + */ + +import * as aps from 'aws-cdk-lib/aws-aps'; +import * as cloudwatch from 'aws-cdk-lib/aws-cloudwatch'; + +import type { StackBuildProps } from '../app.ts'; +import { IdeaBaseStack } from '../base-stack.ts'; +import { ExistingSocaCluster } from '../constructs/existing-resources.ts'; + +export const METRICS_PROVIDER_CLOUDWATCH = 'cloudwatch'; +export const METRICS_PROVIDER_AMAZON_MANAGED_PROMETHEUS = 'amazon_managed_prometheus'; +export const METRICS_PROVIDER_PROMETHEUS = 'prometheus'; +export const METRICS_PROVIDER_DOGSTATSD = 'dogstatsd'; + +export class MetricsStack extends IdeaBaseStack { + readonly cluster: ExistingSocaCluster; + cloudwatchDashboard: cloudwatch.Dashboard | undefined; + amazonPrometheusWorkspace: aps.CfnWorkspace | undefined; + + constructor(props: StackBuildProps) { + super({ + scope: props.app, + ctx: props.ctx, + moduleName: props.moduleName, + deploymentId: props.deploymentId, + terminationProtection: props.terminationProtection, + env: props.env, + }); + + this.cluster = new ExistingSocaCluster(this.context, this.stack); + + const provider = this.metricsProvider(); + if (provider === METRICS_PROVIDER_CLOUDWATCH) { + this.buildCloudWatch(); + } else if (provider === METRICS_PROVIDER_AMAZON_MANAGED_PROMETHEUS) { + this.buildAmazonManagedPrometheus(); + } else if (provider === METRICS_PROVIDER_PROMETHEUS) { + this.buildPrometheus(); + } else if (provider === METRICS_PROVIDER_DOGSTATSD) { + // The metrics agent is deployed with the modules, not by this stack. + } else { + throw new Error(`metrics provider: ${provider} not supported`); + } + + this.buildClusterSettings(); + } + + metricsProvider(): string { + return this.context.config.getString('metrics.provider', undefined, { required: true }) as string; + } + + buildCloudWatch(): void { + const dashboardName = this.context.config.getString('metrics.cloudwatch.dashboard_name', undefined, { + required: true, + }) as string; + // The dashboard body is empty. + this.cloudwatchDashboard = new cloudwatch.Dashboard(this.stack, 'cloudwatch-dashboard', { + dashboardName, + }); + } + + buildAmazonManagedPrometheus(): void { + const workspaceName = this.context.config.getString( + 'metrics.amazon_managed_prometheus.workspace_name', + undefined, + { required: true }, + ) as string; + this.amazonPrometheusWorkspace = new aps.CfnWorkspace(this.stack, 'prometheus-workspace', { + alias: workspaceName, + }); + this.addCommonTags(this.amazonPrometheusWorkspace); + } + + /** Validate and do nothing: `prometheus` provisions no resources. */ + buildPrometheus(): void { + this.context.config.getString('metrics.prometheus.remote_write.url', undefined, { required: true }); + this.context.config.getString('metrics.prometheus.query.url', undefined, { required: true }); + } + + buildClusterSettings(): void { + const clusterSettings: Record = { deployment_id: this.deploymentId }; + const provider = this.metricsProvider(); + if (provider === METRICS_PROVIDER_CLOUDWATCH) { + clusterSettings['cloudwatch.dashboard_arn'] = (this.cloudwatchDashboard as cloudwatch.Dashboard) + .dashboardArn; + } else if (provider === METRICS_PROVIDER_AMAZON_MANAGED_PROMETHEUS) { + const workspace = this.amazonPrometheusWorkspace as aps.CfnWorkspace; + clusterSettings['amazon_managed_prometheus.workspace_id'] = workspace.attrWorkspaceId; + clusterSettings['amazon_managed_prometheus.workspace_arn'] = workspace.attrArn; + clusterSettings['prometheus.remote_write.url'] = `${workspace.attrPrometheusEndpoint}api/v1/remote_write`; + clusterSettings['prometheus.remote_read.url'] = `${workspace.attrPrometheusEndpoint}api/v1/query`; + } + this.updateClusterSettings(clusterSettings); + } +} + +export function buildStack(props: StackBuildProps): void { + new MetricsStack(props); +} diff --git a/source/idea/ideactl/src/cdk/stacks/scheduler.ts b/source/idea/ideactl/src/cdk/stacks/scheduler.ts new file mode 100644 index 00000000..f62bcf32 --- /dev/null +++ b/source/idea/ideactl/src/cdk/stacks/scheduler.ts @@ -0,0 +1,649 @@ +/** + * Everything the stack references from another module is a literal read out of the cluster + * config at synth time: there are no exports, no `Fn::ImportValue` and no outputs. The three + * IAM policies take CDK's default `PolicyName`, which is the logical id, so the construct ids + * here are load bearing twice over. + */ + +import { CustomResource, Duration, Fn, RemovalPolicy, Tags } from 'aws-cdk-lib'; +import * as cognito from 'aws-cdk-lib/aws-cognito'; +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as elbv2 from 'aws-cdk-lib/aws-elasticloadbalancingv2'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as route53 from 'aws-cdk-lib/aws-route53'; +import * as sqs from 'aws-cdk-lib/aws-sqs'; + +import { ArnBuilder } from '../../config/arn-builder.ts'; +import type { ClusterConfig } from '../../config/cluster-config.ts'; +import type { StackBuildProps } from '../app.ts'; +import { IdeaBaseStack } from '../base-stack.ts'; +import { IDEA_TAG_CLUSTER_NAME, IDEA_TAG_MODULE_ID, IDEA_TAG_MODULE_NAME, IDEA_TAG_NODE_TYPE } from '../constructs/base.ts'; +import { InstanceProfile, Policy, Role, SQSQueue } from '../constructs/common.ts'; +import { OAuthClientIdAndSecret } from '../constructs/directory-service.ts'; +import { + ExistingSocaCluster, + lookupClusterDns, + lookupEbsKmsKey, + lookupKeyPair, +} from '../constructs/existing-resources.ts'; +import { ComputeNodeSecurityGroup, SchedulerSecurityGroup } from '../constructs/network.ts'; +import { buildBootstrapUserData } from '../userdata.ts'; + +const MODULE_SCHEDULER = 'scheduler'; +const MODULE_CLUSTER_MANAGER = 'cluster-manager'; +const NODE_TYPE_APP = 'app'; +const OS_AMAZONLINUX2 = 'amazonlinux2'; +const OS_AMAZONLINUX2023 = 'amazonlinux2023'; +/** `constants.SQS_MAX_RECEIVE_COUNT_SCHEDULER_JOB_STATUS`. */ +const SQS_MAX_RECEIVE_COUNT_SCHEDULER_JOB_STATUS = 10; + +/** + * `get_int(key, required=True)` and `get_list(key, required=True)`. Unlike `getString`, these two + * getters have no overload for an undefined default, so the call is typed here instead. + */ +type RequiredGet = (key: string, defaultValue?: T, options?: { required?: boolean }) => T; + +function requiredInt(config: ClusterConfig, key: string): number { + return (config.getInt as RequiredGet)(key, undefined, { required: true }); +} + +function requiredList(config: ClusterConfig, key: string): string[] { + return (config.getList as RequiredGet)(key, undefined, { required: true }); +} + +/** `Utils.get_ec2_block_device_name`. */ +export function ec2BlockDeviceName(baseOs: string): string { + return baseOs === OS_AMAZONLINUX2 || baseOs === OS_AMAZONLINUX2023 ? '/dev/xvda' : '/dev/sda1'; +} + +export class SchedulerStack extends IdeaBaseStack { + readonly cluster: ExistingSocaCluster; + readonly arnBuilder: ArnBuilder; + readonly bootstrapPackageUri: string; + readonly userPool: cognito.IUserPool; + private readonly ecsEnabled: boolean; + private readonly hostsPresent: boolean; + + oauth2ClientSecret!: OAuthClientIdAndSecret; + schedulerRole!: Role; + schedulerInstanceProfile!: InstanceProfile; + computeNodeRole!: Role; + computeNodeInstanceProfile!: InstanceProfile; + spotFleetRequestRole!: Role; + schedulerSecurityGroup!: SchedulerSecurityGroup; + computeNodeSecurityGroup!: ComputeNodeSecurityGroup; + jobStatusSqsQueue!: SQSQueue; + ec2Instance!: ec2.CfnInstance; + clusterDnsRecordSet!: route53.RecordSet; + externalEndpoint!: CustomResource; + internalEndpoint!: CustomResource; + + constructor(props: StackBuildProps) { + super({ + scope: props.app, + ctx: props.ctx, + moduleName: props.moduleName, + deploymentId: props.deploymentId, + terminationProtection: props.terminationProtection, + env: props.env, + }); + + this.bootstrapPackageUri = this.lookupBootstrapPackageUri(); + this.cluster = new ExistingSocaCluster(this.context, this.stack); + this.arnBuilder = new ArnBuilder(this.context.config); + this.ecsEnabled = this.context.config.getBool("ecs.enabled", false); + // `ecs.enabled` alone routes the endpoints to the container service and deletes the instance + // that serves them in one change set, so nothing proves the new target before the old one is + // gone. With `ecs.retain_existing_hosts` the same deploy routes the endpoints and keeps the + // instance, idle and unregistered, so a later deploy removes it once the container is serving + // and turning the flag off puts the instance back in service. + this.hostsPresent = + !this.ecsEnabled || this.context.config.getBool("ecs.retain_existing_hosts", false); + + this.userPool = this.lookupUserPool(); + + this.buildOauth2Client(); + this.buildAccessControlGroups(this.userPool); + this.buildSqsQueue(); + this.buildIamRoles(); + this.buildSecurityGroups(); + if (this.hostsPresent) this.buildEc2Instance(); + // The container scheduler upserts this record itself, so the stack stops managing it as soon as + // routing moves. An earlier retain-only deploy is what keeps the name alive across that handover. + if (!this.ecsEnabled) this.buildRoute53RecordSet(); + this.buildEndpoints(); + this.buildClusterSettings(); + } + + /** + * The bootstrap package is not a CDK asset: the CLI uploads it and passes its location as the + * `bootstrap_package_uri` context parameter. When absent, the deployment identifier derives + * the package name for a standalone synth. + * + * The naming rule is local so stack synthesis does not load an S3 client. + */ + private lookupBootstrapPackageUri(): string { + const fromContext: unknown = this.stack.node.tryGetContext('bootstrap_package_uri'); + if (typeof fromContext === 'string' && fromContext !== '') return fromContext; + const bucket = this.context.config.getString('cluster.cluster_s3_bucket', undefined, { + required: true, + }) as string; + return `s3://${bucket}/idea/bootstrap/bootstrap-${this.moduleId}-${this.deploymentId}.tar.gz`; + } + + buildOauth2Client(): void { + const resourceServer = this.userPool.addResourceServer('resource-server', { + identifier: this.moduleId, + scopes: [ + new cognito.ResourceServerScope({ scopeName: 'read', scopeDescription: 'Allow Read Access' }), + new cognito.ResourceServerScope({ scopeName: 'write', scopeDescription: 'Allow Write Access' }), + ], + }); + + const client = this.userPool.addClient(`${this.moduleId}-client`, { + accessTokenValidity: Duration.hours(1), + generateSecret: true, + idTokenValidity: Duration.hours(1), + oAuth: { + flows: { clientCredentials: true }, + scopes: [ + cognito.OAuthScope.custom(`${this.moduleId}/read`), + cognito.OAuthScope.custom(`${this.moduleId}/write`), + cognito.OAuthScope.custom(`${this.context.config.moduleId(MODULE_CLUSTER_MANAGER)}/read`), + ], + }, + refreshTokenValidity: Duration.days(30), + userPoolClientName: this.moduleId, + }); + client.node.addDependency(resourceServer); + + const oauthCredentialsLambdaArn = this.context.config.getString( + 'identity-provider.cognito.oauth_credentials_lambda_arn', + undefined, + { required: true }, + ) as string; + const clientSecret = new CustomResource(this.stack, `${this.moduleId}-creds`, { + serviceToken: oauthCredentialsLambdaArn, + properties: { + UserPoolId: this.userPool.userPoolId, + ClientId: client.userPoolClientId, + }, + resourceType: 'Custom::GetOAuthCredentials', + }); + + this.oauth2ClientSecret = new OAuthClientIdAndSecret( + this.context, + this.moduleId, + MODULE_SCHEDULER, + this.stack, + client.userPoolClientId, + clientSecret.getAttString('ClientSecret'), + ); + } + + buildIamRoles(): void { + const ec2ManagedPolicies = this.getEc2InstanceManagedPolicies(); + + // Deduplication preserves first-seen order. + // These two lists are deduplicated in first-seen order. The reference implementation + // deduplicated through an unordered set of strings, so the array order it emitted is a per + // process permutation: no seed reproduces all three captured templates, and the two real + // clusters carry the opposite order to the development one. IAM attaches a set, so a + // permutation changes no permission and replaces nothing, but it is an ordered array in the + // template and a comparison that reads order sees it. This is the only place the reference did + // that, which is why only these two roles differ. + const schedulerPolicyArns = [ + ...new Set([ + ...this.context.config.getList('cluster.iam.scheduler_iam_policy_arns', []), + ...ec2ManagedPolicies, + ]), + ]; + this.schedulerRole = new Role(this.context, `${this.moduleId}-role`, this.stack, { + description: 'IAM role assigned to the scheduler', + assumedBy: ['ssm', 'ec2'], + managedPolicies: schedulerPolicyArns, + }); + if (this.hostsPresent) { + this.schedulerInstanceProfile = new InstanceProfile( + this.context, + `${this.moduleId}-scheduler-instance-profile`, + this.stack, + [this.schedulerRole], + ); + } + + const computeNodePolicyArns = [ + ...new Set([ + ...this.context.config.getList('cluster.iam.compute_node_iam_policy_arns', []), + ...ec2ManagedPolicies, + ]), + ]; + this.computeNodeRole = new Role(this.context, `${this.moduleId}-compute-node-role`, this.stack, { + description: 'IAM role assigned to the compute nodes', + assumedBy: ['ssm', 'ec2'], + managedPolicies: computeNodePolicyArns, + }); + this.computeNodeInstanceProfile = new InstanceProfile( + this.context, + `${this.moduleId}-compute-node-instance-profile`, + this.stack, + [this.computeNodeRole], + ); + + this.spotFleetRequestRole = new Role(this.context, `${this.moduleId}-spot-fleet-request-role`, this.stack, { + description: 'IAM role to manage SpotFleet requests', + assumedBy: ['spotfleet'], + }); + + const vars = { + scheduler_role_arn: this.schedulerRole.roleArn, + compute_node_role_arn: this.computeNodeRole.roleArn, + spot_fleet_request_role_arn: this.spotFleetRequestRole.roleArn, + }; + + this.schedulerRole.attachInlinePolicy( + new Policy(this.context, 'scheduler-policy', this.stack, { + policyTemplateName: 'scheduler.yml', + vars, + moduleId: this.moduleId, + }), + ); + this.computeNodeRole.attachInlinePolicy( + new Policy(this.context, 'compute-node-policy', this.stack, { + policyTemplateName: 'compute-node.yml', + vars, + moduleId: this.moduleId, + }), + ); + this.spotFleetRequestRole.attachInlinePolicy( + new Policy(this.context, 'spot-fleet-policy', this.stack, { + policyTemplateName: 'spot-fleet-request.yml', + vars, + moduleId: this.moduleId, + }), + ); + } + + /** + * Compute nodes run under project roles only when the cluster integration and the scheduler + * opt-in are both on. Read at synth time, so the grants that depend on it exist only in a + * deployment that asked for them. + */ + isBedrockEnabledForJobs(): boolean { + const clusterManagerModuleId = this.context.config.moduleId(MODULE_CLUSTER_MANAGER); + return ( + this.context.config.getBool(`${clusterManagerModuleId}.bedrock.enabled`, false) && + this.context.config.getBool(`${this.moduleId}.bedrock.enabled`, false) + ); + } + + buildSqsQueue(): void { + const kmsKeyId = this.context.config.getString('cluster.sqs.kms_key_id'); + + // The dead-letter queue is created before the queue that references it. + const deadLetterQueue = new SQSQueue(this.context, 'job-status-events-dlq', this.stack, { + queueName: `${this.clusterName}-${this.moduleId}-job-status-events-dlq`, + encryptionMasterKey: kmsKeyId, + isDeadLetterQueue: true, + }); + this.jobStatusSqsQueue = new SQSQueue(this.context, 'job-status-events', this.stack, { + queueName: `${this.clusterName}-${this.moduleId}-job-status-events`, + encryptionMasterKey: kmsKeyId, + deadLetterQueue: { + maxReceiveCount: SQS_MAX_RECEIVE_COUNT_SCHEDULER_JOB_STATUS, + queue: deadLetterQueue, + } satisfies sqs.DeadLetterQueue, + }); + // Re-tags `Name` from the queue id to the module id: the last write wins. + this.addCommonTags(this.jobStatusSqsQueue); + this.addCommonTags(deadLetterQueue); + + if (this.isBedrockEnabledForJobs()) { + // Bedrock project-role compute nodes carry the dcv host policy, not the compute-node + // policy, so the execution hooks' job-status send needs its own grant, scoped to + // per-project IAM roles. + this.jobStatusSqsQueue.addToResourcePolicy( + new iam.PolicyStatement({ + sid: 'ProjectRoleJobStatusEvents', + effect: iam.Effect.ALLOW, + actions: ['sqs:SendMessage'], + resources: [this.jobStatusSqsQueue.queueArn], + principals: [new iam.AnyPrincipal()], + conditions: { ArnLike: { 'aws:PrincipalArn': this.arnBuilder.getProjectRoleArn() } }, + }), + ); + } + } + + buildSecurityGroups(): void { + this.schedulerSecurityGroup = new SchedulerSecurityGroup( + this.context, + `${this.moduleId}-security-group`, + this.stack, + this.cluster.vpc, + this.cluster.getSecurityGroup('bastion-host') as ec2.ISecurityGroup, + this.cluster.getSecurityGroup('external-load-balancer') as ec2.ISecurityGroup, + ); + // The rule exists to reach a host. It goes when the last host does, not when routing moves. + if (!this.hostsPresent) this.removeBastionHostIngressRule(this.schedulerSecurityGroup); + + this.computeNodeSecurityGroup = new ComputeNodeSecurityGroup( + this.context, + `${this.moduleId}-compute-node-security-group`, + this.stack, + this.cluster.vpc, + ); + } + + private removeBastionHostIngressRule(securityGroup: ec2.SecurityGroup): void { + const ingressRule = securityGroup.node.children.find( + (child) => + child instanceof ec2.CfnSecurityGroupIngress && + child.description === "Allow SSH from Bastion Host", + ); + if (ingressRule === undefined) { + throw new Error("Scheduler security group has no bastion SSH ingress rule"); + } + securityGroup.node.tryRemoveChild(ingressRule.node.id); + } + + private ecsTargetGroupArn(index: number): string { + const targetGroupArns = this.context.config.getList( + "ecs.scheduler.target_group_arns", + [], + { required: true }, + ); + const targetGroupArn = targetGroupArns[index]; + if (targetGroupArn === undefined || targetGroupArn === "") { + throw new Error(`ecs.scheduler.target_group_arns[${index}] is required when ecs.enabled is true`); + } + return targetGroupArn; + } + + buildEc2Instance(): void { + const config = this.context.config; + const isPublic = config.getBool('scheduler.public', false); + const baseOs = config.getString('scheduler.base_os', undefined, { required: true }) as string; + const instanceAmi = config.getString('scheduler.instance_ami', undefined, { required: true }) as string; + const instanceType = config.getString('scheduler.instance_type', undefined, { required: true }) as string; + const volumeSize = config.getInt('scheduler.volume_size', 200); + const keyPair = lookupKeyPair(this.context, this.stack); + const enableDetailedMonitoring = config.getBool('scheduler.ec2.enable_detailed_monitoring', false); + const enableTerminationProtection = config.getBool('scheduler.ec2.enable_termination_protection', false); + const metadataHttpTokens = config.getString('scheduler.ec2.metadata_http_tokens', undefined, { + required: true, + }) as string; + const httpsProxy = config.getString('cluster.network.https_proxy', ''); + const noProxy = config.getString('cluster.network.no_proxy', ''); + const proxyConfig: Record = + httpsProxy === '' ? {} : { http_proxy: httpsProxy, https_proxy: httpsProxy, no_proxy: noProxy }; + const ebsKmsKey = lookupEbsKmsKey(this.context, this.stack); + + const usePublicSubnets = isPublic && this.cluster.publicSubnets.length > 0; + const subnetIds = usePublicSubnets + ? this.cluster.existingVpc.getPublicSubnetIds() + : this.cluster.existingVpc.getPrivateSubnetIds(); + // Python indexes the list and raises IndexError on an empty one. Without this the template + // synthesizes with no SubnetId and the deploy fails halfway, after the roles and the queues. + const subnetId = subnetIds[0]; + if (subnetId === undefined) { + throw new Error( + `cluster.network.${usePublicSubnets ? 'public' : 'private'}_subnets is empty: no subnet to launch the scheduler into`, + ); + } + + const blockDeviceName = ec2BlockDeviceName(baseOs); + const blockDeviceTypeString = config.getString('scheduler.volume_type', 'gp3'); + const blockDeviceVolumeType = + blockDeviceTypeString === 'gp3' ? ec2.EbsDeviceVolumeType.GP3 : ec2.EbsDeviceVolumeType.GP2; + + const userData = buildBootstrapUserData({ + awsRegion: this.awsRegion, + bootstrapPackageUri: this.bootstrapPackageUri, + installCommands: ['/bin/bash scheduler/setup.sh'], + proxyConfig, + baseOs, + }); + + const launchTemplate = new ec2.LaunchTemplate(this.stack, `${this.moduleId}-lt`, { + instanceType: new ec2.InstanceType(instanceType), + machineImage: ec2.MachineImage.genericLinux({ [this.awsRegion]: instanceAmi }), + userData: ec2.UserData.custom(Fn.sub(userData)), + keyPair, + blockDevices: [ + { + deviceName: blockDeviceName, + volume: ec2.BlockDeviceVolume.ebs(volumeSize, { + encrypted: true, + kmsKey: ebsKmsKey, + volumeType: blockDeviceVolumeType, + }), + }, + ], + requireImdsv2: metadataHttpTokens === 'required', + }); + + this.ec2Instance = new ec2.CfnInstance(this.stack, `${this.moduleId}-instance`, { + blockDeviceMappings: [ + { + deviceName: blockDeviceName, + ebs: { volumeSize, volumeType: blockDeviceTypeString }, + }, + ], + disableApiTermination: enableTerminationProtection, + // The profile name, not a Ref: there is no dependency edge to the instance profile. + iamInstanceProfile: this.schedulerInstanceProfile.instanceProfileName as string, + instanceType, + imageId: instanceAmi, + keyName: keyPair.keyPairName, + launchTemplate: { + version: launchTemplate.latestVersionNumber, + launchTemplateId: launchTemplate.launchTemplateId as string, + }, + networkInterfaces: [ + { + deviceIndex: '0', + associatePublicIpAddress: isPublic, + groupSet: [this.schedulerSecurityGroup.securityGroupId], + subnetId, + }, + ], + userData: Fn.base64(Fn.sub(userData)), + monitoring: enableDetailedMonitoring, + }); + Tags.of(this.ec2Instance).add('Name', this.buildResourceName(this.moduleId)); + Tags.of(this.ec2Instance).add(IDEA_TAG_NODE_TYPE, NODE_TYPE_APP); + this.addBackupTags(this.ec2Instance); + + this.addNagSuppression( + [{ rule_id: 'AwsSolutions-EC26', reason: 'EBS Encryption is enforced via Launch Template' }], + this.ec2Instance, + ); + + if (!enableDetailedMonitoring) { + this.addNagSuppression( + [ + { + rule_id: 'AwsSolutions-EC28', + reason: 'detailed monitoring is a configurable option to save costs', + }, + ], + this.ec2Instance, + ); + } + + if (!enableTerminationProtection) { + this.addNagSuppression( + [ + { + rule_id: 'AwsSolutions-EC29', + reason: + 'termination protection not supported in CDK L2 construct. enable termination protection via AWS EC2 console after deploying the cluster.', + }, + ], + this.ec2Instance, + ); + } + } + + buildRoute53RecordSet(): void { + const hostname = this.context.config.getString('scheduler.hostname', undefined, { + required: true, + }) as string; + this.clusterDnsRecordSet = new route53.RecordSet(this.stack, `${this.moduleId}-dns-record`, { + recordType: route53.RecordType.A, + target: route53.RecordTarget.fromIpAddresses(this.ec2Instance.attrPrivateIp), + ttl: Duration.minutes(5), + recordName: hostname, + zone: lookupClusterDns(this.context, this.stack), + }); + // The record is the name clients and execution hosts resolve. Retaining it lets a later + // deploy stop managing it, which is what the container path does, without CloudFormation + // deleting the name. Deploy this on its own before turning `ecs.enabled` on. + if (this.context.config.getBool('scheduler.retain_dns_record', false)) { + this.clusterDnsRecordSet.applyRemovalPolicy(RemovalPolicy.RETAIN); + } + } + + buildEndpoints(): void { + const config = this.context.config; + const externalTargetGroupArn = this.ecsEnabled + ? this.ecsTargetGroupArn(0) + : new elbv2.CfnTargetGroup( + this.stack, + `${this.moduleId}-external-target-group`, + { + port: 8443, + protocol: 'HTTPS', + targetType: 'ip', + vpcId: this.cluster.vpc.vpcId, + name: this.getTargetGroupName('sched-ext'), + targets: [{ id: this.ec2Instance.attrPrivateIp }], + healthCheckPath: '/healthcheck', + }, + ).ref; + const clusterEndpointsLambdaArn = config.getString('cluster.cluster_endpoints_lambda_arn', undefined, { + required: true, + }) as string; + const externalHttpsListenerArn = config.getString( + 'cluster.load_balancers.external_alb.https_listener_arn', + undefined, + { required: true }, + ) as string; + const externalEndpointPriority = requiredInt(config, 'scheduler.endpoints.external.priority'); + const externalEndpointPathPatterns = requiredList(config, 'scheduler.endpoints.external.path_patterns'); + + this.externalEndpoint = new CustomResource(this.stack, 'external-endpoint', { + serviceToken: clusterEndpointsLambdaArn, + properties: { + endpoint_name: `${this.moduleId}-external-endpoint`, + listener_arn: externalHttpsListenerArn, + priority: externalEndpointPriority, + target_group_arn: externalTargetGroupArn, + conditions: [{ Field: 'path-pattern', Values: externalEndpointPathPatterns }], + actions: [{ Type: 'forward', TargetGroupArn: externalTargetGroupArn }], + tags: { + [IDEA_TAG_CLUSTER_NAME]: this.clusterName, + [IDEA_TAG_MODULE_ID]: this.moduleId, + [IDEA_TAG_MODULE_NAME]: MODULE_SCHEDULER, + }, + }, + resourceType: 'Custom::SchedulerEndpointExternal', + }); + + const internalHttpsListenerArn = config.getString( + 'cluster.load_balancers.internal_alb.https_listener_arn', + undefined, + { required: true }, + ) as string; + const internalEndpointPriority = requiredInt(config, 'scheduler.endpoints.internal.priority'); + const internalEndpointPathPatterns = requiredList(config, 'scheduler.endpoints.internal.path_patterns'); + + const internalTargetGroupArn = this.ecsEnabled + ? this.ecsTargetGroupArn(1) + : new elbv2.CfnTargetGroup( + this.stack, + `${this.moduleId}-internal-target-group`, + { + port: 8443, + protocol: 'HTTPS', + targetType: 'ip', + vpcId: this.cluster.vpc.vpcId, + name: this.getTargetGroupName('sched-int'), + targets: [{ id: this.ec2Instance.attrPrivateIp }], + healthCheckPath: '/healthcheck', + }, + ).ref; + + this.internalEndpoint = new CustomResource(this.stack, 'internal-endpoint', { + serviceToken: clusterEndpointsLambdaArn, + properties: { + endpoint_name: `${this.moduleId}-internal-endpoint`, + listener_arn: internalHttpsListenerArn, + priority: internalEndpointPriority, + target_group_arn: internalTargetGroupArn, + conditions: [{ Field: 'path-pattern', Values: internalEndpointPathPatterns }], + actions: [{ Type: 'forward', TargetGroupArn: internalTargetGroupArn }], + tags: { + [IDEA_TAG_CLUSTER_NAME]: this.clusterName, + [IDEA_TAG_MODULE_ID]: this.moduleId, + [IDEA_TAG_MODULE_NAME]: MODULE_SCHEDULER, + }, + }, + resourceType: 'Custom::SchedulerEndpointInternal', + }); + } + + buildClusterSettings(): void { + // The scheduler and the execution hosts both derive the PBS server name from + // private_dns_name. Pointing it at the cluster DNS record rather than the instance lets a + // replaced scheduler keep its name, so execution hosts do not need reconfiguring and running + // jobs survive. Off by default: turning it on for an existing cluster renames its PBS server, + // and execution hosts already running jobs would not follow the change. + const useStableServerName = + this.ecsEnabled || this.context.config.getBool('scheduler.use_stable_server_name', false); + const privateDnsName = useStableServerName + ? (this.context.config.getString('scheduler.hostname', undefined, { required: true }) as string) + : this.ec2Instance.attrPrivateDnsName; + + const clusterSettings: Record = { + deployment_id: this.deploymentId, + private_dns_name: privateDnsName, + }; + + if (this.ecsEnabled) { + // The container path has no instance address to pin the name with, so the row that makes the + // scheduler and the execution hosts use the DNS name is written, not assumed. + clusterSettings['use_stable_server_name'] = true; + } else { + clusterSettings['private_ip'] = this.ec2Instance.attrPrivateIp; + if (this.context.config.getBool('scheduler.public', false)) { + clusterSettings['public_ip'] = this.ec2Instance.attrPublicIp; + } + clusterSettings['instance_id'] = this.ec2Instance.ref; + } + + clusterSettings['client_id'] = this.oauth2ClientSecret.clientId.ref; + clusterSettings['client_secret'] = this.oauth2ClientSecret.clientSecret.ref; + clusterSettings['security_group_id'] = this.schedulerSecurityGroup.securityGroupId; + clusterSettings['iam_role_arn'] = this.schedulerRole.roleArn; + clusterSettings['compute_node_security_group_ids'] = [this.computeNodeSecurityGroup.securityGroupId]; + clusterSettings['compute_node_iam_role_arn'] = this.computeNodeRole.roleArn; + clusterSettings['compute_node_instance_profile_arn'] = this.computeNodeInstanceProfile.ref; + clusterSettings['spot_fleet_request_iam_role_arn'] = this.spotFleetRequestRole.roleArn; + clusterSettings['job_status_sqs_queue_url'] = this.jobStatusSqsQueue.queueUrl; + + // iam:PassRole for project roles is granted only at deploy time when bedrock-for-jobs is + // enabled, not by toggling it at runtime; written under the same gate so the scheduler can + // detect a refusal. + if (this.isBedrockEnabledForJobs()) { + clusterSettings['bedrock.project_pass_role_arn'] = this.arnBuilder.getProjectRoleArn(); + } + + this.updateClusterSettings(clusterSettings); + } +} + +export function buildStack(props: StackBuildProps): void { + new SchedulerStack(props); +} diff --git a/source/idea/ideactl/src/cdk/stacks/shared-storage.ts b/source/idea/ideactl/src/cdk/stacks/shared-storage.ts new file mode 100644 index 00000000..5bd6d20b --- /dev/null +++ b/source/idea/ideactl/src/cdk/stacks/shared-storage.ts @@ -0,0 +1,224 @@ +/** + * One security group, then a pass over every top-level key under `shared-storage`. A key is a file + * system only when its value is a config subtree carrying both `provider` and `mount_dir`, which is + * what keeps `deployment_id` and `security_group_id` out of the loop. `use_existing_fs` registers an + * id and provisions nothing; `efs` and `fsx_lustre` provision; the other three providers + * (`fsx_cache`, `fsx_netapp_ontap`, `fsx_openzfs`, `fsx_windows_file_server`) are accepted by the + * validation and then provision nothing, so a cluster whose `data` volume is ONTAP synthesizes with + * no `data-storage-efs` at all. + * + * Two quirks the live templates depend on: + * + * - the EFS file systems carry `DeletionPolicy` and no `UpdateReplacePolicy`. Nothing here may + * call `applyRemovalPolicy`. + * - the `dns` guard reads `shared-storage..dns`, not the file system's key, so it never + * finds a value and the `dns` setting is always rewritten. Every node's bootstrap mounts /apps + * and /data from that key. + */ + +import type { StackBuildProps } from '../app.ts'; +import { IdeaBaseStack } from '../base-stack.ts'; +import { ConfigKeyNotFound, isEmpty, type ClusterConfig } from '../../config/cluster-config.ts'; +import { ExistingSocaCluster } from '../constructs/existing-resources.ts'; +import { SharedStorageSecurityGroup } from '../constructs/network.ts'; +import { AmazonEFS, FSxForLustre, valueAsDict } from '../constructs/storage.ts'; + +export const STORAGE_PROVIDER_EFS = 'efs'; +export const STORAGE_PROVIDER_FSX_CACHE = 'fsx_cache'; +export const STORAGE_PROVIDER_FSX_LUSTRE = 'fsx_lustre'; +export const STORAGE_PROVIDER_FSX_NETAPP_ONTAP = 'fsx_netapp_ontap'; +export const STORAGE_PROVIDER_FSX_OPENZFS = 'fsx_openzfs'; +export const STORAGE_PROVIDER_FSX_WINDOWS_FILE_SERVER = 'fsx_windows_file_server'; + +/** `constants.SUPPORTED_STORAGE_PROVIDERS`, in declaration order. */ +export const SUPPORTED_STORAGE_PROVIDERS: readonly string[] = [ + STORAGE_PROVIDER_EFS, + STORAGE_PROVIDER_FSX_CACHE, + STORAGE_PROVIDER_FSX_LUSTRE, + STORAGE_PROVIDER_FSX_NETAPP_ONTAP, + STORAGE_PROVIDER_FSX_OPENZFS, + STORAGE_PROVIDER_FSX_WINDOWS_FILE_SERVER, +]; + +/** Storage file-system properties. */ +export interface FileSystemHolder { + name: string; + provider: string; + fileSystemId: string; + /** Absent for a file system that was registered from `use_existing_fs` rather than provisioned. */ + fileSystem?: AmazonEFS | FSxForLustre; +} + +function isSubtree(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** + * `config.getConfig('shared-storage', required=true)`: the whole module subtree. + * + * `getRealKey` renders a single-segment key as `.`, which yields an empty segment. + * This helper skips empty segments and throws `ConfigKeyNotFound` for a missing required value. + */ +function moduleSubtree(config: ClusterConfig, moduleKey: string): Record { + const realKey = config.getRealKey(moduleKey); + const tree = (config as unknown as { tree: Record }).tree; + let node: unknown = tree; + for (const part of realKey.split('.').filter((segment) => segment !== '')) { + if (!isSubtree(node) || !(part in node)) { + throw new ConfigKeyNotFound(`'${part}', key: ${realKey}`); + } + node = node[part]; + } + return isSubtree(node) && !isEmpty(node) ? node : {}; +} + +/** `ConfigTree.get` with a dotted path, scoped to one storage config subtree. */ +function subtreeGet(subtree: Record, path: string): unknown { + let node: unknown = subtree; + for (const part of path.split('.')) { + if (!isSubtree(node)) return undefined; + node = node[part]; + } + return node; +} + +export class SharedStorageStack extends IdeaBaseStack { + readonly cluster: ExistingSocaCluster; + readonly fileSystems: FileSystemHolder[] = []; + securityGroup: SharedStorageSecurityGroup | undefined; + + constructor(props: StackBuildProps) { + super({ + scope: props.app, + ctx: props.ctx, + moduleName: props.moduleName, + deploymentId: props.deploymentId, + terminationProtection: props.terminationProtection, + env: props.env, + }); + + this.cluster = new ExistingSocaCluster(this.context, this.stack); + + // `apps` and `data` are mandatory: assert the two keys exist before anything is built. + this.context.config.getString('shared-storage.apps.provider', undefined, { required: true }); + this.context.config.getString('shared-storage.data.provider', undefined, { required: true }); + + this.buildSecurityGroup(); + this.buildSharedStorage(); + this.buildClusterSettings(); + } + + buildSecurityGroup(): void { + this.securityGroup = new SharedStorageSecurityGroup( + this.context, + 'shared-storage-security-group', + this.stack, + this.cluster.vpc, + ); + } + + buildSharedStorage(): void { + const storageConfigs = moduleSubtree(this.context.config, 'shared-storage'); + + for (const [name, value] of Object.entries(storageConfigs)) { + // Non-storage keys under `shared-storage` (deployment_id, security_group_id) are scalars. + if (!isSubtree(value)) continue; + + const provider = subtreeGet(value, 'provider'); + const mountDir = subtreeGet(value, 'mount_dir'); + if (isEmpty(provider) || isEmpty(mountDir)) continue; + + const providerName = provider as string; + if (!SUPPORTED_STORAGE_PROVIDERS.includes(providerName)) { + throw new Error(`file system provider: ${providerName} not supported`); + } + + if (subtreeGet(value, `${providerName}.use_existing_fs`)) { + const fileSystemId = subtreeGet(value, `${providerName}.file_system_id`); + if (isEmpty(fileSystemId)) { + throw new Error(`shared-storage.${name}.${providerName}.file_system_id is required`); + } + this.fileSystems.push({ name, provider: providerName, fileSystemId: fileSystemId as string }); + continue; + } + + if (providerName === STORAGE_PROVIDER_EFS) { + this.buildEfs(name, value); + } else if (providerName === STORAGE_PROVIDER_FSX_LUSTRE) { + this.buildFsxLustre(name, value); + } + } + } + + buildEfs(name: string, storageConfig: Record): void { + const efs = new AmazonEFS(this.context, `${name}-storage-efs`, this.stack, { + vpc: this.cluster.vpc, + efsConfig: valueAsDict('efs', storageConfig), + securityGroup: this.securityGroup as SharedStorageSecurityGroup, + subnets: this.cluster.privateSubnets, + }); + this.fileSystems.push({ + name, + provider: STORAGE_PROVIDER_EFS, + fileSystemId: efs.fileSystem.ref, + fileSystem: efs, + }); + } + + buildFsxLustre(name: string, storageConfig: Record): void { + const fsxLustre = new FSxForLustre(this.context, `${name}-storage-fsx-lustre`, this.stack, { + vpc: this.cluster.vpc, + fsxLustreConfig: valueAsDict('fsx_lustre', storageConfig), + securityGroup: this.securityGroup as SharedStorageSecurityGroup, + subnets: this.cluster.privateSubnets, + }); + this.fileSystems.push({ + name, + provider: STORAGE_PROVIDER_FSX_LUSTRE, + fileSystemId: fsxLustre.fileSystem.ref, + fileSystem: fsxLustre, + }); + } + + /** `FileSystemHolder.file_system_dns`: `...`. */ + fileSystemDns(fileSystem: FileSystemHolder): string { + const config = this.context.config; + const awsRegion = config.getString('cluster.aws.region', undefined, { required: true }) as string; + const dnsSuffix = config.getString('cluster.aws.dns_suffix', undefined, { required: true }) as string; + const fsType = fileSystem.provider === STORAGE_PROVIDER_EFS ? 'efs' : 'fsx'; + return `${fileSystem.fileSystemId}.${fsType}.${awsRegion}.${dnsSuffix}`; + } + + buildClusterSettings(): void { + const clusterSettings: Record = { + deployment_id: this.deploymentId, + security_group_id: (this.securityGroup as SharedStorageSecurityGroup).securityGroupId, + }; + + for (const fileSystem of this.fileSystems) { + if ( + fileSystem.provider === STORAGE_PROVIDER_EFS || + fileSystem.provider === STORAGE_PROVIDER_FSX_LUSTRE + ) { + // The module id, not the file system key: the lookup never resolves, so `dns` is always + // rewritten. See the file header. + const configuredDns = this.context.config.getString(`shared-storage.${this.moduleId}.dns`); + if (isEmpty(configuredDns)) { + clusterSettings[`${fileSystem.name}.${fileSystem.provider}.dns`] = + this.fileSystemDns(fileSystem); + } + } + + // An existing file system already carries its settings; only provisioned ones write back. + if (fileSystem.fileSystem === undefined) continue; + clusterSettings[`${fileSystem.name}.${fileSystem.provider}.file_system_id`] = + fileSystem.fileSystemId; + } + + this.updateClusterSettings(clusterSettings); + } +} + +export function buildStack(props: StackBuildProps): void { + new SharedStorageStack(props); +} diff --git a/source/idea/ideactl/src/cdk/stacks/vdc.ts b/source/idea/ideactl/src/cdk/stacks/vdc.ts new file mode 100644 index 00000000..0b32a2cb --- /dev/null +++ b/source/idea/ideactl/src/cdk/stacks/vdc.ts @@ -0,0 +1,1446 @@ +/** + * Three EC2 host groups (controller, DCV broker, DCV connection gateway), the external network + * load balancer the gateway sits behind, the DCV host (VDI) identity used by session instances, + * the SQS/SNS plumbing the controller listens on, and the scheduled-event transformer Lambda. + * + * Build order is load-bearing: it fixes the statement order in + * the controller role's CDK-generated `DefaultPolicy` and the order of the security-group rules. + * + * Load-bearing behaviour: + * + * - the DCV broker *agent* endpoint registers itself as `broker-client-endpoint`; the name is the + * custom resource's physical identity, so correcting it would re-register the ALB rule; + * - `instanceMonitoring` is passed alongside a launch template. `Monitoring.BASIC` is 0, so the + * CDK guard against launch-configuration properties does not fire; detailed monitoring would + * throw at synth. The config key the settings template writes never matches the one read here, + * so the value is always `BASIC`; + * - `cluster.backups.enabled` is read with `getString`, so a boolean `false` arrives as the + * non-empty string `false` and passes the guard. The second guard on + * `vdi_host_backup.enabled` is what actually decides; + * - target groups are attached by overwriting `targetGroupArns` on the L1. The array order is + * part of the deployed template. + */ + +import { CustomResource, Duration, Fn, Tags } from 'aws-cdk-lib'; +import * as autoscaling from 'aws-cdk-lib/aws-autoscaling'; +import * as cognito from 'aws-cdk-lib/aws-cognito'; +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as elbv2 from 'aws-cdk-lib/aws-elasticloadbalancingv2'; +import * as events from 'aws-cdk-lib/aws-events'; +import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; +import * as sns from 'aws-cdk-lib/aws-sns'; +import * as snsSubscriptions from 'aws-cdk-lib/aws-sns-subscriptions'; +import * as sqs from 'aws-cdk-lib/aws-sqs'; + +import { ArnBuilder } from '../../config/arn-builder.ts'; +import type { ClusterConfig } from '../../config/cluster-config.ts'; +import type { StackBuildProps } from '../app.ts'; +import { IdeaBaseStack } from '../base-stack.ts'; +import { IdeaCodeAsset } from '../code-asset.ts'; +import { buildBootstrapUserData } from '../userdata.ts'; +import { + IDEA_TAG_NAME, + IDEA_TAG_NODE_TYPE, + instanceProfileArn, +} from '../constructs/base.ts'; +import { BackupPlan } from '../constructs/backup.ts'; +import { + InstanceProfile, + LambdaFunction, + ManagedPolicy, + Policy, + Role, + SNSTopic, + SQSQueue, +} from '../constructs/common.ts'; +import { OAuthClientIdAndSecret } from '../constructs/directory-service.ts'; +import { + ExistingSocaCluster, + lookupBackupRole, + lookupClusterBackupVault, + lookupClusterS3Bucket, + lookupEbsKmsKey, + lookupEc2StateChangeTopic, + lookupKeyPair, +} from '../constructs/existing-resources.ts'; +import { + VirtualDesktopBastionAccessSecurityGroup, + VirtualDesktopBrokerSecurityGroup, + VirtualDesktopPublicLoadBalancerAccessSecurityGroup, + type SecurityGroup, +} from '../constructs/network.ts'; + +/** `constants.MODULE_VIRTUAL_DESKTOP_CONTROLLER`. */ +const MODULE_VIRTUAL_DESKTOP_CONTROLLER = 'virtual-desktop-controller'; +/** `constants.MODULE_CLUSTER_MANAGER`. */ +const MODULE_CLUSTER_MANAGER = 'cluster-manager'; +/** `constants.IDEA_TAG_MODULE_ID`, with `:` replaced by `_` for the SNS filter policy key. */ +const IDEA_TAG_MODULE_ID = 'idea:ModuleId'; +/** `constants.NODE_TYPE_APP` / `NODE_TYPE_INFRA`. */ +const NODE_TYPE_APP = 'app'; +const NODE_TYPE_INFRA = 'infra'; +/** `constants.SQS_MAX_RECEIVE_COUNT_DEFAULT` and `SQS_VISIBILITY_TIMEOUT_DEFAULT`. */ +const SQS_MAX_RECEIVE_COUNT_DEFAULT = 16; +const SQS_VISIBILITY_TIMEOUT_DEFAULT = 30; +/** GovCloud does not accept `Tags` on `AWS::Events::Rule`. */ +const AWS_PARTITION_GOVCLOUD = 'aws-us-gov'; +const OS_AMAZONLINUX2 = 'amazonlinux2'; +const OS_AMAZONLINUX2023 = 'amazonlinux2023'; + +const COMPONENT_CONTROLLER = 'controller'; +const COMPONENT_DCV_BROKER = 'broker'; +const COMPONENT_DCV_CONNECTION_GATEWAY = 'gateway'; +const COMPONENT_DCV_HOST = 'host'; + +/** Component id -> the name the config keys use. */ +const CONFIG_MAPPING: Record = { + [COMPONENT_CONTROLLER]: 'controller', + [COMPONENT_DCV_CONNECTION_GATEWAY]: 'dcv_connection_gateway', + [COMPONENT_DCV_BROKER]: 'dcv_broker', +}; + +/** + * `get_bool(key, required=True)`, `get_int(...)` and `get_list(...)`. Unlike `getString`, these + * getters have no overload for an undefined default, so the call is typed here instead. + */ +type RequiredGet = (key: string, defaultValue?: T, options?: { required?: boolean }) => T; + +function requiredBool(config: ClusterConfig, key: string): boolean { + return (config.getBool as RequiredGet)(key, undefined, { required: true }); +} + +function requiredInt(config: ClusterConfig, key: string): number { + return (config.getInt as RequiredGet)(key, undefined, { required: true }); +} + +function requiredList(config: ClusterConfig, key: string): string[] { + return (config.getList as RequiredGet)(key, undefined, { required: true }); +} + +/** `Utils.get_ec2_block_device_name`. */ +function ec2BlockDeviceName(baseOs: string): string { + return baseOs === OS_AMAZONLINUX2 || baseOs === OS_AMAZONLINUX2023 ? '/dev/xvda' : '/dev/sda1'; +} + +interface AutoScalingGroupOptions { + componentName: string; + securityGroup: SecurityGroup; + iamRole: Role; + substitutedUserdata: string; + nodeType: string; +} + +export class VirtualDesktopControllerStack extends IdeaBaseStack { + readonly cluster: ExistingSocaCluster; + readonly arnBuilder: ArnBuilder; + readonly userPool: cognito.IUserPool; + + readonly brokerClientCommunicationPort: number; + readonly brokerAgentCommunicationPort: number; + readonly brokerGatewayCommunicationPort: number; + readonly clusterEndpointsLambdaArn: string; + private readonly ecsEnabled: boolean; + private readonly hostsPresent: boolean; + + oauth2ClientSecret!: OAuthClientIdAndSecret; + + dcvHostRole!: Role; + controllerRole!: Role; + dcvBrokerRole!: Role; + dcvConnectionGatewayRole!: Role; + scheduledEventTransformerLambdaRole!: Role; + dcvHostInstanceProfile!: InstanceProfile; + dcvHostPolicy!: ManagedPolicy; + + dcvHostSecurityGroup!: VirtualDesktopBastionAccessSecurityGroup; + controllerSecurityGroup!: VirtualDesktopPublicLoadBalancerAccessSecurityGroup; + dcvConnectionGatewaySecurityGroup!: VirtualDesktopPublicLoadBalancerAccessSecurityGroup; + dcvBrokerSecurityGroup!: VirtualDesktopBrokerSecurityGroup; + + dcvConnectionGatewaySelfSignedCert: CustomResource | undefined; + externalNlb!: elbv2.NetworkLoadBalancer; + + eventSqsQueue!: SQSQueue; + eventSqsQueueDlq!: SQSQueue; + controllerSqsQueue!: SQSQueue; + controllerSqsQueueDlq!: SQSQueue; + ssmCommandsSnsTopic!: SNSTopic; + ssmCommandPassRole!: Role; + + controllerAutoScalingGroup!: autoscaling.AutoScalingGroup; + dcvBrokerAutoScalingGroup!: autoscaling.AutoScalingGroup; + dcvConnectionGatewayAutoScalingGroup!: autoscaling.AutoScalingGroup; + + backupPlan: BackupPlan | undefined; + + constructor(props: StackBuildProps) { + super({ + scope: props.app, + ctx: props.ctx, + moduleName: props.moduleName, + deploymentId: props.deploymentId, + terminationProtection: props.terminationProtection, + env: props.env, + }); + + const config = this.context.config; + this.ecsEnabled = config.getBool("ecs.enabled", false); + // `ecs.enabled` alone routes the endpoints to the container services and deletes the three host + // groups that serve them in one change set, so nothing proves the new targets before the old + // ones are gone. With `ecs.retain_existing_hosts` the same deploy routes the endpoints and keeps + // the groups, idle and unregistered, so a later deploy removes them once the containers are + // serving and turning the flag off puts the groups back in service. + this.hostsPresent = !this.ecsEnabled || config.getBool("ecs.retain_existing_hosts", false); + this.brokerClientCommunicationPort = requiredInt( + config, + 'virtual-desktop-controller.dcv_broker.client_communication_port', + ); + this.brokerAgentCommunicationPort = requiredInt( + config, + 'virtual-desktop-controller.dcv_broker.agent_communication_port', + ); + this.brokerGatewayCommunicationPort = requiredInt( + config, + 'virtual-desktop-controller.dcv_broker.gateway_communication_port', + ); + this.clusterEndpointsLambdaArn = config.getString('cluster.cluster_endpoints_lambda_arn', undefined, { + required: true, + }) as string; + + this.cluster = new ExistingSocaCluster(this.context, this.stack); + this.arnBuilder = new ArnBuilder(config); + + this.userPool = this.lookupUserPool(); + + this.buildOauth2Client(); + this.buildAccessControlGroups(this.userPool); + + this.buildSqsQueues(); + this.buildScheduledEventNotificationInfra(); + this.subscribeToEc2NotificationEvents(); + + this.buildVirtualDesktopController(); + this.buildDcvBroker(); + this.buildDcvConnectionGateway(); + this.buildDcvHostInfra(); + + this.buildControllerSsmCommandsNotificationInfra(); + this.buildBackups(); + + this.setupEgressRulesForQuic(); + this.buildClusterSettings(); + } + + // --- helpers ---------------------------------------------------------------------------------- + + private requiredString(key: string): string { + return this.context.config.getString(key, undefined, { required: true }) as string; + } + + /** A cluster security group by name; `undefined` here is a cluster-config error. */ + private clusterSecurityGroup(name: string): ec2.ISecurityGroup { + const securityGroup = this.cluster.getSecurityGroup(name); + if (securityGroup === undefined) { + throw new Error(`cluster.network.security_groups.${name} not found`); + } + return securityGroup; + } + + /** The `http_proxy`/`https_proxy`/`no_proxy` block every host group shares. */ + private proxyConfig(): Record { + const httpsProxy = this.context.config.getString('cluster.network.https_proxy', ''); + if (httpsProxy === '') return {}; + return { + http_proxy: httpsProxy, + https_proxy: httpsProxy, + no_proxy: this.context.config.getString('cluster.network.no_proxy', ''), + }; + } + + /** A `` config key under the module namespace. */ + private componentKey(componentName: string, suffix: string): string { + return `virtual-desktop-controller.${CONFIG_MAPPING[componentName] as string}.${suffix}`; + } + + + /** + * Bootstrap package location for one desktop component. + * + * The deploy path uploads the package and passes its location as context. A standalone synthesis + * has no context, and this used to fall back to the literal string for absent, which rendered + * straight into host user data: the host then tried to download a package called "not-provided". + * It also made every comparison against a real cluster's deployed template report three + * differences that were the harness's fault rather than the port's. The sibling stacks derive the + * location from the deployment identifier instead, by the same naming rule the uploader uses, so + * this does too. + */ + private componentBootstrapPackageUri(contextKey: string, componentSuffix: string): string { + const fromContext: unknown = this.stack.node.tryGetContext(contextKey); + if (typeof fromContext === 'string' && fromContext !== '') return fromContext; + const bucket = this.context.config.getString('cluster.cluster_s3_bucket', undefined, { + required: true, + }) as string; + return ( + `s3://${bucket}/idea/bootstrap/bootstrap-${this.moduleId}-${componentSuffix}` + + `-${this.deploymentId}.tar.gz` + ); + } + + private removeBastionHostIngressRule(securityGroup: SecurityGroup): void { + const ingressRule = securityGroup.node.children.find( + (child) => + child instanceof ec2.CfnSecurityGroupIngress && + child.description === "Allow SSH from Bastion Host", + ); + if (ingressRule === undefined) { + throw new Error("VDC security group has no bastion SSH ingress rule"); + } + securityGroup.node.tryRemoveChild(ingressRule.node.id); + } + + private ecsTargetGroupArn(role: string, index: number): string { + const targetGroupArns = this.context.config.getList( + `ecs.${role}.target_group_arns`, + [], + { required: true }, + ); + const targetGroupArn = targetGroupArns[index]; + if (targetGroupArn === undefined || targetGroupArn === "") { + throw new Error(`ecs.${role}.target_group_arns[${index}] is required when ecs.enabled is true`); + } + return targetGroupArn; + } + + private ecsServiceName(role: string): string { + const serviceArn = this.requiredString(`ecs.${role}.service_arn`); + const serviceName = serviceArn.split("/").at(-1); + if (serviceName === undefined || serviceName === "") { + throw new Error(`ecs.${role}.service_arn does not contain a service name`); + } + return serviceName; + } + + // --- QUIC ------------------------------------------------------------------------------------- + + setupEgressRulesForQuic(): void { + const quicSupported = requiredBool( + this.context.config, + 'virtual-desktop-controller.dcv_session.quic_support', + ); + if (!quicSupported) return; + + this.dcvHostSecurityGroup.addEgressRule( + ec2.Peer.ipv4('0.0.0.0/0'), + ec2.Port.udpRange(0, 65535), + 'Allow all egress for UDP for QUIC Support on DCV Host', + ); + this.dcvHostSecurityGroup.addEgressRule( + ec2.Peer.ipv6('::/0'), + ec2.Port.udpRange(0, 65535), + 'Allow all egress for UDP for QUIC Support on DCV Host', + ); + + this.dcvConnectionGatewaySecurityGroup.addEgressRule( + ec2.Peer.ipv4('0.0.0.0/0'), + ec2.Port.udpRange(0, 65535), + 'Allow all egress for UDP for QUIC Support on DCV Connection Gateway', + ); + this.dcvConnectionGatewaySecurityGroup.addEgressRule( + ec2.Peer.ipv6('::/0'), + ec2.Port.udpRange(0, 65535), + 'Allow all egress for UDP for QUIC Support on DCV Connection Gateway', + ); + } + + // --- messaging -------------------------------------------------------------------------------- + + buildSqsQueues(): void { + const config = this.context.config; + + this.eventSqsQueueDlq = new SQSQueue( + this.context, + 'virtual-desktop-controller-events-queue-dlq', + this.stack, + { + queueName: `${this.clusterName}-${this.moduleId}-events-dlq.fifo`, + fifo: true, + fifoThroughputLimit: sqs.FifoThroughputLimit.PER_MESSAGE_GROUP_ID, + deduplicationScope: sqs.DeduplicationScope.MESSAGE_GROUP, + contentBasedDeduplication: true, + encryptionMasterKey: config.getString('cluster.sqs.kms_key_id'), + isDeadLetterQueue: true, + }, + ); + this.eventSqsQueue = new SQSQueue(this.context, 'virtual-desktop-controller-events-queue', this.stack, { + queueName: `${this.clusterName}-${this.moduleId}-events.fifo`, + fifo: true, + fifoThroughputLimit: sqs.FifoThroughputLimit.PER_MESSAGE_GROUP_ID, + deduplicationScope: sqs.DeduplicationScope.MESSAGE_GROUP, + contentBasedDeduplication: true, + encryptionMasterKey: config.getString('cluster.sqs.kms_key_id'), + deadLetterQueue: { maxReceiveCount: SQS_MAX_RECEIVE_COUNT_DEFAULT, queue: this.eventSqsQueueDlq }, + }); + this.addCommonTags(this.eventSqsQueue); + this.addCommonTags(this.eventSqsQueueDlq); + + const kmsKey = config.getString('cluster.sqs.kms_key_id'); + const encryptAtRest = kmsKey !== undefined && kmsKey !== ''; + + this.controllerSqsQueueDlq = new SQSQueue( + this.context, + 'virtual-desktop-controller-queue-dlq', + this.stack, + { + queueName: `${this.clusterName}-${this.moduleId}-controller-dlq`, + encryptAtRest, + encryptionMasterKey: kmsKey, + isDeadLetterQueue: true, + }, + ); + this.controllerSqsQueue = new SQSQueue(this.context, 'virtual-desktop-controller-queue', this.stack, { + queueName: `${this.clusterName}-${this.moduleId}-controller`, + encryptAtRest, + encryptionMasterKey: kmsKey, + visibilityTimeout: Duration.seconds(SQS_VISIBILITY_TIMEOUT_DEFAULT), + deadLetterQueue: { + maxReceiveCount: SQS_MAX_RECEIVE_COUNT_DEFAULT, + queue: this.controllerSqsQueueDlq, + }, + }); + this.addCommonTags(this.controllerSqsQueue); + this.addCommonTags(this.controllerSqsQueueDlq); + } + + buildControllerSsmCommandsNotificationInfra(): void { + this.ssmCommandPassRole = new Role( + this.context, + `${this.moduleId}-ssm-commands-sns-topic-role`, + this.stack, + { + assumedBy: ['ssm'], + description: 'IAM role for SSM Commands to send notifications via SNS', + }, + ); + + this.ssmCommandPassRole.attachInlinePolicy( + new Policy( + this.context, + `${this.clusterName}-${this.moduleId}-ssm-commands-sns-topic-role-policy`, + this.stack, + { policyTemplateName: 'controller-ssm-command-pass-role.yml' }, + ), + ); + this.ssmCommandPassRole.grantPassRole(this.controllerRole); + + this.ssmCommandsSnsTopic = new SNSTopic( + this.context, + 'virtual-desktop-controller-sns-topic', + this.stack, + { + topicName: `${this.clusterName}-${this.moduleId}-ssm-commands-sns-topic`, + displayName: `${this.clusterName}-${this.moduleId}-ssm-commands-topic`, + masterKey: this.context.config.getString('cluster.sns.kms_key_id'), + }, + ); + this.addCommonTags(this.ssmCommandsSnsTopic); + this.ssmCommandsSnsTopic.addSubscription( + new snsSubscriptions.SqsSubscription(this.controllerSqsQueue, { + deadLetterQueue: this.controllerSqsQueueDlq, + }), + ); + } + + subscribeToEc2NotificationEvents(): void { + const ec2EventSnsTopic = lookupEc2StateChangeTopic(this.context, this.stack); + + ec2EventSnsTopic.addSubscription( + new snsSubscriptions.SqsSubscription(this.controllerSqsQueue, { + deadLetterQueue: this.controllerSqsQueueDlq, + filterPolicy: { + [IDEA_TAG_MODULE_ID.replace(':', '_')]: sns.SubscriptionFilter.stringFilter({ + allowlist: [this.moduleId], + }), + }, + }), + ); + } + + // --- scheduled events ------------------------------------------------------------------------- + + buildScheduledEventNotificationInfra(): void { + const lambdaName = `${this.moduleId}-scheduled-event-transformer`; + this.scheduledEventTransformerLambdaRole = new Role(this.context, `${lambdaName}-role`, this.stack, { + assumedBy: ['lambda'], + description: `${lambdaName}-role`, + }); + + this.scheduledEventTransformerLambdaRole.attachInlinePolicy( + new Policy(this.context, `${lambdaName}-policy`, this.stack, { + policyTemplateName: 'controller-scheduled-event-transformer-lambda.yml', + }), + ); + + const scheduledEventTransformerLambda = new LambdaFunction(this.context, lambdaName, this.stack, { + description: `${this.moduleId} lambda to intercept all scheduled events and transform to the required event object.`, + environment: { IDEA_CONTROLLER_EVENTS_QUEUE_URL: this.eventSqsQueue.queueUrl }, + timeoutSeconds: 180, + role: this.scheduledEventTransformerLambdaRole, + ideaCodeAsset: new IdeaCodeAsset('idea_controller_scheduled_event_transformer'), + }); + + this.addNagSuppression( + [{ rule_id: 'AwsSolutions-L1', reason: 'Python Runtime is selected for stability.' }], + scheduledEventTransformerLambda, + ); + + const scheduleTriggerRule = new events.Rule( + this.stack, + `${this.clusterName}-${this.moduleId}-schedule-rule`, + { + enabled: true, + ruleName: `${this.clusterName}-${this.moduleId}-schedule-rule`, + description: 'Event Rule to Trigger schedule check EVERY 30 minutes on VDC Controller', + schedule: events.Schedule.cron({ minute: '0/30' }), + }, + ); + + scheduleTriggerRule.addTarget(new eventsTargets.LambdaFunction(scheduledEventTransformerLambda)); + + // CloudFormation does not support tags on EventBridge rules in GovCloud. + if (this.requiredString('cluster.aws.partition') !== AWS_PARTITION_GOVCLOUD) { + this.addCommonTags(scheduleTriggerRule); + } + } + + // --- OAuth ------------------------------------------------------------------------------------ + + buildOauth2Client(): void { + const resourceServer = this.userPool.addResourceServer('resource-server', { + identifier: this.moduleId, + scopes: [ + new cognito.ResourceServerScope({ scopeName: 'read', scopeDescription: 'Allow Read Access' }), + new cognito.ResourceServerScope({ scopeName: 'write', scopeDescription: 'Allow Write Access' }), + ], + }); + + // DCV session manager external authentication. + const sessionManagerResourceServer = this.userPool.addResourceServer( + 'dcv-session-manager-resource-server', + { + identifier: 'dcv-session-manager', + scopes: [new cognito.ResourceServerScope({ scopeName: 'sm_scope', scopeDescription: 'sm_scope' })], + }, + ); + + const client = this.userPool.addClient(`${this.moduleId}-client`, { + accessTokenValidity: Duration.hours(1), + generateSecret: true, + idTokenValidity: Duration.hours(1), + oAuth: { + flows: { clientCredentials: true }, + scopes: [ + cognito.OAuthScope.custom(`${this.moduleId}/read`), + cognito.OAuthScope.custom(`${this.moduleId}/write`), + cognito.OAuthScope.custom(`${this.context.config.moduleId(MODULE_CLUSTER_MANAGER)}/read`), + cognito.OAuthScope.custom('dcv-session-manager/sm_scope'), + ], + }, + refreshTokenValidity: Duration.days(30), + userPoolClientName: this.moduleId, + }); + client.node.addDependency(sessionManagerResourceServer); + client.node.addDependency(resourceServer); + + const oauthCredentialsLambdaArn = this.requiredString( + 'identity-provider.cognito.oauth_credentials_lambda_arn', + ); + const clientSecret = new CustomResource(this.stack, `${this.moduleId}-creds`, { + serviceToken: oauthCredentialsLambdaArn, + properties: { + UserPoolId: this.userPool.userPoolId, + ClientId: client.userPoolClientId, + }, + resourceType: 'Custom::GetOAuthCredentials', + }); + + this.oauth2ClientSecret = new OAuthClientIdAndSecret( + this.context, + this.moduleId, + MODULE_VIRTUAL_DESKTOP_CONTROLLER, + this.stack, + client.userPoolClientId, + clientSecret.getAttString('ClientSecret'), + ); + } + + // --- DCV host (VDI) --------------------------------------------------------------------------- + + buildDcvHostInfra(): void { + // A customer-managed policy so the same base permissions can be attached to project roles. + this.dcvHostPolicy = new ManagedPolicy( + this.context, + `${this.moduleId}-${COMPONENT_DCV_HOST}-policy`, + this.stack, + { + managedPolicyName: `${this.clusterName}-${this.awsRegion}-${this.moduleId}-${COMPONENT_DCV_HOST}`, + description: `Permissions assigned to virtual-desktop-${COMPONENT_DCV_HOST}`, + policyTemplateName: 'virtual-desktop-dcv-host.yml', + }, + ); + this.dcvHostRole = this.buildIamRole( + `IAM role assigned to virtual-desktop-${COMPONENT_DCV_HOST}`, + COMPONENT_DCV_HOST, + ); + this.dcvHostRole.addManagedPolicy(this.dcvHostPolicy); + this.dcvHostRole.grantPassRole(this.controllerRole); + + this.dcvHostInstanceProfile = new InstanceProfile( + this.context, + `${this.moduleId}-${COMPONENT_DCV_HOST}-instance-profile`, + this.stack, + [this.dcvHostRole], + ); + + this.dcvHostSecurityGroup = new VirtualDesktopBastionAccessSecurityGroup( + this.context, + `${this.moduleId}-dcv-host-security-group`, + this.stack, + this.cluster.vpc, + { + bastionHostSecurityGroup: this.clusterSecurityGroup('bastion-host'), + description: 'Security Group for DCV Host', + directoryServiceAccess: true, + componentName: 'DCV Host', + }, + ); + } + + // --- DCV broker ------------------------------------------------------------------------------- + + buildDcvBroker(): void { + const clientTargetGroupArn = this.ecsEnabled + ? this.ecsTargetGroupArn("dcv-broker", 0) + : (() => { + const targetGroup = new elbv2.ApplicationTargetGroup( + this.stack, + `${COMPONENT_DCV_BROKER}-client-target-group`, + { + port: this.brokerClientCommunicationPort, + targetType: elbv2.TargetType.INSTANCE, + protocol: elbv2.ApplicationProtocol.HTTPS, + vpc: this.cluster.vpc, + targetGroupName: this.getTargetGroupName(`${COMPONENT_DCV_BROKER}-c`), + }, + ); + targetGroup.configureHealthCheck({ enabled: true, path: '/health' }); + return targetGroup.targetGroupArn; + })(); + + new CustomResource(this.stack, 'dcv-broker-client-endpoint', { + serviceToken: this.clusterEndpointsLambdaArn, + properties: { + endpoint_name: 'broker-client-endpoint', + listener_arn: this.requiredString( + 'cluster.load_balancers.internal_alb.dcv_broker_client_listener_arn', + ), + priority: 0, + default_action: true, + actions: [{ Type: 'forward', TargetGroupArn: clientTargetGroupArn }], + }, + resourceType: 'Custom::DcvBrokerClientEndpointInternal', + }); + + const agentTargetGroupArn = this.ecsEnabled + ? this.ecsTargetGroupArn("dcv-broker", 1) + : (() => { + const targetGroup = new elbv2.ApplicationTargetGroup( + this.stack, + `${COMPONENT_DCV_BROKER}-agent-target-group`, + { + port: this.brokerAgentCommunicationPort, + targetType: elbv2.TargetType.INSTANCE, + protocol: elbv2.ApplicationProtocol.HTTPS, + vpc: this.cluster.vpc, + targetGroupName: this.getTargetGroupName(`${COMPONENT_DCV_BROKER}-a`), + }, + ); + targetGroup.configureHealthCheck({ enabled: true, path: '/health' }); + return targetGroup.targetGroupArn; + })(); + + new CustomResource(this.stack, 'dcv-broker-agent-endpoint', { + serviceToken: this.clusterEndpointsLambdaArn, + properties: { + // The agent endpoint registers under the client endpoint's name; the name is the custom + // resource's physical identity, so it stays as deployed. + endpoint_name: 'broker-client-endpoint', + listener_arn: this.requiredString( + 'cluster.load_balancers.internal_alb.dcv_broker_agent_listener_arn', + ), + priority: 0, + default_action: true, + actions: [{ Type: 'forward', TargetGroupArn: agentTargetGroupArn }], + }, + resourceType: 'Custom::DcvBrokerAgentEndpointInternal', + }); + + const gatewayTargetGroupArn = this.ecsEnabled + ? this.ecsTargetGroupArn("dcv-broker", 2) + : (() => { + const targetGroup = new elbv2.ApplicationTargetGroup( + this.stack, + `${COMPONENT_DCV_BROKER}-gateway-target-group`, + { + port: this.brokerGatewayCommunicationPort, + targetType: elbv2.TargetType.INSTANCE, + protocol: elbv2.ApplicationProtocol.HTTPS, + vpc: this.cluster.vpc, + targetGroupName: this.getTargetGroupName(`${COMPONENT_DCV_BROKER}-g`), + }, + ); + targetGroup.configureHealthCheck({ enabled: true, path: '/health' }); + return targetGroup.targetGroupArn; + })(); + + new CustomResource(this.stack, 'dcv-broker-gateway-endpoint', { + serviceToken: this.clusterEndpointsLambdaArn, + properties: { + endpoint_name: 'broker-gateway-endpoint', + listener_arn: this.requiredString( + 'cluster.load_balancers.internal_alb.dcv_broker_gateway_listener_arn', + ), + priority: 0, + default_action: true, + actions: [{ Type: 'forward', TargetGroupArn: gatewayTargetGroupArn }], + }, + resourceType: 'Custom::DcvBrokerGatewayEndpointInternal', + }); + + this.dcvBrokerSecurityGroup = new VirtualDesktopBrokerSecurityGroup( + this.context, + `${this.moduleId}-${COMPONENT_DCV_BROKER}-security-group`, + this.stack, + this.cluster.vpc, + { + bastionHostSecurityGroup: this.clusterSecurityGroup('bastion-host'), + publicLoadbalancerSecurityGroup: this.clusterSecurityGroup('external-load-balancer'), + description: 'Security Group for Virtual Desktop DCV Broker', + componentName: 'DCV Broker', + }, + ); + // The rule exists to reach a host. It goes when the last host does, not when routing moves. + if (!this.hostsPresent) this.removeBastionHostIngressRule(this.dcvBrokerSecurityGroup); + + const dcvBrokerPackageUri = this.componentBootstrapPackageUri( + 'dcv_broker_bootstrap_package_uri', + 'dcv-broker', + ); + const proxyConfig = this.proxyConfig(); + + const brokerUserdata = buildBootstrapUserData({ + awsRegion: this.awsRegion, + bootstrapPackageUri: dcvBrokerPackageUri, + installCommands: ['/bin/bash dcv-broker/setup.sh'], + infraConfig: { + BROKER_CLIENT_TARGET_GROUP_ARN: '${__BROKER_CLIENT_TARGET_GROUP_ARN__}', + CONTROLLER_EVENTS_QUEUE_URL: '${__CONTROLLER_EVENTS_QUEUE_URL__}', + }, + proxyConfig, + baseOs: this.requiredString('virtual-desktop-controller.dcv_broker.autoscaling.base_os'), + }); + const substitutedUserdata = Fn.sub(brokerUserdata, { + __BROKER_CLIENT_TARGET_GROUP_ARN__: clientTargetGroupArn, + __CONTROLLER_EVENTS_QUEUE_URL__: this.eventSqsQueue.queueUrl, + }); + + this.dcvBrokerRole = this.buildIamRole( + `IAM role assigned to virtual-desktop-${COMPONENT_DCV_BROKER}`, + COMPONENT_DCV_BROKER, + 'virtual-desktop-dcv-broker.yml', + ); + + if (!this.hostsPresent) return; + + this.dcvBrokerAutoScalingGroup = this.buildAutoScalingGroup({ + componentName: COMPONENT_DCV_BROKER, + securityGroup: this.dcvBrokerSecurityGroup, + iamRole: this.dcvBrokerRole, + substitutedUserdata, + nodeType: NODE_TYPE_INFRA, + }); + this.dcvBrokerAutoScalingGroup.node.addDependency(this.eventSqsQueue); + + // Under the container flag these ARNs are the container target groups, which take IP targets, + // so a retained group registers with nothing and stands idle until a rollback recreates its own. + if (this.ecsEnabled) return; + + // An ASG cannot be added to a second target group through the L2, so the L1 property is + // written directly. The order is part of the deployed template. + ( + this.dcvBrokerAutoScalingGroup.node.defaultChild as autoscaling.CfnAutoScalingGroup + ).targetGroupArns = [ + agentTargetGroupArn, + clientTargetGroupArn, + gatewayTargetGroupArn, + ]; + } + + // --- IAM -------------------------------------------------------------------------------------- + + private buildIamRole(roleDescription: string, componentName: string, componentJinja?: string): Role { + const ec2ManagedPolicies = this.getEc2InstanceManagedPolicies(); + + const role = new Role(this.context, `${this.moduleId}-${componentName}-role`, this.stack, { + description: roleDescription, + assumedBy: ['ssm', 'ec2'], + managedPolicies: ec2ManagedPolicies, + }); + if (componentJinja === undefined) return role; + role.attachInlinePolicy( + new Policy( + this.context, + `${this.clusterName}-${this.moduleId}-${componentName}-policy`, + this.stack, + { policyTemplateName: componentJinja, vars: { role_arn: role.roleArn } }, + ), + ); + return role; + } + + // --- controller ------------------------------------------------------------------------------- + + buildVirtualDesktopController(): void { + const config = this.context.config; + this.controllerSecurityGroup = new VirtualDesktopPublicLoadBalancerAccessSecurityGroup( + this.context, + `${this.moduleId}-${COMPONENT_CONTROLLER}-security-group`, + this.stack, + this.cluster.vpc, + { + bastionHostSecurityGroup: this.clusterSecurityGroup('bastion-host'), + publicLoadbalancerSecurityGroup: this.clusterSecurityGroup('external-load-balancer'), + description: 'Security Group for Virtual Desktop Controller', + directoryServiceAccess: true, + componentName: 'Virtual Desktop Controller', + }, + ); + // The rule exists to reach a host. It goes when the last host does, not when routing moves. + if (!this.hostsPresent) this.removeBastionHostIngressRule(this.controllerSecurityGroup); + + const controllerBootstrapPackageUri = this.componentBootstrapPackageUri( + 'controller_bootstrap_package_uri', + 'controller', + ); + + this.controllerRole = this.buildIamRole( + `IAM role assigned to virtual-desktop-${COMPONENT_CONTROLLER}`, + COMPONENT_CONTROLLER, + 'virtual-desktop-controller.yml', + ); + + const proxyConfig = this.proxyConfig(); + + if (this.hostsPresent) { + this.controllerAutoScalingGroup = this.buildAutoScalingGroup({ + componentName: COMPONENT_CONTROLLER, + securityGroup: this.controllerSecurityGroup, + iamRole: this.controllerRole, + substitutedUserdata: Fn.sub( + buildBootstrapUserData({ + awsRegion: this.awsRegion, + bootstrapPackageUri: controllerBootstrapPackageUri, + installCommands: ['/bin/bash virtual-desktop-controller/setup.sh'], + proxyConfig, + baseOs: this.requiredString('virtual-desktop-controller.controller.autoscaling.base_os'), + }), + ), + nodeType: NODE_TYPE_APP, + }); + + this.controllerAutoScalingGroup.node.addDependency(this.eventSqsQueue); + this.controllerAutoScalingGroup.node.addDependency(this.controllerSqsQueue); + } + + const externalTargetGroupArn = this.ecsEnabled + ? this.ecsTargetGroupArn("vdc", 0) + : (() => { + const targetGroup = new elbv2.ApplicationTargetGroup( + this.stack, + 'controller-target-group-ext', + { + port: 8443, + protocol: elbv2.ApplicationProtocol.HTTPS, + protocolVersion: elbv2.ApplicationProtocolVersion.HTTP1, + targetType: elbv2.TargetType.INSTANCE, + vpc: this.cluster.vpc, + targetGroupName: this.getTargetGroupName('vdc-ext'), + }, + ); + targetGroup.configureHealthCheck({ enabled: true, path: '/healthcheck' }); + return targetGroup.targetGroupArn; + })(); + + new CustomResource(this.stack, 'controller-endpoint-ext', { + serviceToken: this.clusterEndpointsLambdaArn, + properties: { + endpoint_name: `${this.moduleId}-controller-endpoint-ext`, + listener_arn: this.requiredString('cluster.load_balancers.external_alb.https_listener_arn'), + priority: requiredInt(config, 'virtual-desktop-controller.controller.endpoints.external.priority'), + conditions: [ + { + Field: 'path-pattern', + Values: requiredList( + this.context.config, + 'virtual-desktop-controller.controller.endpoints.external.path_patterns', + ), + }, + ], + actions: [{ Type: 'forward', TargetGroupArn: externalTargetGroupArn }], + }, + resourceType: 'Custom::ControllerEndpointExternal', + }); + + const internalTargetGroupArn = this.ecsEnabled + ? this.ecsTargetGroupArn("vdc", 1) + : (() => { + const targetGroup = new elbv2.ApplicationTargetGroup( + this.stack, + 'controller-target-group-int', + { + port: 8443, + protocol: elbv2.ApplicationProtocol.HTTPS, + protocolVersion: elbv2.ApplicationProtocolVersion.HTTP1, + targetType: elbv2.TargetType.INSTANCE, + vpc: this.cluster.vpc, + targetGroupName: this.getTargetGroupName('vdc-int'), + }, + ); + targetGroup.configureHealthCheck({ enabled: true, path: '/healthcheck' }); + return targetGroup.targetGroupArn; + })(); + + new CustomResource(this.stack, 'controller-endpoint-int', { + serviceToken: this.clusterEndpointsLambdaArn, + properties: { + endpoint_name: `${this.moduleId}-controller-endpoint-int`, + listener_arn: this.requiredString('cluster.load_balancers.internal_alb.https_listener_arn'), + priority: requiredInt(config, 'virtual-desktop-controller.controller.endpoints.internal.priority'), + conditions: [ + { + Field: 'path-pattern', + Values: requiredList( + this.context.config, + 'virtual-desktop-controller.controller.endpoints.internal.path_patterns', + ), + }, + ], + actions: [{ Type: 'forward', TargetGroupArn: internalTargetGroupArn }], + }, + resourceType: 'Custom::ControllerEndpointInternal', + }); + + // Under the container flag these ARNs are the container target groups, which take IP targets, + // so a retained group registers with nothing and stands idle until a rollback recreates its own. + if (!this.ecsEnabled) { + ( + this.controllerAutoScalingGroup.node.defaultChild as autoscaling.CfnAutoScalingGroup + ).targetGroupArns = [internalTargetGroupArn, externalTargetGroupArn]; + } + } + + // --- host groups ------------------------------------------------------------------------------ + + private buildAutoScalingGroup(options: AutoScalingGroupOptions): autoscaling.AutoScalingGroup { + const config = this.context.config; + const { componentName } = options; + + const isPublic = + config.getBool(this.componentKey(componentName, 'autoscaling.public'), false) && + this.cluster.publicSubnets.length > 0; + const vpcSubnets: ec2.SubnetSelection = isPublic + ? { subnets: this.cluster.publicSubnets } + : { subnets: this.cluster.privateSubnets }; + + const baseOs = this.requiredString(this.componentKey(componentName, 'autoscaling.base_os')); + const blockDeviceName = ec2BlockDeviceName(baseOs); + const blockDeviceTypeString = config.getString('virtual-desktop-controller.volume_type', 'gp3'); + const blockDeviceVolumeType = + blockDeviceTypeString === 'gp3' ? ec2.EbsDeviceVolumeType.GP3 : ec2.EbsDeviceVolumeType.GP2; + + const enableDetailedMonitoring = config.getBool( + this.componentKey(componentName, 'autoscaling.enable_detailed_monitoring'), + false, + ); + const metadataHttpTokens = this.requiredString( + this.componentKey(componentName, 'autoscaling.metadata_http_tokens'), + ); + + const ebsKmsKey = lookupEbsKmsKey(this.context, this.stack, componentName); + + const launchTemplate = new ec2.LaunchTemplate(this.stack, `${componentName}-lt`, { + instanceType: new ec2.InstanceType( + this.requiredString(this.componentKey(componentName, 'autoscaling.instance_type')), + ), + machineImage: ec2.MachineImage.genericLinux({ + [this.awsRegion]: this.requiredString( + this.componentKey(componentName, 'autoscaling.instance_ami'), + ), + }), + securityGroup: options.securityGroup, + userData: ec2.UserData.custom(options.substitutedUserdata), + keyPair: lookupKeyPair(this.context, this.stack, `${componentName}-key-pair`), + blockDevices: [ + { + deviceName: blockDeviceName, + volume: ec2.BlockDeviceVolume.ebs( + config.getInt(this.componentKey(componentName, 'autoscaling.volume_size'), 200), + { encrypted: true, kmsKey: ebsKmsKey, volumeType: blockDeviceVolumeType }, + ), + }, + ], + role: options.iamRole, + requireImdsv2: metadataHttpTokens === 'required', + }); + + const autoScalingGroup = new autoscaling.AutoScalingGroup(this.stack, `${componentName}-asg`, { + vpc: this.cluster.vpc, + vpcSubnets, + autoScalingGroupName: `${this.clusterName}-${this.moduleId}-${componentName}-asg`, + launchTemplate, + // `Monitoring.BASIC` is 0, which is why passing this alongside a launch template does not + // trip the CDK guard. Detailed monitoring would throw at synth. + instanceMonitoring: enableDetailedMonitoring + ? autoscaling.Monitoring.DETAILED + : autoscaling.Monitoring.BASIC, + groupMetrics: [autoscaling.GroupMetrics.all()], + minCapacity: config.getInt(this.componentKey(componentName, 'autoscaling.min_capacity'), 1), + maxCapacity: config.getInt(this.componentKey(componentName, 'autoscaling.max_capacity'), 3), + newInstancesProtectedFromScaleIn: config.getBool( + this.componentKey(componentName, 'autoscaling.new_instances_protected_from_scale_in'), + true, + ), + cooldown: Duration.minutes( + config.getInt(this.componentKey(componentName, 'autoscaling.cooldown_minutes'), 5), + ), + healthChecks: autoscaling.HealthChecks.withAdditionalChecks({ + additionalTypes: [autoscaling.AdditionalHealthCheckType.ELB], + gracePeriod: Duration.minutes( + config.getInt( + this.componentKey(componentName, 'autoscaling.elb_healthcheck.grace_time_minutes'), + 15, + ), + ), + }), + updatePolicy: autoscaling.UpdatePolicy.rollingUpdate({ + maxBatchSize: config.getInt( + this.componentKey(componentName, 'autoscaling.rolling_update_policy.max_batch_size'), + 1, + ), + minInstancesInService: config.getInt( + this.componentKey( + componentName, + 'autoscaling.rolling_update_policy.min_instances_in_service', + ), + 1, + ), + pauseTime: Duration.minutes( + config.getInt( + this.componentKey(componentName, 'autoscaling.rolling_update_policy.pause_time_minutes'), + 15, + ), + ), + }), + terminationPolicies: [autoscaling.TerminationPolicy.DEFAULT], + }); + + autoScalingGroup.scaleOnCpuUtilization('cpu-utilization-scaling-policy', { + targetUtilizationPercent: config.getInt( + this.componentKey( + componentName, + 'autoscaling.cpu_utilization_scaling_policy.target_utilization_percent', + ), + 80, + ), + estimatedInstanceWarmup: Duration.minutes( + config.getInt( + this.componentKey( + componentName, + 'autoscaling.cpu_utilization_scaling_policy.estimated_instance_warmup_minutes', + ), + 15, + ), + ), + }); + + Tags.of(autoScalingGroup).add(IDEA_TAG_NODE_TYPE, options.nodeType); + Tags.of(autoScalingGroup).add( + IDEA_TAG_NAME, + `${this.clusterName}-${this.moduleId}-${componentName}`, + ); + + if (!enableDetailedMonitoring) { + this.addNagSuppression( + [ + { + rule_id: 'AwsSolutions-EC28', + reason: 'detailed monitoring is a configurable option to save costs', + }, + ], + autoScalingGroup, + true, + ); + } + + this.addNagSuppression( + [ + { + rule_id: 'AwsSolutions-AS3', + reason: 'ASG notifications scaling notifications can be managed via AWS Console', + }, + ], + autoScalingGroup, + ); + return autoScalingGroup; + } + + // --- DCV connection gateway ------------------------------------------------------------------- + + private buildDcvConnectionGatewayInstanceInfrastructure(): void { + this.dcvConnectionGatewaySecurityGroup = new VirtualDesktopPublicLoadBalancerAccessSecurityGroup( + this.context, + `${this.moduleId}-${COMPONENT_DCV_CONNECTION_GATEWAY}-security-group`, + this.stack, + this.cluster.vpc, + { + bastionHostSecurityGroup: this.clusterSecurityGroup('bastion-host'), + publicLoadbalancerSecurityGroup: this.clusterSecurityGroup('external-load-balancer'), + description: 'Security Group for Virtual Desktop DCV Connection Gateway', + directoryServiceAccess: false, + componentName: 'DCV Connection Gateway', + }, + ); + // The rule exists to reach a host. It goes when the last host does, not when routing moves. + if (!this.hostsPresent) this.removeBastionHostIngressRule(this.dcvConnectionGatewaySecurityGroup); + + if (!this.hostsPresent) { + this.dcvConnectionGatewayRole = this.buildIamRole( + `IAM role assigned to virtual-desktop-${COMPONENT_DCV_CONNECTION_GATEWAY}`, + COMPONENT_DCV_CONNECTION_GATEWAY, + 'virtual-desktop-dcv-connection-gateway.yml', + ); + return; + } + + const gatewayBootstrapPackageUri = this.componentBootstrapPackageUri( + 'dcv_connection_gateway_package_uri', + 'dcv-connection-gateway', + ); + + const proxyConfig = this.proxyConfig(); + + const connectionGatewayUserdata = buildBootstrapUserData({ + awsRegion: this.awsRegion, + bootstrapPackageUri: gatewayBootstrapPackageUri, + installCommands: ['/bin/bash dcv-connection-gateway/setup.sh'], + infraConfig: { + CERTIFICATE_SECRET_ARN: '${__CERTIFICATE_SECRET_ARN__}', + PRIVATE_KEY_SECRET_ARN: '${__PRIVATE_KEY_SECRET_ARN__}', + }, + proxyConfig, + baseOs: this.requiredString( + 'virtual-desktop-controller.dcv_connection_gateway.autoscaling.base_os', + ), + }); + + const externalCertificateProvided = requiredBool( + this.context.config, + 'virtual-desktop-controller.dcv_connection_gateway.certificate.provided', + ); + const selfSignedCert = this.dcvConnectionGatewaySelfSignedCert; + const substitutedUserdata = !externalCertificateProvided + ? Fn.sub(connectionGatewayUserdata, { + __CERTIFICATE_SECRET_ARN__: (selfSignedCert as CustomResource).getAttString( + 'certificate_secret_arn', + ), + __PRIVATE_KEY_SECRET_ARN__: (selfSignedCert as CustomResource).getAttString( + 'private_key_secret_arn', + ), + }) + : Fn.sub(connectionGatewayUserdata, { + __CERTIFICATE_SECRET_ARN__: this.requiredString( + 'virtual-desktop-controller.dcv_connection_gateway.certificate.certificate_secret_arn', + ), + __PRIVATE_KEY_SECRET_ARN__: this.requiredString( + 'virtual-desktop-controller.dcv_connection_gateway.certificate.private_key_secret_arn', + ), + }); + + this.dcvConnectionGatewayRole = this.buildIamRole( + `IAM role assigned to virtual-desktop-${COMPONENT_DCV_CONNECTION_GATEWAY}`, + COMPONENT_DCV_CONNECTION_GATEWAY, + 'virtual-desktop-dcv-connection-gateway.yml', + ); + + this.dcvConnectionGatewayAutoScalingGroup = this.buildAutoScalingGroup({ + componentName: COMPONENT_DCV_CONNECTION_GATEWAY, + securityGroup: this.dcvConnectionGatewaySecurityGroup, + iamRole: this.dcvConnectionGatewayRole, + substitutedUserdata, + nodeType: NODE_TYPE_INFRA, + }); + } + + private buildDcvConnectionGatewayNetworkInfrastructure(): void { + const config = this.context.config; + const isPublic = config.getBool('cluster.load_balancers.external_alb.public', true); + const externalNlbSubnets = isPublic ? this.cluster.publicSubnets : this.cluster.privateSubnets; + this.externalNlb = new elbv2.NetworkLoadBalancer( + this.stack, + `${this.clusterName}-${this.moduleId}-external-nlb`, + { + loadBalancerName: `${this.clusterName}-${this.moduleId}-external-nlb`, + vpc: this.cluster.vpc, + internetFacing: isPublic, + vpcSubnets: { subnets: externalNlbSubnets }, + }, + ); + + if (config.getBool('virtual-desktop-controller.external_nlb.access_logs', false)) { + const accessLogDestination = lookupClusterS3Bucket(this.context, this.stack); + this.externalNlb.logAccessLogs( + accessLogDestination, + `logs/${this.moduleId}/external-nlb-access-logs`, + ); + } + + const quicSupported = requiredBool(config, 'virtual-desktop-controller.dcv_session.quic_support'); + const protocol = quicSupported ? elbv2.Protocol.TCP_UDP : elbv2.Protocol.TCP; + // TN: TCP network. TUN: TCP + UDP network. + const tgSuffix = quicSupported ? 'TUN' : 'TN'; + + let dcvConnectionGatewayTargetGroup: elbv2.INetworkTargetGroup; + if (this.ecsEnabled) { + dcvConnectionGatewayTargetGroup = elbv2.NetworkTargetGroup.fromTargetGroupAttributes( + this.stack, + 'ecs-dcv-connection-gateway-target-group-nlb', + { + // The container stack publishes the one gateway target group that matches this + // setting, so there is a single entry whichever protocol is in use. + targetGroupArn: this.ecsTargetGroupArn("dcv-gateway", 0), + }, + ); + } else { + const targetGroup = new elbv2.NetworkTargetGroup( + this.stack, + 'dcv-connection-gateway-target-group-nlb', + { + port: 8443, + protocol, + targetType: elbv2.TargetType.INSTANCE, + vpc: this.cluster.vpc, + targets: [this.dcvConnectionGatewayAutoScalingGroup], + targetGroupName: this.getTargetGroupName( + `${COMPONENT_DCV_CONNECTION_GATEWAY}-${tgSuffix}`, + ), + healthCheck: { port: '8989', protocol: elbv2.Protocol.TCP }, + connectionTermination: true, + }, + ); + // Stickiness attributes have no construct property on a network target group. + targetGroup.setAttribute('stickiness.enabled', 'true'); + targetGroup.setAttribute('stickiness.type', 'source_ip'); + dcvConnectionGatewayTargetGroup = targetGroup; + } + + new elbv2.NetworkListener(this.externalNlb, 'dcv-connection-gateway-nlb-listener', { + loadBalancer: this.externalNlb, + protocol, + port: 443, + defaultAction: elbv2.NetworkListenerAction.forward([dcvConnectionGatewayTargetGroup]), + }); + + this.dcvConnectionGatewaySecurityGroup.addIngressRule( + ec2.Peer.ipv4(this.cluster.vpc.vpcCidrBlock), + ec2.Port.tcp(8989), + 'Allow TCP traffic access for HealthCheck to DCV Connection Gateway', + ); + + const clusterPrefixListId = this.requiredString('cluster.network.cluster_prefix_list_id'); + this.dcvConnectionGatewaySecurityGroup.addIngressRule( + ec2.Peer.prefixList(clusterPrefixListId), + ec2.Port.allTraffic(), + 'Allow all Traffic access from Cluster Prefix List to DCV Connection Gateway', + ); + + for (const prefixListId of config.getList('cluster.network.prefix_list_ids', [])) { + this.dcvConnectionGatewaySecurityGroup.addIngressRule( + ec2.Peer.prefixList(prefixListId), + ec2.Port.allTraffic(), + 'Allow all traffic access from Prefix List to DCV Connection Gateway', + ); + } + } + + private buildSelfSignedCertForDcvConnectionGateway(): void { + const selfSignedCertificateLambdaArn = this.requiredString( + 'cluster.self_signed_certificate_lambda_arn', + ); + this.dcvConnectionGatewaySelfSignedCert = new CustomResource( + this.stack, + `${this.clusterName}-${this.moduleId}-external-cert-${COMPONENT_DCV_CONNECTION_GATEWAY}`, + { + serviceToken: selfSignedCertificateLambdaArn, + properties: { + domain_name: `${this.moduleId}.${this.clusterName}.idea.default`, + certificate_name: `${this.clusterName}-${this.moduleId}-${COMPONENT_DCV_CONNECTION_GATEWAY}-certificate`, + create_acm_certificate: false, + kms_key_id: this.context.config.getString('cluster.secretsmanager.kms_key_id'), + tags: { + Name: `${this.clusterName}-${this.moduleId}-${COMPONENT_DCV_CONNECTION_GATEWAY} Self Signed Certificate`, + 'idea:ClusterName': this.clusterName, + 'idea:ModuleName': MODULE_VIRTUAL_DESKTOP_CONTROLLER, + }, + }, + resourceType: 'Custom::SelfSignedCertificateConnectionGateway', + }, + ); + } + + buildDcvConnectionGateway(): void { + const externalCertificateProvided = requiredBool( + this.context.config, + 'virtual-desktop-controller.dcv_connection_gateway.certificate.provided', + ); + if (!externalCertificateProvided) { + this.buildSelfSignedCertForDcvConnectionGateway(); + } + + this.buildDcvConnectionGatewayInstanceInfrastructure(); + this.buildDcvConnectionGatewayNetworkInfrastructure(); + } + + // --- backups ---------------------------------------------------------------------------------- + + buildBackups(): void { + // A boolean `false` renders as `False`, a non-empty string that passes this guard. + const clusterBackupsEnabled = this.context.config.getString('cluster.backups.enabled'); + if (!clusterBackupsEnabled) return; + + const vdiHostBackupEnabled = this.context.config.getBool( + 'virtual-desktop-controller.vdi_host_backup.enabled', + false, + ); + if (!vdiHostBackupEnabled) return; + + const backupRole = lookupBackupRole(this.context, this.stack); + const backupVault = lookupClusterBackupVault(this.context, this.stack); + + const backupPlanConfig = this.context.config.getConfig( + 'virtual-desktop-controller.vdi_host_backup.backup_plan', + ); + + this.backupPlan = new BackupPlan(this.stack, { + backupPlanName: `${this.clusterName}-${this.moduleId}`, + backupPlanConfig, + backupVault, + backupRole, + }); + } + + // --- cluster settings ------------------------------------------------------------------------- + + buildClusterSettings(): void { + const config = this.context.config; + const clusterSettings: Record = { + deployment_id: this.deploymentId, + client_id: this.oauth2ClientSecret.clientId.ref, + client_secret: this.oauth2ClientSecret.clientSecret.ref, + dcv_host_security_group_id: this.dcvHostSecurityGroup.securityGroupId, + dcv_host_role_arn: this.dcvHostRole.roleArn, + dcv_host_role_name: this.dcvHostRole.roleName, + dcv_host_role_id: this.dcvHostRole.roleId, + dcv_host_policy_arn: this.dcvHostPolicy.managedPolicyArn, + dcv_broker_role_arn: this.dcvBrokerRole.roleArn, + dcv_broker_role_name: this.dcvBrokerRole.roleName, + dcv_broker_role_id: this.dcvBrokerRole.roleId, + scheduled_event_transformer_lambda_role_arn: this.scheduledEventTransformerLambdaRole.roleArn, + scheduled_event_transformer_lambda_role_name: this.scheduledEventTransformerLambdaRole.roleName, + scheduled_event_transformer_lambda_role_id: this.scheduledEventTransformerLambdaRole.roleId, + dcv_host_instance_profile_arn: instanceProfileArn(this.context, this.dcvHostInstanceProfile.ref), + ssm_commands_sns_topic_arn: this.ssmCommandsSnsTopic.topicArn, + ssm_commands_sns_topic_name: this.ssmCommandsSnsTopic.topicName, + ssm_commands_pass_role_arn: this.ssmCommandPassRole.roleArn, + ssm_commands_pass_role_id: this.ssmCommandPassRole.roleId, + ssm_commands_pass_role_name: this.ssmCommandPassRole.roleName, + controller_iam_role_arn: this.controllerRole.roleArn, + controller_iam_role_name: this.controllerRole.roleName, + controller_iam_role_id: this.controllerRole.roleId, + events_sqs_queue_url: this.eventSqsQueue.queueUrl, + events_sqs_queue_arn: this.eventSqsQueue.queueArn, + controller_sqs_queue_url: this.controllerSqsQueue.queueUrl, + controller_sqs_queue_arn: this.controllerSqsQueue.queueArn, + 'external_nlb.load_balancer_dns_name': this.externalNlb.loadBalancerDnsName, + 'controller.asg_name': this.ecsEnabled + ? this.ecsServiceName("vdc") + : this.controllerAutoScalingGroup.autoScalingGroupName, + 'controller.asg_arn': this.ecsEnabled + ? this.requiredString("ecs.vdc.service_arn") + : this.controllerAutoScalingGroup.autoScalingGroupArn, + 'dcv_broker.asg_name': this.ecsEnabled + ? this.ecsServiceName("dcv-broker") + : this.dcvBrokerAutoScalingGroup.autoScalingGroupName, + 'dcv_broker.asg_arn': this.ecsEnabled + ? this.requiredString("ecs.dcv-broker.service_arn") + : this.dcvBrokerAutoScalingGroup.autoScalingGroupArn, + 'dcv_connection_gateway.asg_name': this.ecsEnabled + ? this.ecsServiceName("dcv-gateway") + : this.dcvConnectionGatewayAutoScalingGroup.autoScalingGroupName, + 'dcv_connection_gateway.asg_arn': this.ecsEnabled + ? this.requiredString("ecs.dcv-gateway.service_arn") + : this.dcvConnectionGatewayAutoScalingGroup.autoScalingGroupArn, + }; + + if ( + !config.getBool('virtual-desktop-controller.dcv_connection_gateway.certificate.provided', false) + ) { + const selfSignedCert = this.dcvConnectionGatewaySelfSignedCert as CustomResource; + clusterSettings['dcv_connection_gateway.certificate.certificate_secret_arn'] = + selfSignedCert.getAttString('certificate_secret_arn'); + clusterSettings['dcv_connection_gateway.certificate.private_key_secret_arn'] = + selfSignedCert.getAttString('private_key_secret_arn'); + } else { + clusterSettings['dcv_connection_gateway.certificate.provided'] = this.requiredString( + 'virtual-desktop-controller.dcv_connection_gateway.certificate.provided', + ); + clusterSettings['dcv_connection_gateway.certificate.certificate_secret_arn'] = + this.requiredString( + 'virtual-desktop-controller.dcv_connection_gateway.certificate.certificate_secret_arn', + ); + clusterSettings['dcv_connection_gateway.certificate.private_key_secret_arn'] = + this.requiredString( + 'virtual-desktop-controller.dcv_connection_gateway.certificate.private_key_secret_arn', + ); + clusterSettings['dcv_connection_gateway.certificate.custom_dns_name'] = this.requiredString( + 'virtual-desktop-controller.dcv_connection_gateway.certificate.custom_dns_name', + ); + } + + if (this.backupPlan !== undefined) { + clusterSettings['vdi_host_backup.backup_plan.arn'] = this.backupPlan.getBackupPlanArn(); + } + + // The controller's iam:PassRole for project roles is granted at deploy time when bedrock is + // enabled, so the setting is written under the same gate and the web portal can tell a + // redeploy is owed. + const clusterManagerModuleId = config.moduleId(MODULE_CLUSTER_MANAGER); + if (config.getBool(`${clusterManagerModuleId}.bedrock.enabled`, false)) { + clusterSettings['bedrock.project_pass_role_arn'] = this.arnBuilder.getProjectRoleArn(); + } + + this.updateClusterSettings(clusterSettings); + } +} + +export function buildStack(props: StackBuildProps): void { + new VirtualDesktopControllerStack(props); +} diff --git a/source/idea/ideactl/src/cdk/stateful.ts b/source/idea/ideactl/src/cdk/stateful.ts new file mode 100644 index 00000000..9b9437ec --- /dev/null +++ b/source/idea/ideactl/src/cdk/stateful.ts @@ -0,0 +1,62 @@ +/** + * One definition of "stateful" for the two layers that act on it. + * + * The membership rule is what three years of deployment history on the real clusters shows has + * never been replaced by an upgrade. Compute instances and load-balancer target groups are + * deliberately absent: both are replaced routinely and by design, so marking them Retain would + * leave an orphan every time rather than protecting anything. Nothing here has ever been lost. + * This is insurance against the first plausible occasion, which is a control-plane cutover. + * + * The synthesis marks these resources `UpdateReplacePolicy: Retain` (`RetainStatefulOnUpdateReplace` + * in `app.ts`), so an update that forces a replacement orphans the old resource instead of + * deleting it. The deploy-time change-set guard + * in `src/cli/cdk-invoker.ts` refuses a `Remove` of the same set. Both import from here, so the + * two layers cannot come to disagree about what counts as stateful. + * + * Membership is by CloudFormation namespace rather than by exact type, so a type added to one of + * these services later is covered without anyone remembering to add it. The cost is that the + * association and policy members of those namespaces are covered too, and one of those retained + * is litter rather than a saved copy. That trade is deliberate: an orphan can be inspected and + * deleted by hand, a deleted file system cannot be recovered at all. + * + * `OpenSearchService`/`Elasticsearch` and `KinesisFirehose` are the same services under their two + * CloudFormation namespaces; both spellings are listed so a template that uses the older one is + * still covered. + * + * This module holds no imports on purpose: the command line imports it too, and it must not pull + * the CDK library into a process that only reads a change set. + */ + +export const STATEFUL_TYPE_PREFIXES: readonly string[] = [ + // The identity store, the directory, the file systems, the search domain, the streams, the + // secrets, the DNS zone and records, the queues, the topics and the logs. + 'AWS::Cognito::', + 'AWS::DirectoryService::', + 'AWS::EFS::', + 'AWS::FSx::', + 'AWS::OpenSearchService::', + 'AWS::OpenSearchServerless::', + 'AWS::Elasticsearch::', + 'AWS::Kinesis::', + 'AWS::KinesisFirehose::', + 'AWS::SecretsManager::', + 'AWS::Route53::', + 'AWS::SQS::', + 'AWS::SNS::', + 'AWS::Logs::', + // A vault holds recovery points, and a plan or selection that stops existing stops producing + // them. + 'AWS::Backup::', + // Not built by any stack today. Listed so the first one that is arrives protected, because + // these are the namespaces where a deletion is unrecoverable by definition. + 'AWS::S3::Bucket', + 'AWS::DynamoDB::', + 'AWS::RDS::', + 'AWS::EC2::Volume', + 'AWS::KMS::Key', +]; + +export function isStatefulType(resourceType: string | undefined): boolean { + if (resourceType === undefined) return false; + return STATEFUL_TYPE_PREFIXES.some((prefix) => resourceType.startsWith(prefix)); +} diff --git a/source/idea/ideactl/src/cdk/synth-reads.ts b/source/idea/ideactl/src/cdk/synth-reads.ts new file mode 100644 index 00000000..8665e764 --- /dev/null +++ b/source/idea/ideactl/src/cdk/synth-reads.ts @@ -0,0 +1,122 @@ +/** + * Provides the five AWS reads that CDK synthesis needs behind one interface, so + * synthesis can run without credentials. + * + * Replay is keyed `service:action:JSON(params)`, where params are the API parameters the + * callers pass. A missing key throws rather than falling back to a default. + */ + +import { readFileSync } from 'node:fs'; +import type { Listener } from '@aws-sdk/client-elastic-load-balancing-v2'; +import type { UserPoolType } from '@aws-sdk/client-cognito-identity-provider'; +import type { Role } from '@aws-sdk/client-iam'; +import type { DomainStatus } from '@aws-sdk/client-opensearch'; +import { awsClientOptions } from "../cli/aws-client-options.ts"; + +export type UserPool = UserPoolType; + +export interface SynthReads { + callerIdentity(): Promise<{ account: string; arn: string }>; + describeListener(listenerArn: string): Promise; + describeUserPool(userPoolId: string): Promise; + listServiceLinkedRoles(pathPrefix: string): Promise; + describeDomain(domainName: string): Promise; +} + +export class SynthReadMiss extends Error { + readonly key: string; + constructor(key: string) { + super(`SynthReadMiss: ${key}`); + this.name = 'SynthReadMiss'; + this.key = key; + } +} + +/** The replay key for one read. Used by both the replay reader and `capture.ts`. */ +export function synthReadKey(service: string, action: string, params: Record): string { + return `${service}:${action}:${JSON.stringify(params)}`; +} + +export const CALLER_IDENTITY_KEY = synthReadKey('sts', 'GetCallerIdentity', {}); +export const listenerKey = (listenerArn: string) => synthReadKey('elbv2', 'DescribeListeners', { ListenerArns: [listenerArn] }); +export const userPoolKey = (userPoolId: string) => synthReadKey('cognito-idp', 'DescribeUserPool', { UserPoolId: userPoolId }); +export const listRolesKey = (pathPrefix: string) => synthReadKey('iam', 'ListRoles', { PathPrefix: pathPrefix }); +export const describeDomainKey = (domainName: string) => synthReadKey('opensearch', 'DescribeDomain', { DomainName: domainName }); + +/** + * Replays a `synth-reads.json` fixture. Values are unwrapped interface results: + * one `Listener`, one `UserPool`, a `Role[]`, or one `DomainStatus`. + */ +export function replaySynthReads(file: string): SynthReads { + const reads = JSON.parse(readFileSync(file, 'utf8')) as Record; + const get = (key: string): T => { + if (!(key in reads)) throw new SynthReadMiss(key); + return reads[key] as T; + }; + return { + async callerIdentity() { + return get<{ account: string; arn: string }>(CALLER_IDENTITY_KEY); + }, + async describeListener(listenerArn) { + return get(listenerKey(listenerArn)); + }, + async describeUserPool(userPoolId) { + return get(userPoolKey(userPoolId)); + }, + async listServiceLinkedRoles(pathPrefix) { + return get(listRolesKey(pathPrefix)); + }, + async describeDomain(domainName) { + return get(describeDomainKey(domainName)); + }, + }; +} + +/** Reads live AWS data. Clients load lazily so replay tests do not load the SDK. */ +export function liveSynthReads(region: string, profile?: string): SynthReads { + // The CDK app loads cluster configuration after this call through a client it does not own. + if (profile !== undefined && profile.trim() !== "") process.env.AWS_PROFILE = profile; + + const once = (make: () => Promise): (() => Promise) => { + let p: Promise | undefined; + return () => (p ??= make()); + }; + const config = once(() => awsClientOptions(region, profile)); + const sts = once(async () => new (await import('@aws-sdk/client-sts')).STSClient(await config())); + const elbv2 = once(async () => new (await import('@aws-sdk/client-elastic-load-balancing-v2')).ElasticLoadBalancingV2Client(await config())); + const cognito = once(async () => new (await import('@aws-sdk/client-cognito-identity-provider')).CognitoIdentityProviderClient(await config())); + const iam = once(async () => new (await import('@aws-sdk/client-iam')).IAMClient(await config())); + const opensearch = once(async () => new (await import('@aws-sdk/client-opensearch')).OpenSearchClient(await config())); + + return { + async callerIdentity() { + const { GetCallerIdentityCommand } = await import('@aws-sdk/client-sts'); + const result = await (await sts()).send(new GetCallerIdentityCommand({})); + return { account: result.Account ?? '', arn: result.Arn ?? '' }; + }, + async describeListener(listenerArn) { + const { DescribeListenersCommand } = await import('@aws-sdk/client-elastic-load-balancing-v2'); + const result = await (await elbv2()).send(new DescribeListenersCommand({ ListenerArns: [listenerArn] })); + const listener = result.Listeners?.[0]; + if (!listener) throw new Error(`no such listener: ${listenerArn}`); + return listener; + }, + async describeUserPool(userPoolId) { + const { DescribeUserPoolCommand } = await import('@aws-sdk/client-cognito-identity-provider'); + const result = await (await cognito()).send(new DescribeUserPoolCommand({ UserPoolId: userPoolId })); + if (!result.UserPool) throw new Error(`no such user pool: ${userPoolId}`); + return result.UserPool; + }, + async listServiceLinkedRoles(pathPrefix) { + const { ListRolesCommand } = await import('@aws-sdk/client-iam'); + const result = await (await iam()).send(new ListRolesCommand({ PathPrefix: pathPrefix })); + return result.Roles ?? []; + }, + async describeDomain(domainName) { + const { DescribeDomainCommand } = await import('@aws-sdk/client-opensearch'); + const result = await (await opensearch()).send(new DescribeDomainCommand({ DomainName: domainName })); + if (!result.DomainStatus) throw new Error(`no such opensearch domain: ${domainName}`); + return result.DomainStatus; + }, + }; +} diff --git a/source/idea/ideactl/src/cdk/userdata.ts b/source/idea/ideactl/src/cdk/userdata.ts new file mode 100644 index 00000000..38208b1d --- /dev/null +++ b/source/idea/ideactl/src/cdk/userdata.ts @@ -0,0 +1,173 @@ +/** + * Builds Linux and Windows user data before CDK wraps it in `Fn::Sub` or + * `Fn::Base64`. It preserves trailing indentation, empty config here-docs, + * and a newline after each install command. + */ + +export interface BootstrapUserDataParams { + baseOs: string; + awsRegion: string; + bootstrapPackageUri: string; + installCommands: string[]; + infraConfig?: Record | null; + proxyConfig?: Record | null; + /** Defaults to true. False emits `${VAR}` instead of Fn::Sub-escaped `${!VAR}`. */ + substitutionSupport?: boolean; +} + +/** + * The shell body shared by both Linux variants, written in the substitution form. + * The two forms differ only in `${!` versus `${`. + */ +const LINUX_BODY = ` + +timestamp=$(date +%s) +mkdir -p /root/bootstrap/logs +if [[ -f /root/bootstrap/logs/userdata.log ]]; then + mv /root/bootstrap/logs/userdata.log /root/bootstrap/logs/userdata.log.\${!timestamp} +fi +exec > /root/bootstrap/logs/userdata.log 2>&1 + +export PATH="\${!PATH}:/usr/local/bin" + +function install_aws_cli () { + if [[ "\${!BASE_OS}" == "amazonlinux2023" ]]; then + yum remove -y awscli + fi + cd /root/bootstrap + local machine=$(uname -m) + if [[ \${!machine} == "x86_64" ]]; then + curl -s \${!AWSCLI_X86_64_URL} -o "awscliv2.zip" + elif [[ \${!machine} == "aarch64" ]]; then + curl -s \${!AWSCLI_AARCH64_URL} -o "awscliv2.zip" + fi + which unzip > /dev/null 2>&1 + if [[ "$?" != "0" ]]; then + if [[ $BASE_OS =~ ^ubuntu ]]; then + apt install -y unzip + else + yum install -y unzip + fi + fi + unzip -q awscliv2.zip + ./aws/install --bin-dir /bin --update + rm -rf aws awscliv2.zip +} + +echo "#!/bin/bash +PACKAGE_DOWNLOAD_URI=\\\${!1} +PACKAGE_ARCHIVE=\\$(basename \\\${!PACKAGE_DOWNLOAD_URI}) +PACKAGE_NAME=\\\${!PACKAGE_ARCHIVE%.tar.gz*} +INSTANCE_REGION=\\$(TOKEN=\\$(curl --silent -X PUT 'http://169.254.169.254/latest/api/token' -H 'X-aws-ec2-metadata-token-ttl-seconds: 900') && curl --silent -H \\"X-aws-ec2-metadata-token: \\\${!TOKEN}\\" 'http://169.254.169.254/latest/meta-data/placement/region') +if [[ \\\${!PACKAGE_DOWNLOAD_URI} == s3://* ]]; then + AWS=\\$(command -v aws) + S3_BUCKET=\\$(echo \\\${!PACKAGE_DOWNLOAD_URI} | cut -f3 -d/) + if [[ \\\${!INSTANCE_REGION} =~ ^us-gov-[a-z]+-[0-9]+$ ]]; then + S3_BUCKET_REGION=\\$(curl -s --head https://\\\${!S3_BUCKET}.s3.us-gov-west-1.amazonaws.com | grep bucket-region | awk '{print \\$2}' | tr -d '\\r\\n') + \\$AWS --region \\\${!S3_BUCKET_REGION} s3 cp \\\${!PACKAGE_DOWNLOAD_URI} /root/bootstrap/ + else + #S3_BUCKET_REGION=\\$(curl -s --head https://\\\${!S3_BUCKET}.s3.us-east-1.amazonaws.com | grep bucket-region | awk '{print \\$2}' | tr -d '\\r\\n') + \\$AWS --region \\\${!INSTANCE_REGION} s3 cp \\\${!PACKAGE_DOWNLOAD_URI} /root/bootstrap/ + fi +else + cp \\\${!PACKAGE_DOWNLOAD_URI} /root/bootstrap/ +fi +PACKAGE_DIR=/root/bootstrap/\\\${!PACKAGE_NAME} +if [[ -d \\\${!PACKAGE_DIR} ]]; then + rm -rf \\\${!PACKAGE_DIR} +fi +mkdir -p \\\${!PACKAGE_DIR} +tar -xvf /root/bootstrap/\\\${!PACKAGE_ARCHIVE} -C \\\${!PACKAGE_DIR} +rm /root/bootstrap/latest +ln -sf \\\${!PACKAGE_DIR} /root/bootstrap/latest +" > /root/bootstrap/download_bootstrap.sh + +chmod +x /root/bootstrap/download_bootstrap.sh + `; + +const windowsUserData = (uri: string): string => ` + + $BootstrapDir = "C:\\Users\\Administrator\\IDEA\\bootstrap" + function Download-Idea-Package { + Param( + [ValidateNotNullOrEmpty()] + [Parameter(Mandatory=$true)] + [String] $PackageDownloadURI + ) + if (!(Test-Path "$BootstrapDir")) { + New-Item -itemType Directory -Path "$BootstrapDir" + } + cd "$BootstrapDir" + Write-Output $PackageDownloadURI + $PackageArchive=Split-Path $PackageDownloadURI -Leaf + $PackageName = [System.IO.Path]::GetFileNameWithoutExtension($PackageDownloadURI) + if ($PackageDownloadURI -like "s3\`://*") { + $urlParts = $PackageDownloadURI -Split "/", 4 + $bucketName = $urlParts[2] + $key = $urlParts[3] + Copy-S3Object -BucketName $bucketName -Key $key -LocalFile "$BootstrapDir\\$PackageArchive" -Force + } else { + Copy-Item -Path $PackageDownloadURI -Destination "$BootstrapDir\\$PackageArchive" + } + Tar -xf "$BootstrapDir\\$PackageArchive" + } + Download-Idea-Package ${uri} +`; + +export function buildBootstrapUserData(params: BootstrapUserDataParams): string { + const { + baseOs, + awsRegion, + bootstrapPackageUri, + installCommands, + infraConfig, + proxyConfig, + substitutionSupport = true, + } = params; + + if (baseOs.toLowerCase().includes('windows')) { + if (infraConfig && Object.keys(infraConfig).length > 0) { + throw new Error('infra config is not supported for windows'); + } + return ( + windowsUserData(bootstrapPackageUri) + + installCommands.map((c) => `${c}\n`).join('') + + '' + ); + } + + const infra = Object.entries(infraConfig ?? {}) + .map(([k, v]) => `${k}=${v}\n`) + .join(''); + const proxy = Object.entries(proxyConfig ?? {}) + .map(([k, v]) => `export ${k}=${v}\n`) + .join(''); + + let userdata = + `#!/bin/bash\n` + + `\n` + + `set -x\n` + + `mkdir -p /root/bootstrap\n` + + `AWS_REGION="${awsRegion}"\n` + + `BASE_OS="${baseOs}"\n` + + `DEFAULT_AWS_REGION="${awsRegion}"\n` + + `AWSCLI_X86_64_URL="https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip"\n` + + `AWSCLI_AARCH64_URL="https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip"\n`; + + // The substitution variant writes infra.cfg, including when it is empty. + if (substitutionSupport) { + userdata += `\necho "\n${infra}\n" > /root/bootstrap/infra.cfg\n `; + } + + userdata += `\necho "${proxy}\n" > /root/bootstrap/proxy.cfg\nsource /root/bootstrap/proxy.cfg\n `; + + userdata += substitutionSupport ? LINUX_BODY : LINUX_BODY.replaceAll('${!', '${'); + + userdata += + `\ninstall_aws_cli\n` + + `bash /root/bootstrap/download_bootstrap.sh "${bootstrapPackageUri}"\n` + + `\n` + + `cd /root/bootstrap/latest\n`; + + return userdata + installCommands.map((c) => `${c}\n`).join(''); +} diff --git a/source/idea/ideactl/src/cli/aws-client-options.ts b/source/idea/ideactl/src/cli/aws-client-options.ts new file mode 100644 index 00000000..d8a6ad8a --- /dev/null +++ b/source/idea/ideactl/src/cli/aws-client-options.ts @@ -0,0 +1,73 @@ +/** + * Builds AWS client options that bind a requested shared configuration profile + * to the client credential provider. + */ + +type ProfileCredentialsProvider = ReturnType< + typeof import("@aws-sdk/credential-provider-ini")["fromIni"] +>; + +export interface AwsClientOptions { + region: string; + credentials?: ProfileCredentialsProvider; +} + +export interface AwsCallerIdentity { + account: string; + arn: string; +} + +/** Reports profile resolution failures without allowing another credential source to run. */ +export class AwsProfileCredentialsError extends Error { + readonly profile: string; + + constructor(profile: string, cause: unknown) { + super( + `AWS profile ${profile} was not found in the shared config/credentials files. Create the profile, or pass an existing name with --aws-profile. AWS_PROFILE is also read.`, + { cause }, + ); + this.name = "AwsProfileCredentialsError"; + this.profile = profile; + } +} + +/** Return the explicitly requested profile, including one selected through the environment. */ +function requestedProfile(profile: string | undefined): string | undefined { + const value = profile === undefined || profile.trim() === "" ? process.env.AWS_PROFILE : profile; + return value === undefined || value.trim() === "" ? undefined : value; +} + +/** + * Build service client options. A named profile uses only that profile and + * converts every resolution error into a message that names it. + */ +export async function awsClientOptions( + region: string, + profile?: string, +): Promise { + const selected = requestedProfile(profile); + if (selected === undefined) return { region }; + + const { fromIni } = await import("@aws-sdk/credential-provider-ini"); + const resolve = fromIni({ profile: selected }); + return { + region, + credentials: async () => { + try { + return await resolve(); + } catch (cause) { + throw new AwsProfileCredentialsError(selected, cause); + } + }, + }; +} + +/** Format the account and principal returned by STS before an account operation starts. */ +export function formatAwsIdentity( + identity: AwsCallerIdentity, + profile?: string, +): string { + const selected = requestedProfile(profile); + const suffix = selected === undefined ? "" : `, profile ${selected}`; + return `AWS identity: account ${identity.account}, identity ${identity.arn}${suffix}`; +} diff --git a/source/idea/ideactl/src/cli/bootstrap-context.ts b/source/idea/ideactl/src/cli/bootstrap-context.ts new file mode 100644 index 00000000..cd08ae4a --- /dev/null +++ b/source/idea/ideactl/src/cli/bootstrap-context.ts @@ -0,0 +1,939 @@ +/** + * Builds the template context used by module host bootstrap packages. + * + * The property and method names are intentionally snake case because the bootstrap templates call + * the same public surface as the reference implementation. + */ + +import type { ClusterConfig } from "../config/cluster-config.ts"; +import { GeneralException, isEmpty } from "../config/cluster-config.ts"; +import { toYaml } from "../config/jinja.ts"; +import { ideaVersion } from "../version.ts"; +import type { BootstrapContextInput } from "./cdk-invoker.ts"; + +type JsonObject = Record; +type NodeType = "app" | "infra"; + +interface LogFile { + file_path: string; + log_group_name: string; + log_stream_name: string; +} + +interface HostRole { + baseOs: string; + instanceType: string; + metricsNamespace: string; + nodeType: NodeType; + enableLogging: boolean; + logFiles: LogFile[]; + vars: JsonObject; +} + +interface ConfigCall { + key: string; + fallback: unknown; + required: boolean; +} + +/** Returns true only for ordinary key-value objects. */ +function isRecord(value: unknown): value is JsonObject { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** Reads keyword arguments emitted by the template renderer. */ +function keywordArguments(args: readonly unknown[]): JsonObject { + const last = args.at(-1); + return isRecord(last) && last.__keywords === true ? last : {}; +} + +/** Reads one positional or named argument from a template call. */ +function callArgument( + args: readonly unknown[], + position: number, + name: string, +): unknown { + const keywords = keywordArguments(args); + const positionalLength = Object.keys(keywords).length === 0 ? args.length : args.length - 1; + return position < positionalLength ? args[position] : keywords[name]; +} + +/** Parses the common `(key, default=None, required=False)` getter shape. */ +function configCall(args: readonly unknown[]): ConfigCall { + const key = callArgument(args, 0, "key"); + if (typeof key !== "string" || key.length === 0) { + throw new GeneralException("config key is required"); + } + return { + key, + fallback: callArgument(args, 1, "default"), + required: callArgument(args, 2, "required") === true, + }; +} + +/** Exposes the configuration methods used by the bootstrap templates. */ +function configFacade(config: ClusterConfig): object { + return { + get_string: (...args: unknown[]) => { + const call = configCall(args); + return Reflect.apply(config.getString, config, [ + call.key, + call.fallback, + { required: call.required }, + ]); + }, + get_bool: (...args: unknown[]) => { + const call = configCall(args); + return Reflect.apply(config.getBool, config, [ + call.key, + call.fallback, + { required: call.required }, + ]); + }, + get_int: (...args: unknown[]) => { + const call = configCall(args); + return Reflect.apply(config.getInt, config, [ + call.key, + call.fallback, + { required: call.required }, + ]); + }, + get_list: (...args: unknown[]) => { + const call = configCall(args); + return Reflect.apply(config.getList, config, [ + call.key, + call.fallback, + { required: call.required }, + ]); + }, + get_config: (...args: unknown[]) => { + const call = configCall(args); + return Reflect.apply(config.getConfig, config, [ + call.key, + call.fallback, + { required: call.required }, + ]); + }, + get_module_id: (moduleName: string) => config.moduleId(moduleName), + is_module_enabled: (moduleName: string) => config.isModuleEnabled(moduleName), + get_cluster_internal_endpoint: () => config.getClusterInternalEndpoint(), + get_cluster_external_endpoint: () => config.getClusterExternalEndpoint(), + }; +} + +/** Resolves the release URI that the package publisher already uploaded. */ +function releaseUri(input: BootstrapContextInput, packagePrefix: string): string { + const entry = Object.entries(input.releasePackageUris).find(([name]) => + name.startsWith(packagePrefix), + ); + if (entry === undefined || isEmpty(entry[1])) { + throw new GeneralException( + `release package URI not found for module ${input.moduleId}: ${packagePrefix}`, + ); + } + return entry[1]; +} + +/** Creates one log-file entry in the order used by the reference model. */ +function logFile( + filePath: string, + logGroupName: string, + logStreamName: string, +): LogFile { + return { + file_path: filePath, + log_group_name: logGroupName, + log_stream_name: logStreamName, + }; +} + +/** Resolves the role-specific inputs for one package plan. */ +function hostRole(input: BootstrapContextInput): HostRole { + const clusterName = requiredString(input.config, "cluster.cluster_name"); + const moduleLogGroup = `/${clusterName}/${input.moduleId}`; + const moduleMetrics = `${clusterName}/${input.moduleId}`; + + if (input.moduleName === "directoryservice") { + const name = `${moduleLogGroup}/openldap-server`; + return { + baseOs: input.baseOs, + instanceType: input.instanceType, + metricsNamespace: `${moduleMetrics}/openldap-server`, + nodeType: "infra", + enableLogging: + input.config.getBool("directoryservice.cloudwatch_logs.enabled", false) === + true, + logFiles: [ + logFile("/var/log/messages", name, "system_{ip_address}"), + logFile("/var/log/syslog", name, "syslog_{ip_address}"), + logFile("/var/log/slapd.log", name, "slapd_{ip_address}"), + ], + vars: {}, + }; + } + + if (input.moduleName === "cluster-manager") { + return { + baseOs: input.baseOs, + instanceType: input.instanceType, + metricsNamespace: moduleMetrics, + nodeType: "app", + enableLogging: + input.config.getBool("cluster-manager.cloudwatch_logs.enabled", false) === + true, + logFiles: [ + logFile("/opt/idea/app/logs/**.log", moduleLogGroup, "application_{ip_address}"), + logFile("/var/log/messages", moduleLogGroup, "system_{ip_address}"), + logFile("/var/log/syslog", moduleLogGroup, "syslog_{ip_address}"), + ], + vars: { + app_package_uri: releaseUri(input, "idea-cluster-manager-"), + }, + }; + } + + if (input.moduleName === "scheduler") { + const logFiles = [ + logFile("/opt/idea/app/logs/**.log", moduleLogGroup, "application_{ip_address}"), + logFile("/var/log/messages", moduleLogGroup, "system_{ip_address}"), + logFile("/var/log/syslog", moduleLogGroup, "syslog_{ip_address}"), + ]; + if (requiredString(input.config, "scheduler.provider") === "openpbs") { + const openPbsLogGroup = `${moduleLogGroup}/openpbs`; + logFiles.push( + logFile( + "/var/spool/pbs/server_logs/**.log", + openPbsLogGroup, + "server_logs_{ip_address}", + ), + logFile( + "/var/spool/pbs/sched_logs/**.log", + openPbsLogGroup, + "sched_logs_{ip_address}", + ), + logFile( + "/var/spool/pbs/server_priv/accounting/**.log", + openPbsLogGroup, + "accounting_logs_{ip_address}", + ), + ); + } + return { + baseOs: input.baseOs, + instanceType: input.instanceType, + metricsNamespace: moduleMetrics, + nodeType: "app", + enableLogging: + input.config.getBool("scheduler.cloudwatch_logs.enabled", false) === true, + logFiles, + vars: { + app_package_uri: releaseUri(input, "idea-scheduler-"), + }, + }; + } + + if (input.moduleName === "bastion-host") { + return { + baseOs: input.baseOs, + instanceType: input.instanceType, + metricsNamespace: moduleMetrics, + nodeType: "infra", + enableLogging: + input.config.getBool("bastion-host.cloudwatch_logs.enabled", false) === true, + logFiles: [ + logFile("/var/log/messages", moduleLogGroup, "system_{ip_address}"), + logFile("/var/log/syslog", moduleLogGroup, "syslog_{ip_address}"), + logFile("/var/log/secure", moduleLogGroup, "secure_{ip_address}"), + logFile("/var/log/auth.log", moduleLogGroup, "auth_{ip_address}"), + ], + vars: {}, + }; + } + + if (input.moduleName !== "virtual-desktop-controller") { + throw new GeneralException( + `bootstrap context is not defined for module: ${input.moduleName}`, + ); + } + + const vdcLogging = + input.config.getBool( + "virtual-desktop-controller.cloudwatch_logs.enabled", + false, + ) === true; + if (input.plan.contextParameter === "controller_bootstrap_package_uri") { + const name = `${moduleLogGroup}/controller`; + return { + baseOs: input.baseOs, + instanceType: input.instanceType, + metricsNamespace: `${moduleMetrics}/controller`, + nodeType: "app", + enableLogging: vdcLogging, + logFiles: [ + logFile("/opt/idea/app/logs/**.log", name, "application_{ip_address}"), + logFile("/var/log/messages", name, "system_{ip_address}"), + logFile("/var/log/syslog", name, "syslog_{ip_address}"), + ], + vars: { + controller_package_uri: releaseUri( + input, + "idea-virtual-desktop-controller-", + ), + }, + }; + } + + if (input.plan.contextParameter === "dcv_broker_bootstrap_package_uri") { + const name = `${moduleLogGroup}/dcv-broker`; + return { + baseOs: requiredString( + input.config, + `${input.moduleId}.dcv_broker.autoscaling.base_os`, + ), + instanceType: requiredString( + input.config, + `${input.moduleId}.dcv_broker.autoscaling.instance_type`, + ), + metricsNamespace: `${moduleMetrics}/dcv-broker`, + nodeType: "infra", + enableLogging: vdcLogging, + logFiles: [ + logFile( + "/var/log/dcv-session-manager-broker/**.log", + name, + "dcv-session-manager-broker_{ip_address}", + ), + logFile("/var/log/messages", name, "system_{ip_address}"), + logFile("/var/log/syslog", name, "syslog_{ip_address}"), + ], + vars: {}, + }; + } + + if (input.plan.contextParameter === "dcv_connection_gateway_package_uri") { + const name = `${moduleLogGroup}/dcv-connection-gateway`; + return { + baseOs: requiredString( + input.config, + `${input.moduleId}.dcv_connection_gateway.autoscaling.base_os`, + ), + instanceType: requiredString( + input.config, + `${input.moduleId}.dcv_connection_gateway.autoscaling.instance_type`, + ), + metricsNamespace: `${moduleMetrics}/dcv-connection-gateway`, + nodeType: "infra", + enableLogging: vdcLogging, + logFiles: [ + logFile( + "/var/log/dcv-connection-gateway/**.log", + name, + "dcv-connection-gateway_{ip_address}", + ), + logFile("/var/log/messages", name, "system_{ip_address}"), + logFile("/var/log/syslog", name, "syslog_{ip_address}"), + ], + vars: { + dcv_connection_gateway_package_uri: releaseUri( + input, + "idea-dcv-connection-gateway-", + ), + }, + }; + } + + throw new GeneralException( + `bootstrap context parameter is not defined for module ${input.moduleId}: ${input.plan.contextParameter}`, + ); +} + +/** Produces the default Linux metric sections emitted for module hosts. */ +function defaultCloudWatchMetrics( + interval: number, + baseOs: string, + nvidiaGpu: boolean, +): JsonObject { + const rootDevice = baseOs === "amazonlinux2023" ? "/dev/xvda" : "/dev/sda1"; + const metrics: JsonObject = { + cpu: { + resources: ["*"], + totalcpu: true, + metrics_collection_interval: interval, + measurement: [ + "time_active", + "time_idle", + "time_iowait", + "time_system", + "time_user", + "usage_active", + "usage_idle", + "usage_iowait", + "usage_system", + "usage_user", + ], + }, + disk: { + metrics_collection_interval: interval, + resources: [rootDevice], + measurement: [ + "free", + "total", + "used", + "used_percent", + "inodes_free", + "inodes_used", + "inodes_total", + ], + drop_device: true, + }, + diskio: { + metrics_collection_interval: interval, + resources: [rootDevice], + measurement: [ + "reads", + "writes", + "read_bytes", + "write_bytes", + "read_time", + "write_time", + "io_time", + "iops_in_progress", + ], + }, + swap: { + metrics_collection_interval: interval, + measurement: ["free", "used", "used_percent"], + }, + mem: { + metrics_collection_interval: interval, + measurement: [ + "active", + "available", + "available_percent", + "buffered", + "cached", + "free", + "inactive", + "total", + "used", + "used_percent", + ], + }, + net: { + metrics_collection_interval: interval, + resources: ["*"], + measurement: [ + "bytes_sent", + "bytes_recv", + "drop_in", + "drop_out", + "err_in", + "err_out", + "packets_sent", + "packets_recv", + ], + }, + netstat: { + metrics_collection_interval: interval, + measurement: [ + "tcp_close", + "tcp_close_wait", + "tcp_closing", + "tcp_established", + "tcp_fin_wait1", + "tcp_fin_wait2", + "tcp_last_ack", + "tcp_listen", + "tcp_none", + "tcp_syn_sent", + "tcp_syn_recv", + "tcp_time_wait", + "udp_socket", + ], + }, + processes: { + metrics_collection_interval: interval, + measurement: [ + "blocked", + "dead", + "idle", + "paging", + "running", + "sleeping", + "stopped", + "total", + "total_threads", + "wait", + "zombies", + ], + }, + }; + if (nvidiaGpu) { + metrics.nvidia_gpu = { + metrics_collection_interval: interval, + measurement: [ + "utilization_gpu", + "temperature_gpu", + "power_draw", + "utilization_memory", + "memory_total", + "memory_used", + "memory_free", + "pcie_link_gen_current", + "pcie_link_width_current", + "encoder_stats_session_count", + "encoder_stats_average_fps", + "encoder_stats_average_latency", + "clocks_current_graphics", + "clocks_current_sm", + "clocks_current_memory", + "clocks_current_video", + ], + }; + } + return metrics; +} + +/** Builds the agent configuration attached to every host bootstrap context. */ +function cloudWatchAgentConfig( + config: ClusterConfig, + role: HostRole, + moduleId: string, + nvidiaGpu: boolean, +): JsonObject { + const metricsEnabled = config.getString("metrics.provider") === "cloudwatch"; + const logsEnabled = + config.getBool("cluster.cloudwatch_logs.enabled", false) === true && + role.enableLogging; + const metricsInterval = config.getInt( + "metrics.cloudwatch.metrics_collection_interval", + 60, + ); + const agent: JsonObject = {}; + if (metricsEnabled && metricsInterval !== 0) { + agent.metrics_collection_interval = metricsInterval; + } + agent.region = requiredString(config, "cluster.aws.region"); + agent.logfile = + "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log"; + agent.debug = false; + agent.run_as_user = "root"; + + const result: JsonObject = { agent }; + const useEndpoints = + config.getBool("cluster.network.use_vpc_endpoints", false) === true; + if (logsEnabled) { + const logs: JsonObject = {}; + if ( + useEndpoints && + config.getBool( + "cluster.network.vpc_interface_endpoints.logs.enabled", + false, + ) === true + ) { + logs.endpoint_override = requiredString( + config, + "cluster.network.vpc_interface_endpoints.logs.endpoint_url", + ).replaceAll("https://", ""); + } + logs.log_stream_name = `${moduleId}_default_{ip_address}`; + logs.force_flush_interval = + config.getInt("cluster.cloudwatch_logs.force_flush_interval", 5) || 5; + const retention = config.getInt( + "cluster.cloudwatch_logs.retention_in_days", + 90, + ); + logs.logs_collected = { + files: { + collect_list: role.logFiles.map((file) => ({ + ...file, + retention_in_days: retention, + })), + }, + }; + result.logs = logs; + } + + if (metricsEnabled) { + const metrics: JsonObject = { + namespace: role.metricsNamespace, + }; + if ( + useEndpoints && + config.getBool( + "cluster.network.vpc_interface_endpoints.monitoring.enabled", + false, + ) === true + ) { + metrics.endpoint_override = requiredString( + config, + "cluster.network.vpc_interface_endpoints.monitoring.endpoint_url", + ).replaceAll("https://", ""); + } + metrics.force_flush_interval = + config.getInt("metrics.cloudwatch.force_flush_interval", 60) || 60; + metrics.metrics_collected = defaultCloudWatchMetrics( + metricsInterval || 60, + role.baseOs, + nvidiaGpu, + ); + result.metrics = metrics; + } + return result; +} + +/** Reads a required integer without weakening its return type. */ +function requiredInt(config: ClusterConfig, key: string): number { + const value = config.getInt(key, Number.NaN, { required: true }); + if (Number.isNaN(value)) { + throw new GeneralException(`${key} is required`); + } + return value; +} + +/** Reads a required non-empty string without weakening its return type. */ +function requiredString(config: ClusterConfig, key: string): string { + const value = config.getString(key, "", { required: true }); + if (isEmpty(value)) { + throw new GeneralException(`${key} is required`); + } + return value; +} + +/** Builds the Prometheus configuration attached for either supported provider. */ +function prometheusConfig( + config: ClusterConfig, + role: HostRole, + moduleId: string, +): JsonObject | undefined { + const provider = config.getString("metrics.provider"); + if (provider !== "prometheus" && provider !== "amazon_managed_prometheus") { + return undefined; + } + + const externalLabels: JsonObject = { + ...(config.getConfig("metrics.prometheus.external_labels") ?? {}), + }; + const namespace = role.metricsNamespace.split("/"); + externalLabels.cluster_name = namespace[0]; + externalLabels.module_id = namespace[1]; + if (namespace[2] !== undefined) externalLabels.component = namespace[2]; + + const configuredRemoteWrite = config.getConfig( + "metrics.prometheus.remote_write", + undefined, + { required: true }, + ); + if (configuredRemoteWrite === undefined) { + throw new GeneralException("metrics.prometheus.remote_write is required"); + } + const remoteWrite = { ...configuredRemoteWrite }; + if (provider === "amazon_managed_prometheus") { + remoteWrite.sigv4 = { + region: requiredString(config, "cluster.aws.region"), + }; + } + + const scrapeConfigs: JsonObject[] = [ + { + job_name: "node_exporter", + static_configs: [{ targets: ["localhost:9100"] }], + }, + ]; + if (role.nodeType === "app") { + const app: JsonObject = { + job_name: "app_exporter", + metrics_path: `${requiredString( + config, + `${moduleId}.server.api_context_path`, + )}/metrics`, + scheme: "http", + authorization: { + type: "Bearer", + credentials_file: "/root/metrics_api_token.txt", + }, + static_configs: [ + { + targets: [ + `localhost:${requiredInt( + config, + `${moduleId}.server.port`, + )}`, + ], + }, + ], + }; + if ( + config.getBool( + `${moduleId}.server.enable_tls`, + false, + ) === true + ) { + app.scheme = "https"; + app.tls_config = { insecure_skip_verify: true }; + } + scrapeConfigs.push(app); + } + + return { + global: { + scrape_interval: config.getString( + "metrics.prometheus.scrape_interval", + "60s", + ), + scrape_timeout: config.getString( + "metrics.prometheus.scrape_timeout", + "10s", + ), + external_labels: externalLabels, + }, + remote_write: [remoteWrite], + scrape_configs: scrapeConfigs, + }; +} + +/** Converts the configured `Key=...,Value=...` strings to API tag objects. */ +function customTags(config: ClusterConfig): Array<{ Key: string; Value: string }> { + const values: Record = {}; + for (const configured of config.getList( + "global-settings.custom_tags", + [], + )) { + if (typeof configured !== "string") { + throw new GeneralException("global-settings.custom_tags entries must be strings"); + } + const separator = configured.indexOf(","); + if (separator < 0) { + throw new GeneralException(`invalid custom tag: ${configured}`); + } + const keyToken = configured.slice(0, separator); + const valueToken = configured.slice(separator + 1); + const key = keyToken.split("Key=")[1]?.trim(); + const value = valueToken.split("Value=")[1]?.trim(); + if (key === undefined || value === undefined) { + throw new GeneralException(`invalid custom tag: ${configured}`); + } + if (key !== "" && value !== "") values[key] = value; + } + return Object.entries(values).map(([Key, Value]) => ({ Key, Value })); +} + +/** Returns the login user associated with the supported host operating systems. */ +function defaultSystemUser(baseOs: string): string { + if ( + [ + "amazonlinux2023", + "rhel8", + "rhel9", + "rhel10", + "rocky8", + "rocky9", + "rocky10", + ].includes(baseOs) + ) { + return "ec2-user"; + } + if (["ubuntu2204", "ubuntu2404", "ubuntu2604"].includes(baseOs)) { + return "ubuntu"; + } + throw new GeneralException(`unknown system user name for base_os: ${baseOs}`); +} + +/** Returns whether any configured shared storage uses the requested provider. */ +function hasStorageProvider(config: ClusterConfig, provider: string): boolean { + const storage = config.getConfig("shared-storage") ?? {}; + return Object.values(storage).some( + (entry) => isRecord(entry) && entry.provider === provider, + ); +} + +/** Applies the cluster and module scope rules used for module hosts. */ +function sharedStorageApplies( + moduleName: string, + storage: unknown, +): boolean { + if (!isRecord(storage)) return false; + const scope = Array.isArray(storage.scope) ? storage.scope : []; + if (scope.length === 0 || scope.includes("cluster")) return true; + if (scope.includes("module") && scope.includes("project")) return false; + if (scope.includes("project") && scope.includes("scheduler:queue-profile")) { + return false; + } + if (scope.includes("module")) { + const modules = Array.isArray(storage.modules) ? storage.modules : []; + return modules.length === 0 || modules.includes(moduleName); + } + return false; +} + +/** Returns whether the host's instance family has a configured public driver. */ +function isNvidiaGpu(config: ClusterConfig, instanceType: string): boolean { + const family = instanceType.split(".")[0]; + return Object.hasOwn( + config.getConfig( + "global-settings.gpu_settings.nvidia_public_driver_versions", + {}, + ) ?? {}, + family, + ); +} + +/** Creates the provider consumed by the deployment package publisher. */ +export function buildBootstrapContext(input: BootstrapContextInput): object { + for (const [name, value] of [ + ["module_name", input.moduleName], + ["module_id", input.moduleId], + ["module_set", input.moduleSet], + ["base_os", input.baseOs], + ["instance_type", input.instanceType], + ] as const) { + if (isEmpty(value)) throw new GeneralException(`${name} is required`); + } + const role = hostRole(input); + const clusterName = requiredString(input.config, "cluster.cluster_name"); + const clusterBucket = requiredString( + input.config, + "cluster.cluster_s3_bucket", + ); + const clusterHome = requiredString(input.config, "cluster.home_dir"); + const awsRegion = requiredString(input.config, "cluster.aws.region"); + const proxy = + input.config.getString("cluster.network.https_proxy", "") ?? ""; + const noProxy = + proxy === "" + ? "" + : (input.config.getString("cluster.network.no_proxy", "") ?? ""); + const nvidiaGpu = isNvidiaGpu(input.config, role.instanceType); + const cloudWatchConfig = cloudWatchAgentConfig( + input.config, + role, + input.moduleId, + nvidiaGpu, + ); + const initialPrometheusConfig = prometheusConfig( + input.config, + role, + input.moduleId, + ); + const exporters = + initialPrometheusConfig === undefined + ? [] + : role.nodeType === "app" + ? ["node_exporter", "app_exporter"] + : ["node_exporter"]; + + return { + config: configFacade(input.config), + base_os: role.baseOs, + instance_type: role.instanceType, + module_name: input.moduleName, + module_id: input.moduleId, + module_set: input.moduleSet, + module_version: ideaVersion(), + vars: { + ...role.vars, + cloudwatch_agent_config: cloudWatchConfig, + ...(initialPrometheusConfig === undefined + ? {} + : { + prometheus_config: initialPrometheusConfig, + prometheus_exporters: exporters, + }), + }, + utils: { + to_json: (value: unknown, ...args: unknown[]) => + JSON.stringify( + value, + undefined, + callArgument(args, 0, "indent") === true ? 2 : undefined, + ), + to_yaml: (value: unknown) => toYaml(value), + }, + cluster_name: clusterName, + cluster_s3_bucket: clusterBucket, + cluster_home_dir: clusterHome, + aws_region: awsRegion, + app_deploy_dir: "/opt/idea/app", + https_proxy: proxy, + no_proxy: noProxy, + default_system_user: defaultSystemUser(role.baseOs), + has_storage_provider: (provider: string) => + hasStorageProvider(input.config, provider), + job_has_storage_provider: () => false, + job_has_param: () => false, + eval_shared_storage_scope: (...args: unknown[]) => + sharedStorageApplies( + input.moduleName, + callArgument(args, 0, "shared_storage"), + ), + is_gpu_instance_type: () => { + const family = role.instanceType.split(".")[0]; + return input.config + .getList( + "global-settings.gpu_settings.instance_families", + [], + ) + .includes(family); + }, + is_nvidia_gpu: () => nvidiaGpu, + is_amd_gpu: () => { + const family = role.instanceType.split(".")[0]; + return ( + input.config + .getList( + "global-settings.gpu_settings.instance_families", + [], + ) + .includes(family) && !nvidiaGpu + ); + }, + fail_on_missing_gpu_driver: () => + input.config.getBool( + "global-settings.gpu_settings.fail_on_missing_driver", + true, + ), + get_nvidia_gpu_driver_version: () => { + const family = role.instanceType.split(".")[0]; + return requiredString( + input.config, + `global-settings.gpu_settings.nvidia_public_driver_versions.${family}`, + ); + }, + get_custom_aws_tags: () => customTags(input.config), + get_cloudwatch_agent_config: (...args: unknown[]) => { + const additional = callArgument(args, 0, "additional_log_files"); + if (!Array.isArray(additional) || additional.length === 0) { + return cloudWatchConfig; + } + const logs = cloudWatchConfig.logs; + if (!isRecord(logs)) return cloudWatchConfig; + const collected = logs.logs_collected; + if (!isRecord(collected) || !isRecord(collected.files)) { + return cloudWatchConfig; + } + const existing = collected.files.collect_list; + collected.files.collect_list = [ + ...(Array.isArray(existing) ? existing : []), + ...additional, + ]; + return cloudWatchConfig; + }, + is_metrics_provider_prometheus: () => + initialPrometheusConfig !== undefined, + get_prometheus_config: (...args: unknown[]) => { + if (initialPrometheusConfig === undefined) return undefined; + const additional = callArgument(args, 0, "additional_scrape_configs"); + if (!Array.isArray(additional) || additional.length === 0) { + return initialPrometheusConfig; + } + const existing = initialPrometheusConfig.scrape_configs; + initialPrometheusConfig.scrape_configs = [ + ...(Array.isArray(existing) ? existing : []), + ...additional, + ]; + return initialPrometheusConfig; + }, + is_prometheus_exporter_enabled: (name: string) => + exporters.includes(name), + }; +} diff --git a/source/idea/ideactl/src/cli/bootstrap-package.ts b/source/idea/ideactl/src/cli/bootstrap-package.ts new file mode 100644 index 00000000..100df56d --- /dev/null +++ b/source/idea/ideactl/src/cli/bootstrap-package.ts @@ -0,0 +1,473 @@ +/** + * Builds rendered bootstrap archives for EC2 UserData and uploads them to the + * cluster bucket. Deployment-id-qualified keys cause host replacement. + */ + +import { PutObjectCommand } from "@aws-sdk/client-s3"; +import { + chmodSync, + copyFileSync, + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + utimesSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join, relative } from "node:path"; +import { gzipSync } from "node:zlib"; + +import { jinjaEnv, renderTemplate } from "../config/jinja.ts"; + +export interface BootstrapPackageBuildOptions { + /** The root of the `idea-bootstrap` source tree. */ + sourceDirectory: string; + /** Archive and rendered-directory basename, excluding `.tar.gz`. */ + targetPackageBasename: string; + /** Component directories to include. This array is mutated when `common` is auto-added. */ + components: string[]; + /** The single template variable visible to the bootstrap source tree. */ + context: object; + /** Deployment directory. A system temporary directory is used when absent. */ + tmpDir?: string; + /** Re-render an existing rendered directory instead of archiving it again. */ + forceBuild?: boolean; + /** Skips implicit `common` inclusion for Windows. */ + baseOs?: string; + logger?: (message: string) => void; +} + +export interface BootstrapPackageUploadClient { + send(command: PutObjectCommand): Promise; +} + +export interface UploadBootstrapPackageOptions { + client: BootstrapPackageUploadClient; + clusterS3Bucket: string; + archiveFile: string; + logger?: (message: string) => void; +} + +export interface UploadReleasePackageOptions { + client: BootstrapPackageUploadClient; + clusterS3Bucket: string; + packageDistDir: string; + packageName: string; + upload?: boolean; + logger?: (message: string) => void; +} + +export interface BuildAndUploadBootstrapPackageOptions extends BootstrapPackageBuildOptions { + client: BootstrapPackageUploadClient; + clusterS3Bucket: string; + upload?: boolean; +} + +export interface BootstrapPackageUris { + bootstrapPackageUri?: string; + controllerBootstrapPackageUri?: string; + dcvBrokerBootstrapPackageUri?: string; + dcvConnectionGatewayPackageUri?: string; +} + +export interface BootstrapPackagePlan { + basename: string; + components: string[]; + contextParameter: string; +} + +interface TarEntry { + archiveName: string; + path: string; + isDirectory: boolean; +} + +interface TarHeaderValues { + name: string; + mode: number; + uid: number; + gid: number; + size: number; + mtime: number; + type: string; +} + +const TAR_BLOCK_SIZE = 512; + +/** Invalid bootstrap package input. */ +export class BootstrapPackageError extends Error {} + +/** Missing release archive. */ +export class PackageNotFoundError extends Error {} + +/** + * Returns names for each host role's rendered bootstrap tree. The caller + * supplies its module id because configuration can rename module ids. + */ +export function bootstrapPackageBasenames(moduleId: string, deploymentId: string): { + standard: string; + controller: string; + dcvBroker: string; + dcvConnectionGateway: string; +} { + return { + standard: `bootstrap-${moduleId}-${deploymentId}`, + controller: `bootstrap-${moduleId}-controller-${deploymentId}`, + dcvBroker: `bootstrap-${moduleId}-dcv-broker-${deploymentId}`, + dcvConnectionGateway: `bootstrap-${moduleId}-dcv-connection-gateway-${deploymentId}`, + }; +} + +/** Per-module build inputs used by `upload-packages` and deploy. The builder prepends `common` when needed. */ +export function bootstrapPackagePlans( + moduleName: string, + moduleId: string, + deploymentId: string, + directoryServiceProvider?: string, +): BootstrapPackagePlan[] { + const names = bootstrapPackageBasenames(moduleId, deploymentId); + if (moduleName === "directoryservice") { + // Only OpenLDAP creates a directoryservice host. The AD providers deploy the stack with no package. + return directoryServiceProvider === "openldap" + ? [{ basename: names.standard, components: ["common", "openldap-server"], contextParameter: "bootstrap_package_uri" }] + : []; + } + if (moduleName === "cluster-manager") { + return [{ basename: names.standard, components: ["cluster-manager"], contextParameter: "bootstrap_package_uri" }]; + } + if (moduleName === "scheduler") { + return [{ basename: names.standard, components: ["scheduler"], contextParameter: "bootstrap_package_uri" }]; + } + if (moduleName === "bastion-host") { + return [{ basename: names.standard, components: ["common", "bastion-host"], contextParameter: "bootstrap_package_uri" }]; + } + if (moduleName === "virtual-desktop-controller") { + return [ + { + basename: names.controller, + components: ["virtual-desktop-controller"], + contextParameter: "controller_bootstrap_package_uri", + }, + { + basename: names.dcvBroker, + components: ["dcv-broker"], + contextParameter: "dcv_broker_bootstrap_package_uri", + }, + { + basename: names.dcvConnectionGateway, + components: ["dcv-connection-gateway"], + contextParameter: "dcv_connection_gateway_package_uri", + }, + ]; + } + return []; +} + +/** Release archives uploaded by `upload-packages` for each module. */ +export function releasePackageNames(moduleName: string, releaseVersion: string): string[] { + const packageNameByModule: Record = { + "cluster-manager": ["idea-cluster-manager"], + scheduler: ["idea-scheduler"], + "virtual-desktop-controller": ["idea-virtual-desktop-controller", "idea-dcv-connection-gateway"], + }; + return (packageNameByModule[moduleName] ?? []).map((packageName) => `${packageName}-${releaseVersion}.tar.gz`); +} + +/** The S3 URI returned after a bootstrap archive is uploaded. */ +export function bootstrapPackageUri(clusterS3Bucket: string, archiveFile: string): string { + return `s3://${clusterS3Bucket}/idea/bootstrap/${basename(archiveFile)}`; +} + +/** Returns the release-package URI, whether the package is uploaded or not. */ +export function releasePackageUri(clusterS3Bucket: string, packageName: string): string { + return `s3://${clusterS3Bucket}/idea/releases/${packageName}`; +} + +/** Returns context arguments in insertion order. Host modules use one URI and eVDI uses three role-specific URIs. */ +export function bootstrapContextParameterArgs(uris: BootstrapPackageUris): string[] { + const values: Array<[string, string | undefined]> = [ + ["bootstrap_package_uri", uris.bootstrapPackageUri], + ["controller_bootstrap_package_uri", uris.controllerBootstrapPackageUri], + ["dcv_broker_bootstrap_package_uri", uris.dcvBrokerBootstrapPackageUri], + ["dcv_connection_gateway_package_uri", uris.dcvConnectionGatewayPackageUri], + ]; + return values + .filter((entry): entry is [string, string] => entry[1] !== undefined) + .map(([key, value]) => `-c ${key}=${value}`); +} + +/** + * Recursively copies a direct child directory, preserving source metadata while + * leaving the component directory as the newly created destination directory. + */ +function copyTree(source: string, target: string): void { + mkdirSync(target, { recursive: true }); + for (const entry of readdirSync(source, { withFileTypes: true })) { + const sourcePath = join(source, entry.name); + const targetPath = join(target, entry.name); + if (entry.isDirectory()) { + copyTree(sourcePath, targetPath); + const stats = statSync(sourcePath); + chmodSync(targetPath, stats.mode); + utimesSync(targetPath, stats.atime, stats.mtime); + continue; + } + copyFileSync(sourcePath, targetPath); + const stats = statSync(sourcePath); + chmodSync(targetPath, stats.mode); + utimesSync(targetPath, stats.atime, stats.mtime); + } +} + +/** Archives sorted directory entries depth first. */ +function tarEntries(root: string): TarEntry[] { + const entries: TarEntry[] = [{ archiveName: ".", path: root, isDirectory: true }]; + const visit = (directory: string): void => { + const children = readdirSync(directory, { withFileTypes: true }); + children.sort((left, right) => (left.name < right.name ? -1 : left.name > right.name ? 1 : 0)); + for (const entry of children) { + const path = join(directory, entry.name); + const archiveName = `./${relative(root, path)}`; + const isDirectory = entry.isDirectory(); + entries.push({ archiveName, path, isDirectory }); + if (isDirectory) visit(path); + } + }; + visit(root); + return entries; +} + +function writeString(target: Buffer, value: string, offset: number, length: number): void { + Buffer.from(value, "utf8").copy(target, offset, 0, length); +} + +function writeOctal(target: Buffer, value: number, offset: number, length: number): void { + const encoded = value.toString(8).padStart(length - 1, "0").slice(-(length - 1)); + writeString(target, `${encoded}\0`, offset, length); +} + +/** Writes the common ustar header shape used by Python's PAX writer. */ +function tarHeader(values: TarHeaderValues): Buffer { + const header = Buffer.alloc(TAR_BLOCK_SIZE); + writeString(header, values.name, 0, 100); + writeOctal(header, values.mode, 100, 8); + writeOctal(header, values.uid, 108, 8); + writeOctal(header, values.gid, 116, 8); + writeOctal(header, values.size, 124, 12); + writeOctal(header, values.mtime, 136, 12); + header.fill(0x20, 148, 156); + writeString(header, values.type, 156, 1); + writeString(header, "ustar\0", 257, 6); + writeString(header, "00", 263, 2); + const checksum = header.reduce((total, byte) => total + byte, 0); + writeString(header, `${checksum.toString(8).padStart(6, "0")}\0 `, 148, 8); + return header; +} + +/** + * Formats a POSIX extended-header record, including its byte length prefix. + */ +function paxRecord(key: string, value: string): string { + const body = `${key}=${value}\n`; + let length = Buffer.byteLength(body, "utf8") + 2; + while (true) { + const record = `${length} ${body}`; + const actualLength = Buffer.byteLength(record, "utf8"); + if (actualLength === length) return record; + length = actualLength; + } +} + +/** Matches Python's float formatting for archive modification times. */ +function paxMtime(mtimeMs: number): string { + const seconds = mtimeMs / 1000; + return Number.isInteger(seconds) ? `${seconds}.0` : String(seconds); +} + +/** Appends a tar member header, its contents, and required block padding. */ +function appendTarMember(blocks: Buffer[], header: Buffer, contents: Buffer): void { + blocks.push(header, contents); + const padding = (TAR_BLOCK_SIZE - (contents.length % TAR_BLOCK_SIZE)) % TAR_BLOCK_SIZE; + if (padding > 0) blocks.push(Buffer.alloc(padding)); +} + +/** + * Creates a gzip-compressed POSIX PAX archive compatible with Python's + * `shutil.make_archive(..., "gztar", ...)` member stream. + */ +function createTarGz(directory: string, archiveFile: string): void { + const blocks: Buffer[] = []; + for (const entry of tarEntries(directory)) { + const archiveName = entry.isDirectory ? `${entry.archiveName}/` : entry.archiveName; + const stats = lstatSync(entry.path); + const attributes = [ + ...(Buffer.byteLength(archiveName, "utf8") > 100 ? [paxRecord("path", archiveName)] : []), + paxRecord("mtime", paxMtime(stats.mtimeMs)), + ]; + const extendedHeaderContents = Buffer.from(attributes.join(""), "utf8"); + appendTarMember( + blocks, + tarHeader({ + name: "././@PaxHeader", + mode: 0, + uid: 0, + gid: 0, + size: extendedHeaderContents.length, + mtime: 0, + type: "x", + }), + extendedHeaderContents, + ); + appendTarMember( + blocks, + tarHeader({ + name: archiveName, + mode: stats.mode & 0o7777, + uid: stats.uid, + gid: stats.gid, + size: entry.isDirectory ? 0 : stats.size, + mtime: Math.floor(stats.mtimeMs / 1000), + type: entry.isDirectory ? "5" : "0", + }), + entry.isDirectory ? Buffer.alloc(0) : readFileSync(entry.path), + ); + } + blocks.push(Buffer.alloc(TAR_BLOCK_SIZE * 2)); + writeFileSync(archiveFile, gzipSync(Buffer.concat(blocks), { level: 9 })); +} + +/** + * Renders immediate `.jinja2` component files, copies every other immediate + * child, omits `_templates`, then archives the rendered directory with `.` as + * the tar root. + */ +export class BootstrapPackageBuilder { + private readonly options: BootstrapPackageBuildOptions; + + constructor(options: BootstrapPackageBuildOptions) { + if (options.components.length === 0) { + throw new BootstrapPackageError("components[] is required."); + } + if (!options.components.includes("common")) { + if (options.baseOs === undefined || !options.baseOs.toLowerCase().includes("windows")) { + options.components.unshift("common"); + } + } + this.options = options; + } + + build(): string { + const tmpDirectory = this.options.tmpDir ?? mkdtempSync(join(tmpdir(), "tmp")); + mkdirSync(tmpDirectory, { recursive: true }); + const targetDirectory = join(tmpDirectory, this.options.targetPackageBasename); + const archiveFile = `${targetDirectory}.tar.gz`; + + if (existsSync(targetDirectory)) { + if (this.options.forceBuild === true) { + this.log(`deleting existing directory: ${targetDirectory} ...`); + rmSync(targetDirectory, { recursive: true }); + } else { + this.log( + `found existing bootstrap directory: ${targetDirectory}. use force_build=True to rebuild the bootstrap package.`, + ); + if (existsSync(archiveFile)) rmSync(archiveFile); + createTarGz(targetDirectory, archiveFile); + return archiveFile; + } + } + + const environment = jinjaEnv(this.options.sourceDirectory); + for (const component of readdirSync(this.options.sourceDirectory, { withFileTypes: true })) { + if (component.name === "_templates" || !this.options.components.includes(component.name)) continue; + const sourceComponentDirectory = join(this.options.sourceDirectory, component.name); + const targetComponentDirectory = join(targetDirectory, component.name); + mkdirSync(targetComponentDirectory, { recursive: true }); + + for (const file of readdirSync(sourceComponentDirectory, { withFileTypes: true })) { + if (file.name === "_templates") continue; + const sourceFile = join(sourceComponentDirectory, file.name); + if (file.name.endsWith(".jinja2")) { + const targetFile = join(targetComponentDirectory, file.name.replace(".jinja2", "")); + const content = renderTemplate(environment, `${component.name}/${file.name}`, { + context: this.options.context, + }); + this.log(`rendered template: ${targetFile}`); + writeFileSync(targetFile, content); + } else if (file.isDirectory()) { + const targetFile = join(targetComponentDirectory, file.name); + this.log(`copied directory: ${targetFile}`); + copyTree(sourceFile, targetFile); + } else { + const targetFile = join(targetComponentDirectory, file.name); + this.log(`copied file: ${targetFile}`); + copyFileSync(sourceFile, targetFile); + const stats = statSync(sourceFile); + chmodSync(targetFile, stats.mode); + utimesSync(targetFile, stats.atime, stats.mtime); + } + } + } + + createTarGz(targetDirectory, archiveFile); + return archiveFile; + } + + private log(message: string): void { + this.options.logger?.(message); + } +} + +/** Upload a rendered bootstrap package to `idea/bootstrap/`, then return its S3 URI. */ +export async function uploadBootstrapPackage(options: UploadBootstrapPackageOptions): Promise { + const key = `idea/bootstrap/${basename(options.archiveFile)}`; + const uri = `s3://${options.clusterS3Bucket}/${key}`; + options.logger?.(`uploading bootstrap package ${uri} ...`); + await options.client.send( + new PutObjectCommand({ + Bucket: options.clusterS3Bucket, + Key: key, + Body: readFileSync(options.archiveFile), + }), + ); + return uri; +} + +/** Builds the archive. When `upload` is false, returns undefined. */ +export async function buildAndUploadBootstrapPackage( + options: BuildAndUploadBootstrapPackageOptions, +): Promise { + const archiveFile = new BootstrapPackageBuilder(options).build(); + if (options.upload === false) return undefined; + return uploadBootstrapPackage({ + client: options.client, + clusterS3Bucket: options.clusterS3Bucket, + archiveFile, + logger: options.logger, + }); +} + +/** Uploads a release package to `idea/releases/` after checking that it exists. */ +export async function uploadReleasePackage(options: UploadReleasePackageOptions): Promise { + const packageFile = join(options.packageDistDir, options.packageName); + if (!existsSync(packageFile)) { + throw new PackageNotFoundError(`package not found: ${packageFile}`); + } + const key = `idea/releases/${basename(packageFile)}`; + const uri = `s3://${options.clusterS3Bucket}/${key}`; + if (options.upload !== false) { + options.logger?.(`uploading release package: ${uri} ...`); + await options.client.send( + new PutObjectCommand({ + Bucket: options.clusterS3Bucket, + Key: key, + Body: readFileSync(packageFile), + }), + ); + } + return uri; +} diff --git a/source/idea/ideactl/src/cli/cdk-invoker.ts b/source/idea/ideactl/src/cli/cdk-invoker.ts new file mode 100644 index 00000000..b0ac436c --- /dev/null +++ b/source/idea/ideactl/src/cli/cdk-invoker.ts @@ -0,0 +1,884 @@ +/** + * Port of `app/cdk/cdk_invoker.py`: the argv the CDK CLI is spawned with for one module, plus the + * deploy-time change-set guard. + * + * Two things are worth knowing before changing anything here. + * + * 1. The argv is a contract. Python built a shell string and let the shell split it; this builds + * the tokens directly and spawns without a shell. Every flag, its order, and the per-module + * `--output cdk.out.` isolation are reproduced token for token, because a missing + * `-c bootstrap_package_uri` produces a stack that synthesizes and then boots hosts which + * cannot find their bootstrap package. + * + * 2. `deploy` never executes a change set it has not read. Every deploy runs + * `cdk deploy --method=prepare-change-set`, reads the change set with `DescribeChangeSet`, + * and refuses to execute when a change would replace or remove something that carries state. + * `--allow-replacement ` is the only override and every override is printed. + */ + +import { spawn as spawnProcess, spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { STATEFUL_TYPE_PREFIXES, isStatefulType } from '../cdk/stateful.ts'; +import { ClusterConfig, GeneralException, type ModuleInfo, type TableScanner } from '../config/cluster-config.ts'; +import type { ConfigEntry, ModuleSpec } from '../config/cluster-config-db.ts'; +import { + bootstrapPackagePlans, + bootstrapPackageUri, + buildAndUploadBootstrapPackage, + releasePackageNames, + releasePackageUri, + type BootstrapPackagePlan, +} from './bootstrap-package.ts'; +import { ideaVersion } from '../version.ts'; + +/** Modules the container stack runs as tasks, which therefore need no host packages. */ +const CONTAINER_SERVED_MODULES = ['cluster-manager', 'scheduler', 'virtual-desktop-controller']; + +// --------------------------------------------------------------------------------------------- +// host filesystem layout (`app_props.py`) +// --------------------------------------------------------------------------------------------- + +/** `~/.idea`, or `IDEA_USER_HOME`. */ +export function ideaUserHome(): string { + return process.env.IDEA_USER_HOME ?? join(homedir(), '.idea'); +} + +function ensureDir(path: string): string { + mkdirSync(path, { recursive: true }); + return path; +} + +export function clusterRegionDir(clusterName: string, awsRegion: string, create = true): string { + const dir = join(ideaUserHome(), 'clusters', clusterName, awsRegion); + return create ? ensureDir(dir) : dir; +} + +export function clusterConfigDir(clusterName: string, awsRegion: string, create = true): string { + const dir = join(clusterRegionDir(clusterName, awsRegion, create), 'config'); + return create ? ensureDir(dir) : dir; +} + +export function clusterCdkDir(clusterName: string, awsRegion: string): string { + return ensureDir(join(clusterRegionDir(clusterName, awsRegion), '_cdk')); +} + +export function clusterDeploymentsDir(clusterName: string, awsRegion: string): string { + return ensureDir(join(clusterRegionDir(clusterName, awsRegion), 'deployments')); +} + +export function valuesFilePath(clusterName: string, awsRegion: string): string { + return join(clusterRegionDir(clusterName, awsRegion), 'values.yml'); +} + +/** `ValuesDiff.get_values_file_s3_key()`. */ +export const VALUES_FILE_S3_KEY = 'values/values.yml'; + +/** `~/.idea/downloads`: where the image leaves the release archives. */ +export function downloadsDir(): string { + return ensureDir(join(ideaUserHome(), 'downloads')); +} + +/** + * The bundled CDK CLI. `IDEA_CDK_BIN` wins, then the CLI's own `node_modules`, then the image's + * `~/.idea/lib/idea-cdk` layout. + */ +export function cdkBin(): string { + // An explicit override is taken as given: the image and the tests both point at a path this + // process has no business second-guessing. + const override = process.env.IDEA_CDK_BIN; + if (override !== undefined && override !== '') return override; + const candidates = [ + fileURLToPath(new URL('../../node_modules/aws-cdk/bin/cdk', import.meta.url)), + fileURLToPath(new URL('../../../node_modules/aws-cdk/bin/cdk', import.meta.url)), + join(ideaUserHome(), 'lib', 'idea-cdk', 'node_modules', 'aws-cdk', 'bin', 'cdk'), + ]; + const found = candidates.find((candidate) => existsSync(candidate)); + if (found === undefined) { + throw new GeneralException( + `Unable to find cdk binary at: ${candidates.join(', ')}. Please ensure ideactl is installed correctly.`, + ); + } + return found; +} + +/** `cdk.json` is copied into the cluster's `_cdk` directory the first time it is needed. */ +export function setupClusterCdkDir(clusterName: string, awsRegion: string): string { + const cdkHome = clusterCdkDir(clusterName, awsRegion); + const cdkJson = join(cdkHome, 'cdk.json'); + if (!existsSync(cdkJson)) { + const template = [ + fileURLToPath(new URL('../../cdk.json', import.meta.url)), + fileURLToPath(new URL('../../../cdk.json', import.meta.url)), + ].find((candidate) => existsSync(candidate)); + if (template !== undefined) { + writeFileSync(cdkJson, readFileSync(template)); + } + } + return cdkHome; +} + +// --------------------------------------------------------------------------------------------- +// injected effects +// --------------------------------------------------------------------------------------------- + +/** One child process. Resolves with the exit code; it never throws on a non-zero exit. */ +export type Spawn = (argv: string[], options: { cwd: string; env: Record }) => Promise; + +/** Streams the CDK CLI's output to this process's stdio, as `exec_shell` does. */ +export const liveSpawn: Spawn = (argv, options) => + new Promise((resolve, reject) => { + const [command, ...args] = argv; + if (command === undefined) throw new GeneralException('empty argv'); + const child = spawnProcess(command, args, { cwd: options.cwd, env: options.env, stdio: 'inherit' }); + child.on('error', reject); + child.on('close', (code) => resolve(code ?? 1)); + }); + +/** One change in a `DescribeChangeSet` response, narrowed to the fields the guard reads. */ +export interface ResourceChange { + Action?: string; + LogicalResourceId?: string; + PhysicalResourceId?: string; + ResourceType?: string; + Replacement?: string; +} + +export interface ChangeSetDescription { + Status?: string; + StatusReason?: string; + ExecutionStatus?: string; + Changes?: Array<{ ResourceChange?: ResourceChange }>; + NextToken?: string; +} + +export interface StackDescription { + StackStatus?: string; + StackStatusReason?: string; + Outputs?: Array<{ OutputKey?: string; OutputValue?: string }>; +} + +/** + * The CloudFormation reads and the one write the guard needs. A functional interface rather than a + * client so the tests drive every branch from fixtures, in the style of `SynthReads`. + */ +export interface CloudFormationApi { + describeChangeSet(input: { StackName: string; ChangeSetName: string; NextToken?: string }): Promise; + executeChangeSet(input: { + StackName: string; + ChangeSetName: string; + /** False tells CloudFormation to restore the last stable state when execution fails. */ + DisableRollback: boolean; + }): Promise; + describeStack(stackName: string): Promise; +} + +/** The two S3 calls the CLI makes on its own behalf. */ +export interface S3Api { + putObject(input: { Bucket: string; Key: string; Body: Uint8Array | string }): Promise; + getObject(input: { Bucket: string; Key: string }): Promise; +} + +/** `ClusterConfigDb`, structurally, so a test can record writes without DynamoDB. */ +export interface ConfigWriter { + syncModulesInDb(modules: ModuleSpec[]): Promise; + syncClusterSettingsInDb(entries: ConfigEntry[], overwrite?: boolean): Promise; + setConfigEntry(key: string, value: unknown): Promise; + deleteConfigEntries(configKeyPrefix: string): Promise; +} + +export interface ConfigWriterOptions { + clusterName: string; + awsRegion: string; + awsProfile?: string; + dynamodbKmsKeyId?: string | null; + createDatabase?: boolean; +} + +export interface PromptChoice { + message: string; + /** Present for a three-way prompt; absent for a yes/no. */ + choices?: string[]; + default?: string | boolean; +} + +/** + * Everything the CLI does that is not a pure function. Tests build a partial and pass it in, so no + * command path needs credentials, a network, or a real CDK CLI. + */ +export interface Deps { + spawn: Spawn; + cfn: CloudFormationApi; + s3: S3Api; + /** `.cluster-settings` / `.modules` scanner. */ + scan: TableScanner; + configWriter(options: ConfigWriterOptions): Promise; + /** `sts:GetCallerIdentity`, only for the cluster-bucket name fallback. */ + accountId(): Promise; + /** Identity used by a command, resolved before the command touches its account. */ + callerIdentity?(options: { + awsRegion: string; + awsProfile?: string; + }): Promise<{ account: string; arn: string }>; + /** Read-only effective ECS account settings used by the ECS deployment prerequisite. */ + ecsAccountSettings?: { + listAccountSettings(input: { + awsRegion: string; + effectiveSettings: true; + name: string; + }): Promise>; + }; + /** HTTPS GET returning the status code, or 0 when the request failed. */ + httpStatus(url: string): Promise; + sleep(ms: number): Promise; + now(): number; + uuid(): string; + out(line: string): void; + err(line: string): void; + prompt(choice: PromptChoice): Promise; + /** + * The bootstrap template context for one host module. `BootstrapContext` has not been ported + * yet, so a deploy of a module that needs a rendered bootstrap package requires this hook. + */ + bootstrapContext?(input: BootstrapContextInput): object; + /** Root of the `idea-bootstrap` source tree; defaults to the packaged copy. */ + bootstrapSourceDir?: string; +} + +export interface BootstrapContextInput { + moduleName: string; + moduleId: string; + moduleSet: string; + baseOs: string; + instanceType: string; + plan: BootstrapPackagePlan; + /** Release archives already uploaded for this module, keyed by archive name. */ + releasePackageUris: Record; + config: ClusterConfig; +} + +// --------------------------------------------------------------------------------------------- +// change-set guard +// --------------------------------------------------------------------------------------------- + +/** + * How to re-invoke this tool as the synthesis app. Prefers the name on the path, because that is + * what runs inside the image and from the released artifact, and falls back to this process's own + * runtime and entry point so a checkout works without installing anything. + */ +function cdkAppInvocation(): string { + const onPath = spawnSync('sh', ['-c', 'command -v ideactl'], { encoding: 'utf8' }); + if (onPath.status === 0 && onPath.stdout.trim() !== '') return 'ideactl'; + const entry = process.argv[1]; + if (entry === undefined || entry === '') return 'ideactl'; + return `${JSON.stringify(process.execPath)} ${JSON.stringify(entry)}`; +} + +/** The change set `cdk deploy --method=prepare-change-set` leaves on the stack. */ +export const CDK_DEPLOY_CHANGE_SET_NAME = 'cdk-deploy-change-set'; + +/** + * The change-set guard's notion of stateful is the synthesis's notion of stateful: one list, in + * `src/cdk/stateful.ts`, so the layer that refuses to remove these and the layer that marks them + * Retain on update-replace cannot come to disagree. + */ +export { STATEFUL_TYPE_PREFIXES, isStatefulType }; + +export function isCustomResourceType(resourceType: string | undefined): boolean { + return resourceType !== undefined && resourceType.startsWith('Custom::'); +} + +export type RefusalClass = 'replacement' | 'custom-resource-remove' | 'stateful-remove'; + +export interface ChangeSetFinding { + logicalId: string; + resourceType: string; + action: string; + refusal: RefusalClass; + reason: string; +} + +export interface ChangeSetVerdict { + /** Findings that were not overridden. A non-empty list means the deploy is refused. */ + refusals: ChangeSetFinding[]; + /** Findings an allow entry let through. Every one of these is printed. */ + allowed: Array; + /** True when CloudFormation reports the change set holds no changes. */ + empty: boolean; +} + +/** + * The analytics dashboard target group carries a fresh uuid in its `Name` on every synth, so + * CloudFormation replaces it on every analytics deploy. Python did the same thing; refusing it + * would refuse every analytics deploy and teach operators to pass `--allow-replacement` blindly. + * It is scoped to that one logical ID, that one resource type, and it is printed like any other + * override. + */ +export function builtInAllowedReplacements(clusterName: string): Map { + const dashboardTargetGroup = `${clusterName.replace(/-/g, '')}dashboardtargetgroup`; + return new Map([[dashboardTargetGroup, 'AWS::ElasticLoadBalancingV2::TargetGroup']]); +} + +/** True when CloudFormation created the change set but found nothing to do. */ +export function isEmptyChangeSet(description: ChangeSetDescription): boolean { + if (description.Status !== 'FAILED') return false; + const reason = description.StatusReason ?? ''; + return /didn't contain changes|No updates are to be performed/i.test(reason); +} + +/** + * Classifies every change in a change set. Pure: the caller decides what to do with the verdict. + * + * `allowReplacement` holds the logical IDs given on the command line. `builtIn` holds the + * logical ID -> required resource type pairs from `builtInAllowedReplacements`. + * `allowReplacementOfType` holds the one resource type the replace verb was pointed at, against + * the component name the operator typed, so the allowance is printed in the operator's words. + */ +export function evaluateChangeSet( + description: ChangeSetDescription, + allowReplacement: readonly string[] = [], + builtIn: ReadonlyMap = new Map(), + allowReplacementOfType: ReadonlyMap = new Map(), +): ChangeSetVerdict { + const verdict: ChangeSetVerdict = { refusals: [], allowed: [], empty: isEmptyChangeSet(description) }; + const explicit = new Set(allowReplacement); + + for (const change of description.Changes ?? []) { + const resourceChange = change.ResourceChange; + if (resourceChange === undefined) continue; + const logicalId = resourceChange.LogicalResourceId ?? ''; + const resourceType = resourceChange.ResourceType ?? ''; + const action = resourceChange.Action ?? ''; + + const findings: ChangeSetFinding[] = []; + if (resourceChange.Replacement === 'True') { + findings.push({ + logicalId, + resourceType, + action, + refusal: 'replacement', + reason: `${action} of ${logicalId} (${resourceType}) replaces the resource`, + }); + } else if (resourceChange.Replacement === 'Conditional' && isStatefulType(resourceType)) { + // CloudFormation says Conditional when whether it replaces depends on values it will only + // know at execution time. On anything stateless that is noise. On a stateful resource it is + // a coin toss with the data on one side of it, so it is refused like a certain replacement. + findings.push({ + logicalId, + resourceType, + action, + refusal: 'replacement', + reason: `${action} of ${logicalId} (${resourceType}) may replace the resource; CloudFormation reports Replacement=Conditional`, + }); + } + if (action === 'Remove' && isCustomResourceType(resourceType)) { + findings.push({ + logicalId, + resourceType, + action, + refusal: 'custom-resource-remove', + reason: `Remove of custom resource ${logicalId} (${resourceType}) runs its Delete handler`, + }); + } + if (action === 'Remove' && isStatefulType(resourceType)) { + findings.push({ + logicalId, + resourceType, + action, + refusal: 'stateful-remove', + reason: `Remove of stateful resource ${logicalId} (${resourceType})`, + }); + } + + for (const finding of findings) { + const namedComponent = allowReplacementOfType.get(resourceType); + if (explicit.has(logicalId)) { + verdict.allowed.push({ ...finding, allowedBy: '--allow-replacement' }); + } else if (builtIn.get(logicalId) === resourceType) { + verdict.allowed.push({ ...finding, allowedBy: 'built-in allow list' }); + } else if (namedComponent !== undefined && finding.refusal === 'replacement') { + // Only the replacement class, and only the one resource type the operator named. A remove + // is a different intent and the replace verb never permits it. + verdict.allowed.push({ ...finding, allowedBy: `replace ${namedComponent}` }); + } else { + verdict.refusals.push(finding); + } + } + } + + return verdict; +} + +/** `errorcodes.CONFIG_ERROR`-shaped refusal, so `main` prints it red and exits non-zero. */ +export class ChangeSetRefused extends Error { + readonly verdict: ChangeSetVerdict; + constructor(message: string, verdict: ChangeSetVerdict) { + super(message); + this.name = 'ChangeSetRefused'; + this.verdict = verdict; + } +} + +/** Raised where Python raises `SystemExit(code)`. */ +export class ExitWithCode extends Error { + readonly code: number; + constructor(code: number, message = '') { + super(message); + this.name = 'ExitWithCode'; + this.code = code; + } +} + +const STACK_STATUS_OK = new Set(['CREATE_COMPLETE', 'UPDATE_COMPLETE', 'IMPORT_COMPLETE']); +const STACK_STATUS_IN_PROGRESS = /_IN_PROGRESS$/; + +// --------------------------------------------------------------------------------------------- +// the invoker +// --------------------------------------------------------------------------------------------- + +export interface CdkInvokerOptions { + clusterName: string; + awsRegion: string; + moduleId: string; + moduleSet: string; + awsProfile?: string; + deploymentId?: string; + terminationProtection?: boolean; + rollback?: boolean; + /** Logical IDs whose replacement or removal the operator has explicitly accepted. */ + allowReplacement?: readonly string[]; + /** Resource type -> the component name the replace verb was given. Empty for every other path. */ + allowReplacementOfType?: ReadonlyMap; + /** Poll interval while waiting for the executed change set. */ + pollIntervalMs?: number; + deps: Deps; + /** Set by `open()` from the modules table; `bootstrap` is its own name. */ + moduleName?: string; + config?: ClusterConfig; +} + +export class CdkInvoker { + readonly clusterName: string; + readonly awsRegion: string; + readonly moduleId: string; + readonly moduleName: string; + readonly moduleSet: string; + readonly awsProfile: string | undefined; + readonly deploymentId: string; + readonly terminationProtection: boolean; + readonly rollback: boolean; + readonly allowReplacement: readonly string[]; + readonly allowReplacementOfType: ReadonlyMap; + readonly deploymentDir: string; + readonly cdkHome: string; + private readonly pollIntervalMs: number; + private readonly deps: Deps; + private readonly config: ClusterConfig | undefined; + + constructor(options: CdkInvokerOptions) { + this.clusterName = options.clusterName; + this.awsRegion = options.awsRegion; + this.moduleId = options.moduleId; + this.moduleName = options.moduleName ?? options.moduleId; + this.moduleSet = options.moduleSet; + this.awsProfile = options.awsProfile; + this.deploymentId = options.deploymentId ?? options.deps.uuid(); + this.terminationProtection = options.terminationProtection ?? true; + this.rollback = options.rollback ?? true; + this.allowReplacement = options.allowReplacement ?? []; + this.allowReplacementOfType = options.allowReplacementOfType ?? new Map(); + this.pollIntervalMs = options.pollIntervalMs ?? 15_000; + this.deps = options.deps; + this.config = options.config; + this.deploymentDir = ensureDir(join(clusterDeploymentsDir(this.clusterName, this.awsRegion), this.deploymentId)); + this.cdkHome = setupClusterCdkDir(this.clusterName, this.awsRegion); + } + + /** Resolves the module name from the modules table, as `CdkInvoker.__init__` does. */ + static async open(options: CdkInvokerOptions): Promise { + if (options.moduleId === 'bootstrap') { + return new CdkInvoker({ ...options, moduleName: 'bootstrap' }); + } + const config = + options.config ?? + (await ClusterConfig.fromDynamoDb(options.clusterName, options.awsRegion, { + moduleSet: options.moduleSet, + scan: options.deps.scan, + })); + const moduleInfo = config.moduleInfoById(options.moduleId); + if (moduleInfo === undefined) { + throw new GeneralException(`module not found for module_id: ${options.moduleId}`); + } + return new CdkInvoker({ ...options, moduleName: moduleInfo.name, config }); + } + + get stackName(): string { + return `${this.clusterName}-${this.moduleId}`; + } + + /** `get_cdk_app_cmd`: the `--app` re-entry the CDK CLI runs to synthesize one stack. */ + getCdkAppCmd(): string { + const args = [ + '--cluster-name', + this.clusterName, + '--aws-region', + this.awsRegion, + '--module-id', + this.moduleId, + '--module-name', + this.moduleName, + '--deployment-id', + this.deploymentId, + '--termination-protection', + String(this.terminationProtection), + ]; + if (this.awsProfile !== undefined && this.awsProfile !== '') { + args.push('--aws-profile', this.awsProfile); + } + // The toolkit runs this as a shell command, so it has to name something the shell can find. + // Inside the image and from the released artifact that is the tool itself, on the path. From a + // checkout it is not, so fall back to running this same entry point with the same runtime. + return `${cdkAppInvocation()} cdk cdk-app ${args.join(' ')}`; + } + + /** + * `get_cdk_command`, tokenized. Python assembled a shell string and let the shell split it, so + * `--rollback true` and `-c key=value` are two tokens each here. + */ + getCdkCommand(name: string, params: readonly string[] = [], contextParams: Record = {}): string[] { + const argv = [cdkBin(), ...name.split(' '), ...params]; + if (name === 'deploy') argv.push('--rollback', String(this.rollback)); + if (this.awsProfile !== undefined && this.awsProfile !== '') argv.push('--profile', this.awsProfile); + for (const [key, value] of Object.entries(contextParams)) argv.push('-c', `${key}=${value}`); + // CDK CLI >= 2.1137 locks cdk.out during synth; scope it per module so parallel + // --optimize-deployment runs do not collide on the shared cdk.out in the cluster _cdk directory. + argv.push('--output', `cdk.out.${this.moduleId}`); + return argv; + } + + /** The `deploy` argv, including the two flags that force the change-set path. */ + getDeployArgv(contextParams: Record = {}): string[] { + return this.getCdkCommand( + 'deploy', + [ + '--app', + this.getCdkAppCmd(), + '--outputs-file', + this.outputsFile(), + '--require-approval', + 'never', + // The toolkit rejects the deprecated no-execute flag alongside a method, and this is the + // method that means create the change set without executing it. + '--method=prepare-change-set', + ], + contextParams, + ); + } + + outputsFile(): string { + return join(this.deploymentDir, `${this.moduleName}-outputs.json`); + } + + private env(): Record { + const env = { ...process.env }; + // Keep the CDK's nodejs credential chain on the profile and region given on the command line. + if (this.awsProfile !== undefined && this.awsProfile !== '') { + env.AWS_PROFILE = this.awsProfile; + env.AWS_DEFAULT_PROFILE = this.awsProfile; + } + if (this.awsRegion !== '') env.AWS_DEFAULT_REGION = this.awsRegion; + return env; + } + + /** `exec_shell`: run it in the cluster `_cdk` directory, non-zero exit becomes `SystemExit`. */ + async execCdk(argv: string[]): Promise { + this.deps.out(`shell> ${argv.join(' ')}`); + const code = await this.deps.spawn(argv, { cwd: this.cdkHome, env: this.env() }); + if (code !== 0) throw new ExitWithCode(code); + } + + async cdkSynth(): Promise { + await this.execCdk(this.getCdkCommand('synth', ['--app', this.getCdkAppCmd()])); + } + + async cdkDiff(): Promise { + await this.execCdk(this.getCdkCommand('diff', ['--app', this.getCdkAppCmd()])); + } + + /** Every page of `DescribeChangeSet`, so a large change set is fully inspected. */ + private async describeChangeSetFully(): Promise { + const first = await this.deps.cfn.describeChangeSet({ + StackName: this.stackName, + ChangeSetName: CDK_DEPLOY_CHANGE_SET_NAME, + }); + const changes = [...(first.Changes ?? [])]; + let nextToken = first.NextToken; + while (nextToken !== undefined && nextToken !== '') { + const page = await this.deps.cfn.describeChangeSet({ + StackName: this.stackName, + ChangeSetName: CDK_DEPLOY_CHANGE_SET_NAME, + NextToken: nextToken, + }); + changes.push(...(page.Changes ?? [])); + nextToken = page.NextToken; + } + return { ...first, Changes: changes, NextToken: undefined }; + } + + /** + * Creates the change set, reads it, and executes it only when nothing stateful is replaced or + * removed. This is the artifact that stops a logical-ID mistake from destroying a cluster's + * identity store; the documentation around it is the explanation, this is the enforcement. + */ + async deployThroughChangeSet(contextParams: Record = {}): Promise { + await this.execCdk(this.getDeployArgv(contextParams)); + + const description = await this.describeChangeSetFully(); + const verdict = evaluateChangeSet( + description, + this.allowReplacement, + builtInAllowedReplacements(this.clusterName), + this.allowReplacementOfType, + ); + + for (const allowed of verdict.allowed) { + this.deps.out(`change-set guard: ALLOWED by ${allowed.allowedBy}: ${allowed.reason}`); + } + + if (verdict.refusals.length > 0) { + this.deps.err(`change-set guard: REFUSING to execute change set for stack ${this.stackName}`); + for (const refusal of verdict.refusals) { + this.deps.err(` [${refusal.refusal}] ${refusal.reason}`); + } + this.deps.err( + 'no change was applied. review the change set, then re-run with ' + + `--allow-replacement ${verdict.refusals.map((refusal) => refusal.logicalId).join(' --allow-replacement ')}` + + ' for each resource you have decided to lose.', + ); + throw new ChangeSetRefused( + `change-set guard refused ${verdict.refusals.length} change(s) on stack ${this.stackName}`, + verdict, + ); + } + + if (verdict.empty) { + this.deps.out(`${this.stackName}: no changes`); + return verdict; + } + + this.deps.out(`change-set guard: ${(description.Changes ?? []).length} change(s) accepted, executing`); + await this.deps.cfn.executeChangeSet({ + StackName: this.stackName, + ChangeSetName: CDK_DEPLOY_CHANGE_SET_NAME, + DisableRollback: !this.rollback, + }); + const stack = await this.waitForStack(); + this.writeOutputsFile(stack); + return verdict; + } + + private async waitForStack(): Promise { + for (;;) { + const stack = await this.deps.cfn.describeStack(this.stackName); + const status = stack.StackStatus ?? ''; + if (STACK_STATUS_IN_PROGRESS.test(status)) { + await this.deps.sleep(this.pollIntervalMs); + continue; + } + if (!STACK_STATUS_OK.has(status)) { + const reason = stack.StackStatusReason ?? ''; + throw new ExitWithCode( + 1, + `Stack ${this.stackName} ended ${reason === '' ? status : `${status}: ${reason}`}. No further modules were deployed. Open the stack events in CloudFormation, fix the failing resource, then re-run the same deploy.`, + ); + } + return stack; + } + } + + /** The `--outputs-file` shape the CDK CLI writes; preparing without executing leaves it to us. */ + private writeOutputsFile(stack: StackDescription): void { + const outputs: Record = {}; + for (const output of stack.Outputs ?? []) { + if (output.OutputKey !== undefined) outputs[output.OutputKey] = output.OutputValue ?? ''; + } + writeFileSync(this.outputsFile(), `${JSON.stringify({ [this.stackName]: outputs }, null, 2)}\n`); + } + + // ------------------------------------------------------------------------------------------- + // per-module invocation (`MODULE_MAPPING_INVOKE_MAPPING`) + // ------------------------------------------------------------------------------------------- + + private async clusterConfig(): Promise { + if (this.config !== undefined) { + if (this.config.currentModuleId !== this.moduleId) this.config.setModuleId(this.moduleId); + return this.config; + } + const config = await ClusterConfig.fromDynamoDb(this.clusterName, this.awsRegion, { + moduleSet: this.moduleSet, + moduleId: this.moduleId, + scan: this.deps.scan, + }); + return config; + } + + /** `directoryservice` and `bastion-host` refuse to deploy before their prerequisite module. */ + private assertPrerequisiteDeployed(modules: ModuleInfo[], prerequisiteName: string): void { + for (const module of modules) { + if (module.name === prerequisiteName && module.status === 'not-deployed') { + throw new GeneralException( + `cannot deploy ${this.moduleId}. module: ${module.module_id} is not yet deployed.`, + ); + } + } + } + + /** + * Builds and uploads the packages a host module needs, then returns the context params for the + * deploy. Modules with no host have neither, and get an empty object. + */ + private async publishPackages(config: ClusterConfig, forceBuildBootstrap: boolean): Promise> { + if (this.runsAsContainerTasks(config)) return {}; + const provider = this.moduleName === 'directoryservice' ? config.getString('directoryservice.provider') : undefined; + const plans = bootstrapPackagePlans(this.moduleName, this.moduleId, this.deploymentId, provider); + const releaseNames = releasePackageNames(this.moduleName, ideaVersion()); + if (plans.length === 0 && releaseNames.length === 0) return {}; + + const clusterS3Bucket = config.getString('cluster.cluster_s3_bucket', undefined, { required: true }) as string; + + const releasePackageUris: Record = {}; + for (const packageName of releaseNames) { + const file = join(downloadsDir(), packageName); + if (!existsSync(file)) throw new GeneralException(`package not found: ${file}`); + const uri = releasePackageUri(clusterS3Bucket, packageName); + this.deps.out(`uploading release package: ${uri} ...`); + await this.deps.s3.putObject({ + Bucket: clusterS3Bucket, + Key: `idea/releases/${packageName}`, + Body: readFileSync(file), + }); + releasePackageUris[packageName] = uri; + } + + const contextParams: Record = {}; + for (const plan of plans) { + const uri = await this.publishBootstrapPackage(config, plan, releasePackageUris, clusterS3Bucket, forceBuildBootstrap); + contextParams[plan.contextParameter] = uri; + } + return contextParams; + } + + /** + * Whether this module's processes run as container tasks rather than on hosts. The image + * carries their packages, so nothing downloads a bootstrap archive or a release archive. The + * condition is the one the module stacks build their host resources from, so a run that retains + * hosts still publishes what those hosts read. + */ + private runsAsContainerTasks(config: ClusterConfig): boolean { + if (!CONTAINER_SERVED_MODULES.includes(this.moduleName)) return false; + if (!config.getBool('ecs.enabled', false)) return false; + return !config.getBool('ecs.retain_existing_hosts', false); + } + + private async publishBootstrapPackage( + config: ClusterConfig, + plan: BootstrapPackagePlan, + releasePackageUris: Record, + clusterS3Bucket: string, + forceBuild: boolean, + ): Promise { + const buildContext = this.deps.bootstrapContext; + if (buildContext === undefined) { + // Guessing the URI without uploading the archive would hand the host a package that does + // not exist, so this is a hard stop rather than a warning. + throw new GeneralException( + `Cannot build the bootstrap package for module ${this.moduleId} on this cluster. This build cannot deploy host modules that need a bootstrap archive. Deploy stack-only modules, or use a release that includes bootstrap-package support.`, + ); + } + const baseOs = config.getString(this.baseOsKey(), undefined, { required: true }) as string; + const instanceType = config.getString(this.instanceTypeKey(), undefined, { required: true }) as string; + const context = buildContext({ + moduleName: this.moduleName, + moduleId: this.moduleId, + moduleSet: this.moduleSet, + baseOs, + instanceType, + plan, + releasePackageUris, + config, + }); + const uri = await buildAndUploadBootstrapPackage({ + sourceDirectory: this.deps.bootstrapSourceDir ?? bootstrapSourceDir(), + targetPackageBasename: plan.basename, + components: [...plan.components], + context, + tmpDir: this.deploymentDir, + forceBuild, + baseOs, + client: { send: (command) => this.deps.s3.putObject(command.input as { Bucket: string; Key: string; Body: Uint8Array }) }, + clusterS3Bucket, + logger: (message) => this.deps.out(message), + }); + return uri ?? bootstrapPackageUri(clusterS3Bucket, `${plan.basename}.tar.gz`); + } + + /** `AMI_UPDATE_KEYS`-shaped: where each module keeps the base OS of its host. */ + private baseOsKey(): string { + if (this.moduleName === 'cluster-manager') return `${this.moduleId}.ec2.autoscaling.base_os`; + if (this.moduleName === 'virtual-desktop-controller') return `${this.moduleId}.controller.autoscaling.base_os`; + return `${this.moduleId}.base_os`; + } + + private instanceTypeKey(): string { + if (this.moduleName === 'cluster-manager') return `${this.moduleId}.ec2.autoscaling.instance_type`; + if (this.moduleName === 'virtual-desktop-controller') return `${this.moduleId}.controller.autoscaling.instance_type`; + return `${this.moduleId}.instance_type`; + } + + /** `CdkInvoker.invoke`: a module name outside the mapping is a no-op, exactly as in Python. */ + async invoke(options: { forceBuildBootstrap?: boolean } = {}): Promise { + if (!DEPLOYABLE_MODULE_NAMES.has(this.moduleName)) { + this.deps.out(`module name not found: ${this.moduleName}`); + return; + } + const config = await this.clusterConfig(); + if (this.moduleName === 'directoryservice') this.assertPrerequisiteDeployed(config.modules(), 'cluster'); + if (this.moduleName === 'bastion-host') this.assertPrerequisiteDeployed(config.modules(), 'scheduler'); + + const contextParams = await this.publishPackages(config, options.forceBuildBootstrap === true); + await this.deployThroughChangeSet(contextParams); + } +} + +/** `MODULE_MAPPING_INVOKE_MAPPING` keys. */ +export const DEPLOYABLE_MODULE_NAMES: ReadonlySet = new Set([ + 'cluster', + 'shared-storage', + 'identity-provider', + 'directoryservice', + 'cluster-manager', + 'scheduler', + 'bastion-host', + 'virtual-desktop-controller', + 'analytics', + 'metrics', + // The container control plane deploys at priority 4.5, between shared storage and the + // cluster manager. Without this entry an all-module run prints that it cannot find the + // module and returns, silently updating every stack except this one. + 'ecs', +]); + +/** The packaged `idea-bootstrap` tree. */ +export function bootstrapSourceDir(): string { + const candidates = [ + fileURLToPath(new URL('../../resources/bootstrap', import.meta.url)), + fileURLToPath(new URL('../../../idea-bootstrap', import.meta.url)), + ]; + const found = candidates.find((candidate) => existsSync(candidate)); + if (found === undefined) throw new GeneralException(`bootstrap source tree not found: ${candidates.join(', ')}`); + return found; +} + diff --git a/source/idea/ideactl/src/cli/commands/bootstrap.ts b/source/idea/ideactl/src/cli/commands/bootstrap.ts new file mode 100644 index 00000000..94d5f574 --- /dev/null +++ b/source/idea/ideactl/src/cli/commands/bootstrap.ts @@ -0,0 +1,113 @@ +/** + * The `cdk bootstrap` invocation built by `ideactl bootstrap`. + * + * This performs pure argv construction. The caller supplies resolved bucket, DNS, + * ELB account, tags, and template values. It makes no child-process, AWS, or + * cluster-config calls. + */ + +/** + * Converts `Key=k,Value=v` strings while preserving insertion order, including + * integer-like and `__proto__` keys. + */ +export function customTagsToKeyValuePairs(customTags: readonly string[]): Map { + const result = new Map(); + for (const customTag of customTags) { + const separator = customTag.indexOf(","); + const tokens = separator === -1 + ? [customTag] + : [customTag.slice(0, separator), customTag.slice(separator + 1)]; + const key = indexOne(tokens[0].split("Key="), customTag).trim(); + const value = indexOne(indexOne(tokens, customTag).split("Value="), customTag).trim(); + if (key === "" || value === "") continue; + result.set(key, value); + } + return result; +} + +/** Returns item one, including the malformed-input exception. */ +function indexOne(values: readonly string[], customTag: string): string { + const value = values[1]; + if (value === undefined) { + const error = new RangeError( + `Custom tag ${customTag} is not in Key=k,Value=v form. Fix the tag and re-run bootstrap.`, + ); + error.name = "IndexError"; + throw error; + } + return value; +} + +/** `constants.IDEA_TAG_CLUSTER_NAME`. */ +export const IDEA_TAG_CLUSTER_NAME = "idea:ClusterName"; + +/** Adds the cluster tag after custom tags so it wins a key collision. */ +export function bootstrapTags(clusterName: string, customTags: readonly string[]): Map { + const tags = customTagsToKeyValuePairs(customTags); + tags.set(IDEA_TAG_CLUSTER_NAME, clusterName); + return tags; +} + +export interface BootstrapArgvInput { + /** The binary to execute. */ + cdkBin: string; + /** `CdkInvoker.get_cdk_app_cmd()` output. */ + cdkAppCmd: string; + clusterName: string; + clusterBucket: string; + /** Default: `true`. */ + terminationProtection: boolean; + /** Cluster qualifier. */ + qualifier: string; + templatePath: string; + /** Empty values omit this flag. */ + customPermissionsBoundary?: string; + /** Empty values omit this flag. */ + cloudformationExecutionPolicies?: string; + /** Default: `true`. This flag is always emitted. */ + publicAccessBlockConfiguration?: boolean; + /** Ordered bootstrap tags, including arbitrary custom-tag keys. */ + tags: ReadonlyMap; + awsProfile?: string; +} + +/** `f'{cluster_name}-bootstrap'` (`cdk_invoker.py:510`). */ +export function bootstrapStackName(clusterName: string): string { + return `${clusterName}-bootstrap`; +} + +/** Returns whether an optional string is non-empty. */ +function isNotEmptyString(value: string | undefined): value is string { + return value !== undefined && value.trim().length > 0; +} + +/** + * Returns tokenized `cdk bootstrap` arguments. Flag/value pairs are separate + * tokens for direct execution. + */ +export function buildBootstrapArgv(input: BootstrapArgvInput): string[] { + const argv: string[] = [input.cdkBin, "bootstrap"]; + argv.push("--app", input.cdkAppCmd); + argv.push("--bootstrap-bucket-name", input.clusterBucket); + argv.push("--toolkit-stack-name", bootstrapStackName(input.clusterName)); + argv.push("--termination-protection", String(input.terminationProtection)); + argv.push("--qualifier", input.qualifier); + argv.push("--template", input.templatePath); + + if (isNotEmptyString(input.customPermissionsBoundary)) { + argv.push("--custom-permissions-boundary", input.customPermissionsBoundary); + } + if (isNotEmptyString(input.cloudformationExecutionPolicies)) { + argv.push("--cloudformation-execution-policies", input.cloudformationExecutionPolicies); + } + // Boolean values always emit this flag. + argv.push("--public-access-block-configuration", String(input.publicAccessBlockConfiguration ?? true)); + + for (const [key, value] of input.tags) { + argv.push("--tags", `${key}=${value}`); + } + if (isNotEmptyString(input.awsProfile)) { + argv.push("--profile", input.awsProfile); + } + return argv; +} diff --git a/source/idea/ideactl/src/cli/commands/cdk.ts b/source/idea/ideactl/src/cli/commands/cdk.ts new file mode 100644 index 00000000..05ab8b2c --- /dev/null +++ b/source/idea/ideactl/src/cli/commands/cdk.ts @@ -0,0 +1,102 @@ +/** + * Port of the `cdk` sub-group (`app_main.py:1136-1226`). + * + * `synth` and `diff` drive the CDK CLI for one module. `cdk-app` is the `--app` re-entry the CDK + * CLI itself runs: it builds exactly one stack and synthesizes it. It is not a command for humans. + */ + +import type { Command } from 'commander'; + +import { CdkInvoker, type Deps } from '../cdk-invoker.ts'; + +interface CdkModuleOptions { + clusterName: string; + awsRegion: string; + awsProfile?: string; + deploymentId?: string; + moduleSet: string; +} + +export interface CdkAppCommandOptions { + clusterName: string; + awsRegion: string; + awsProfile?: string; + moduleName: string; + moduleId: string; + deploymentId?: string; + terminationProtection?: string; + configFile?: string; + synthReads?: string; +} + +/** Rebuilds the argv `src/cdk/app.ts` parses, so `--help` stays the command's own contract. */ +export function cdkAppArgv(options: CdkAppCommandOptions): string[] { + const argv = [ + '--cluster-name', + options.clusterName, + '--aws-region', + options.awsRegion, + '--module-id', + options.moduleId, + '--module-name', + options.moduleName, + ]; + if (options.deploymentId !== undefined) argv.push('--deployment-id', options.deploymentId); + argv.push('--termination-protection', options.terminationProtection ?? 'true'); + if (options.awsProfile !== undefined) argv.push('--aws-profile', options.awsProfile); + if (options.configFile !== undefined) argv.push('--config-file', options.configFile); + if (options.synthReads !== undefined) argv.push('--synth-reads', options.synthReads); + return argv; +} + +export function registerCdkCommands(program: Command, deps: Deps): Command { + const cdk = program.command('cdk').description('cdk options'); + + const moduleCommand = (name: string, description: string): Command => + cdk + .command(name) + .description(description) + .requiredOption('--cluster-name ', 'Cluster Name') + .requiredOption('--aws-region ', 'AWS Region') + .option('--aws-profile ', 'AWS Profile Name') + .option('--deployment-id ', 'A UUID to identify the deployment.') + .option('--module-set ', 'Name of the ModuleSet. Default: default', 'default') + .argument('', 'module id'); + + moduleCommand('synth', 'synthesize cloudformation template for a module').action( + async (module: string, options: CdkModuleOptions) => { + const invoker = await CdkInvoker.open({ ...options, moduleId: module, deps }); + await invoker.cdkSynth(); + }, + ); + + moduleCommand('diff', 'compares the specified module with the deployed module').action( + async (module: string, options: CdkModuleOptions) => { + const invoker = await CdkInvoker.open({ ...options, moduleId: module, deps }); + await invoker.cdkDiff(); + }, + ); + + cdk + .command('cdk-app') + .description('cdk app') + .requiredOption('--cluster-name ', 'Cluster Name') + .option('--aws-profile ', 'AWS Profile Name') + .requiredOption('--aws-region ', 'AWS Region') + .requiredOption('--module-name ', 'module name') + .requiredOption('--module-id ', 'module id') + .option('--deployment-id ', 'A UUID to identify the deployment.') + .option( + '--termination-protection ', + 'Toggle termination protection for the cloud formation stack. Default: true', + 'true', + ) + .option('--config-file ', 'Replay cluster settings from a table dump instead of DynamoDB.') + .option('--synth-reads ', 'Replay the synth-time AWS reads from a file.') + .action(async (options: CdkAppCommandOptions) => { + const { main } = await import('../../cdk/app.ts'); + await main(cdkAppArgv(options)); + }); + + return cdk; +} diff --git a/source/idea/ideactl/src/cli/commands/config.ts b/source/idea/ideactl/src/cli/commands/config.ts new file mode 100644 index 00000000..ec392f9d --- /dev/null +++ b/source/idea/ideactl/src/cli/commands/config.ts @@ -0,0 +1,751 @@ +/** + * Port of the `config` command group (`app_main.py:158-978`). + * + * `generate` renders the local `config/` tree from `values.yml`; `update` pushes it into the two + * DynamoDB tables; the rest read, write or export single entries. Nothing here talks to AWS + * directly: every effect arrives through `Deps`. + */ + +import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import yaml from 'js-yaml'; +import { Command, Option } from 'commander'; + +import { + checkAndConvertDecimalValue, + ClusterConfigError, + GeneralException, + isEmpty, + type ModuleInfo, +} from '../../config/cluster-config.ts'; +import { convertConfigToKeyValuePairs, generateConfigFromTemplates, readConfigFromFiles, readModulesFromFiles, type ConfigEntry } from '../../config/generator.ts'; +import type { UpgradeDriftInput, UpgradeDriftReport } from "../../config/upgrade-drift.ts"; +import { loadValuesFile } from '../../config/values.ts'; +import { + collectInstallerValues, + type InstallerChoiceProvider, + type InstallerIdentity, +} from '../installer-params.ts'; +import { TerminalPromptDriver, type PromptDriver } from '../prompts.ts'; +import { + clusterConfigDir, + clusterRegionDir, + ExitWithCode, + valuesFilePath, + VALUES_FILE_S3_KEY, + type Deps, +} from '../cdk-invoker.ts'; + +/** One `.cluster-settings` row as the CLI prints it. */ +export interface SettingsRow { + key: string; + value?: unknown; + version?: number; + source?: string; +} + +// --------------------------------------------------------------------------------------------- +// small shared helpers +// --------------------------------------------------------------------------------------------- + +/** `PrettyTable` with `align = 'l'`: a fixed-width ASCII table, no dependency needed. */ +export function renderTable(headers: readonly string[], rows: ReadonlyArray): string { + const widths = headers.map((header, index) => + Math.max(header.length, ...rows.map((row) => (row[index] ?? '').split('\n')[0]?.length ?? 0)), + ); + const line = `+${widths.map((width) => '-'.repeat(width + 2)).join('+')}+`; + const format = (cells: readonly string[]): string => + `|${cells.map((cell, index) => ` ${(cell ?? '').padEnd(widths[index] ?? 0)} `).join('|')}|`; + return [line, format(headers), line, ...rows.map(format), line].join('\n'); +} + +/** `Utils.get_value_as_string(..., '-')`: Python's `str()` for the shapes cluster settings hold. */ +export function pyStr(value: unknown, defaultValue = '-'): string { + if (value === undefined || value === null) return defaultValue; + if (typeof value === 'string') return value; + if (typeof value === 'boolean') return value ? 'True' : 'False'; + if (Array.isArray(value)) return `[${value.map((item) => (typeof item === 'string' ? `'${item}'` : pyStr(item, 'None'))).join(', ')}]`; + if (typeof value === 'object') return JSON.stringify(value); + return String(value); +} + +/** `Utils.to_yaml`: `yaml.dump(sort_keys=False, width=140)`. */ +export function toYaml(value: unknown): string { + return yaml.dump(value, { sortKeys: false, lineWidth: 140, noRefs: true }); +} + +/** Nests flat `a.b.c` keys, longest-last so `a.b.c` can replace a scalar written at `a.b`. */ +export function buildTree(entries: ReadonlyArray<{ key: string; value?: unknown }>): Record { + const tree: Record = {}; + const sorted = [...entries].sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0)); + for (const entry of sorted) { + const parts = entry.key.split('.'); + let node = tree; + for (const part of parts.slice(0, -1)) { + const child = node[part]; + if (typeof child !== 'object' || child === null || Array.isArray(child)) node[part] = {}; + node = node[part] as Record; + } + const leaf = parts[parts.length - 1]; + if (leaf !== undefined) node[leaf] = entry.value ?? null; + } + return tree; +} + +/** + * `ClusterConfigDB.get_config_entries`: a full scan, `re.match` (start-anchored) against `query`, + * sorted by key. `version` survives, which is why this does not go through `ClusterConfig`. + */ +export async function scanSettings(deps: Deps, clusterName: string, query?: string): Promise { + let pattern: RegExp | undefined; + if (!isEmpty(query)) { + try { + pattern = new RegExp(query as string); + } catch (error) { + throw new ClusterConfigError(`invalid search regex: ${query} - ${(error as Error).message}`); + } + } + const rows: SettingsRow[] = []; + let startKey: Record | undefined; + do { + const page = await deps.scan({ TableName: `${clusterName}.cluster-settings`, ExclusiveStartKey: startKey }); + for (const item of page.Items ?? []) { + const key = item['key']; + if (typeof key !== 'string') continue; + if (pattern !== undefined && pattern.exec(key)?.index !== 0) continue; + rows.push({ + key, + value: checkAndConvertDecimalValue(item['value']), + version: typeof item['version'] === 'number' ? item['version'] : 0, + source: typeof item['source'] === 'string' ? item['source'] : undefined, + }); + } + startKey = page.LastEvaluatedKey; + } while (startKey !== undefined); + rows.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0)); + return rows; +} + +export async function scanModules(deps: Deps, clusterName: string): Promise { + const modules: ModuleInfo[] = []; + let startKey: Record | undefined; + do { + const page = await deps.scan({ TableName: `${clusterName}.modules`, ExclusiveStartKey: startKey }); + for (const item of page.Items ?? []) modules.push(item as unknown as ModuleInfo); + startKey = page.LastEvaluatedKey; + } while (startKey !== undefined); + return modules; +} + +/** + * `get_bucket_name`: the recorded bucket, else the conventional name. The fallback is what lets + * `save-values` work on a cluster whose settings table was lost. + */ +export async function clusterBucketName(deps: Deps, clusterName: string, awsRegion: string): Promise { + const rows = await scanSettings(deps, clusterName, '^cluster\\.cluster_s3_bucket$'); + const recorded = rows[0]?.value; + if (typeof recorded === 'string' && recorded !== '') return recorded; + return `${clusterName}-cluster-${awsRegion}-${await deps.accountId()}`; +} + +function hasConfig(dir: string): boolean { + if (!existsSync(dir) || !statSync(dir).isDirectory()) return false; + return readdirSync(dir).some((entry) => entry !== '.DS_Store'); +} + +function cleanupClusterRegionDir(dir: string, preserveValuesFile: boolean): void { + for (const entry of readdirSync(dir)) { + if (preserveValuesFile && entry === 'values.yml') continue; + rmSync(join(dir, entry), { recursive: true, force: true }); + } +} + +// --------------------------------------------------------------------------------------------- +// commands +// --------------------------------------------------------------------------------------------- + +export interface GenerateOptions { + valuesFile?: string; + configDir?: string; + force?: boolean; + existingResources?: boolean; + regenerate?: boolean; + /** Optional driver for test and replay callers. */ + installerDriver?: PromptDriver; + /** Optional identity resolver for test and replay callers. */ + installerIdentity?: (values: Readonly>) => Promise; + /** Resource-backed choices and validations for the installer. */ + installerChoices?: InstallerChoiceProvider; +} + +/** + * `config generate`. Returns the values map, because `quick-setup` deploys what this returned. + */ +export async function configGenerate(deps: Deps, options: GenerateOptions): Promise> { + if (options.configDir !== undefined && !existsSync(options.configDir)) { + deps.err(`${options.configDir} not found or is not a valid directory.`); + throw new ExitWithCode(1); + } + const values = isEmpty(options.valuesFile) + ? await collectInstallerValues({ + driver: options.installerDriver ?? new TerminalPromptDriver(), + identity: options.installerIdentity ?? (async (answers) => { + const region = answers["aws_region"]; + return installerIdentity(await deps.accountId(), typeof region === "string" ? region : undefined); + }), + existingResources: options.existingResources, + regenerate: options.regenerate, + choices: options.installerChoices, + }) + : loadPreparedValues(options.valuesFile as string); + + requireContainerShapeDecision(deps, values, options.regenerate === true); + + const clusterName = String(values['cluster_name'] ?? ''); + const awsRegion = String(values['aws_region'] ?? ''); + if (clusterName === '' || awsRegion === '') { + deps.err('Cluster name and AWS region are required'); + throw new ExitWithCode(1); + } + + const regionDir = options.configDir ?? clusterRegionDir(clusterName, awsRegion); + const valuesFileCopy = join(regionDir, 'values.yml'); + const preserveValuesFile = options.valuesFile === valuesFileCopy; + + if (options.force !== true && hasConfig(regionDir)) { + const confirm = await deps.prompt({ + message: `Config directory: ${regionDir} is not empty, would you like to overwrite it?`, + default: true, + }); + if (confirm !== true && confirm !== 'Yes') { + deps.out('Aborted!'); + throw new ExitWithCode(0); + } + cleanupClusterRegionDir(regionDir, preserveValuesFile); + } + + mkdirSync(regionDir, { recursive: true }); + deps.out(`saving values to: ${valuesFileCopy}`); + writeFileSync(valuesFileCopy, toYaml(values)); + + deps.out('generating config from templates ...'); + generateConfigFromTemplates(values, join(regionDir, 'config')); + return values; +} + +/** + * Stops a new cluster whose values file says nothing about the control-plane shape. + * + * The installer writes `enable_ecs`, so a file with no key came from somewhere else: written by + * hand, carried over from before the container control plane, or copied from another cluster. + * Generating from it produces a module set with no container module, which is the host shape, and + * the host shape no longer builds because the per-module release archives a control-plane host + * downloads are not produced any more. The failure would land partway through a deploy with + * nothing pointing at the missing key, so it is named here instead, before anything is created. + * + * `--regenerate` is the existing-cluster path. That cluster's shape is whatever it already has, so + * a missing key there is the correct answer and not a question. An explicit `false` is a recorded + * decision rather than an omission, and is left alone: the migration stages exactly that value. + */ +function requireContainerShapeDecision( + deps: Deps, + values: Readonly>, + regenerate: boolean, +): void { + if (regenerate || Object.hasOwn(values, 'enable_ecs')) return; + deps.err( + 'enable_ecs is missing from the values file. A new cluster runs its control plane as container ' + + 'tasks, and that is the only supported shape: without this key the generated module set has no ' + + 'container module, and a control plane on instances cannot be built because its per-module ' + + 'release archives are no longer produced.', + ); + deps.err('Add `enable_ecs: true` to the values file, or run `config generate` with no --values-file to let the installer write it.'); + deps.err('Regenerating the configuration of a cluster that already exists is a different command: pass --regenerate.'); + throw new ExitWithCode(1); +} + +/** Loads a supplied values file without involving the interactive installer flow. */ +function loadPreparedValues(valuesFile: string): Record { + if (!existsSync(valuesFile)) throw new ClusterConfigError(`file not found: ${valuesFile}`); + return loadValuesFile(valuesFile); +} + +/** + * The Python callback derives partition and DNS suffix from the AWS session after the account + * section. The region mapping is the same endpoint partition boundary for supported regions. + */ +function installerIdentity(accountId: string, region: string | undefined): InstallerIdentity { + if (!/^\d{12}$/.test(accountId)) throw new ClusterConfigError("sts:GetCallerIdentity returned an invalid account"); + if (region?.startsWith("cn-") === true) { + return { accountId, partition: "aws-cn", dnsSuffix: "amazonaws.com.cn" }; + } + if (region?.startsWith("us-gov-") === true) { + return { accountId, partition: "aws-us-gov", dnsSuffix: "amazonaws.com" }; + } + if (region?.startsWith("us-iso-") === true) { + return { accountId, partition: "aws-iso", dnsSuffix: "c2s.ic.gov" }; + } + if (region?.startsWith("us-isob-") === true) { + return { accountId, partition: "aws-iso-b", dnsSuffix: "sc2s.sgov.gov" }; + } + return { accountId, partition: "aws", dnsSuffix: "amazonaws.com" }; +} + +export interface UpdateOptions { + clusterName: string; + awsRegion: string; + awsProfile?: string; + moduleSet: string; + force?: boolean; + overwrite?: boolean; + keyPrefix?: string; + configDir?: string; +} + +/** `config update`: the local `config/` tree becomes the cluster settings table. */ +export async function configUpdate(deps: Deps, options: UpdateOptions): Promise { + let configDir: string; + if (!isEmpty(options.configDir)) { + configDir = join(options.configDir as string, 'config'); + if (!existsSync(configDir)) { + deps.err(`${configDir} does not exist`); + throw new ExitWithCode(1); + } + } else { + configDir = clusterConfigDir(options.clusterName, options.awsRegion); + } + + // A --config-dir pointing at another cluster's tree would push that cluster's settings here. + const localConfig = readConfigFromFiles(configDir); + const clusterModuleId = requireLocal( + localConfig, + `global-settings.module_sets.${options.moduleSet}.cluster.module_id`, + ); + const localClusterName = requireLocal(localConfig, `${clusterModuleId}.cluster_name`); + if (localClusterName !== options.clusterName) { + throw new ClusterConfigError( + `local configuration in ${configDir} does not match the given cluster name: ${options.clusterName}`, + ); + } + const localAwsRegion = requireLocal(localConfig, `${clusterModuleId}.aws.region`); + if (localAwsRegion !== options.awsRegion) { + throw new ClusterConfigError( + `local configuration in ${configDir} does not match the given aws region: ${options.awsRegion}`, + ); + } + + const readEntries = (): ConfigEntry[] => { + deps.out(`reading cluster settings from ${configDir} ...`); + const entries = convertConfigToKeyValuePairs(configDir, options.keyPrefix); + deps.out(renderTable(['Key', 'Value'], entries.map((entry) => [entry.key, pyStr(entry.value)]))); + return entries; + }; + + let entries = readEntries(); + if (options.force !== true) { + for (;;) { + const result = await deps.prompt({ + message: + 'Are you sure you want to update cluster settings db with above configuration from local file system?', + choices: ['Yes', 'Reload Changes', 'Exit'], + default: 'Yes', + }); + if (result === 'Exit') { + deps.out('Aborted!'); + throw new ExitWithCode(0); + } + if (result === 'Reload Changes') { + entries = readEntries(); + continue; + } + break; + } + } + + const writer = await deps.configWriter({ + clusterName: options.clusterName, + awsRegion: options.awsRegion, + awsProfile: options.awsProfile, + dynamodbKmsKeyId: lookupLocal(localConfig, `${clusterModuleId}.dynamodb.kms_key_id`) as string | null, + createDatabase: true, + }); + await writer.syncModulesInDb( + readModulesFromFiles(configDir).map((module) => ({ ...module, id: module.id, name: module.name, type: module.type })), + ); + await writer.syncClusterSettingsInDb(entries, options.overwrite === true); +} + +function lookupLocal(config: Record, key: string): unknown { + let node: unknown = config; + for (const part of key.split('.')) { + if (typeof node !== 'object' || node === null) return undefined; + node = (node as Record)[part]; + } + return node; +} + +function requireLocal(config: Record, key: string): string { + const value = lookupLocal(config, key); + if (typeof value !== 'string' || value === '') { + throw new ClusterConfigError(`config key not found: ${key}`); + } + return value; +} + +/** `config set`: `Key=K,Type=T,Value=V` into typed entries. Throws on the first bad entry. */ +export function parseSetEntries(entries: readonly string[]): Array<{ key: string; value: unknown }> { + return entries.map((entry, index) => { + const tokens = splitN(entry, ',', 3); + const key = afterPrefix(tokens[0], 'Key=', index, 'Key'); + const rawType = afterPrefix(tokens[1], 'Type=', index, 'Type'); + const rawValue = afterPrefix(tokens[2], 'Value=', index, 'Value'); + + if (key.includes(',') || key.includes(':')) { + throw new ClusterConfigError( + `[${index}] Invalid Key: ${key}. comma(,) and colon(:) are not allowed in key names.`, + ); + } + + const { dataType, isList } = normalizeType(rawType, index); + if (isList) { + const items = rawValue + .split(',') + .map((token) => token.trim()) + .filter((token) => token !== ''); + if (dataType === 'int') { + for (const item of items) { + if (!/^[+-]?\d+$/.test(item)) { + throw new ClusterConfigError(`[${index}] Value: ${rawValue} is not a valid list<${dataType}>`); + } + } + return { key, value: items.map((item) => Number.parseInt(item, 10)) }; + } + if (dataType === 'float') { + for (const item of items) { + if (!Number.isFinite(Number(item))) { + throw new ClusterConfigError(`[${index}] Value: ${rawValue} is not a valid list<${dataType}>`); + } + } + return { key, value: items.map((item) => Number(item)) }; + } + // Python's list branch is dead code (a duplicate `int` test), so a list stays + // a list of strings, as it does today. + return { key, value: items }; + } + + if (dataType === 'int') { + if (!/^[+-]?\d+$/.test(rawValue)) { + throw new ClusterConfigError(`[${index}] Value: ${rawValue} is not a valid ${dataType}`); + } + return { key, value: Number.parseInt(rawValue, 10) }; + } + if (dataType === 'float') { + if (!Number.isFinite(Number(rawValue))) { + throw new ClusterConfigError(`[${index}] Value: ${rawValue} is not a valid ${dataType}`); + } + return { key, value: Number(rawValue) }; + } + if (dataType === 'bool') { + return { key, value: ['true', 'yes', 'y', '1', 'on'].includes(rawValue.toLowerCase()) }; + } + return { key, value: rawValue }; + }); +} + +function splitN(value: string, separator: string, limit: number): string[] { + const parts: string[] = []; + let rest = value; + while (parts.length < limit - 1) { + const index = rest.indexOf(separator); + if (index === -1) break; + parts.push(rest.slice(0, index)); + rest = rest.slice(index + separator.length); + } + parts.push(rest); + return parts; +} + +function afterPrefix(token: string | undefined, prefix: string, index: number, name: string): string { + const parts = (token ?? '').split(prefix); + const value = parts[1]?.trim() ?? ''; + if (value === '') throw new ClusterConfigError(`[${index}] ${name} is required`); + return value; +} + +function normalizeType(rawType: string, index: number): { dataType: string; isList: boolean } { + const table: Record = { + str: { dataType: 'str', isList: false }, + string: { dataType: 'str', isList: false }, + int: { dataType: 'int', isList: false }, + integer: { dataType: 'int', isList: false }, + bool: { dataType: 'bool', isList: false }, + boolean: { dataType: 'bool', isList: false }, + float: { dataType: 'float', isList: false }, + decimal: { dataType: 'float', isList: false }, + 'list': { dataType: 'str', isList: true }, + 'list': { dataType: 'str', isList: true }, + 'list': { dataType: 'int', isList: true }, + 'list': { dataType: 'int', isList: true }, + 'list': { dataType: 'bool', isList: true }, + 'list': { dataType: 'bool', isList: true }, + 'list': { dataType: 'float', isList: true }, + 'list': { dataType: 'float', isList: true }, + }; + const resolved = table[rawType]; + if (resolved === undefined) throw new ClusterConfigError(`[${index}] Type: ${rawType} not supported`); + return resolved; +} + +/** `config export`: the DB back into a `config/` tree. Refuses a non-empty directory. */ +export async function configExport( + deps: Deps, + options: { clusterName: string; awsRegion: string; awsProfile?: string; exportDir?: string }, +): Promise { + const exportDir = + options.exportDir ?? join(clusterRegionDir(options.clusterName, options.awsRegion), 'config'); + if (hasConfig(exportDir)) { + throw new GeneralException( + `export directory: ${exportDir} already exists and can cause merge conflicts. ` + + 'backup your existing configuration to another directory and try again.', + ); + } + const entries = await scanSettings(deps, options.clusterName); + const modules = await scanModules(deps, options.clusterName); + deps.out(`exporting config from db to ${exportDir} ...`); + mkdirSync(exportDir, { recursive: true }); + + const tree = buildTree(entries); + const ideaConfig: { modules: Array> } = { modules: [] }; + for (const module of modules) { + const moduleDir = join(exportDir, module.module_id); + mkdirSync(moduleDir, { recursive: true }); + writeFileSync(join(moduleDir, 'settings.yml'), toYaml(tree[module.module_id] ?? {})); + ideaConfig.modules.push({ + name: module.name, + id: module.module_id, + type: module.type, + config_files: ['settings.yml'], + }); + } + writeFileSync(join(exportDir, 'idea.yml'), toYaml(ideaConfig)); +} + +/** `config diff`: local `config/` against the DB, as MODIFIED / DELETED / ADDED. */ +export async function configDiff( + deps: Deps, + options: { clusterName: string; awsRegion: string; configDir?: string }, +): Promise> { + const configDir = options.configDir ?? clusterConfigDir(options.clusterName, options.awsRegion); + const dbEntries = new Map(); + for (const row of await scanSettings(deps, options.clusterName)) dbEntries.set(row.key, pyStr(row.value)); + const localEntries = new Map(); + for (const entry of convertConfigToKeyValuePairs(configDir)) localEntries.set(entry.key, pyStr(entry.value)); + + const rows: Array<[string, string, string, string]> = []; + for (const [key, value] of dbEntries) { + if (localEntries.get(key) === value) continue; + if (localEntries.has(key)) rows.push([key, value, localEntries.get(key) as string, 'MODIFIED']); + else rows.push([key, value, 'n/a', 'DELETED']); + } + for (const [key, value] of localEntries) { + if (dbEntries.has(key)) continue; + rows.push([key, 'n/a', value, 'ADDED']); + } + rows.sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)); + return rows; +} + +/** Options shared by the standalone preview and the upgrade pre-write step. */ +export interface ConfigUpgradePreviewOptions { + clusterName: string; + awsRegion: string; + awsProfile?: string; + baseOs?: string; + valuesFile?: string; + skipGlobalSettingsUpdate?: boolean; + modules?: readonly string[]; +} + +/** + * Tests and replay callers can supply the complete comparison input. The live + * command prepares the same input through the upgrade command's read adapters. + */ +export interface ConfigDriftPreviewDeps extends Deps { + loadUpgradeDriftInput?( + options: ConfigUpgradePreviewOptions, + ): Promise; +} + +/** Render the same preview used by `upgrade-cluster`, without changing cluster state. */ +export async function configUpgradePreview( + deps: ConfigDriftPreviewDeps, + options: ConfigUpgradePreviewOptions, +): Promise { + const { compareUpgradeDrift, renderUpgradeDrift } = await import("../../config/upgrade-drift.ts"); + const input = + deps.loadUpgradeDriftInput === undefined + ? await (async () => { + const { createLiveUpgradeDeps, prepareUpgradeDriftInput } = await import("./upgrade.ts"); + return prepareUpgradeDriftInput(createLiveUpgradeDeps(deps), options); + })() + : await deps.loadUpgradeDriftInput(options); + const report = compareUpgradeDrift(input); + deps.out(renderUpgradeDrift(report)); + return report; +} + +// --------------------------------------------------------------------------------------------- +// commander wiring +// --------------------------------------------------------------------------------------------- + +const clusterOptions = (command: Command): Command => + command + .requiredOption('--cluster-name ', 'Cluster Name') + .option('--aws-profile ', 'AWS Profile Name') + .requiredOption('--aws-region ', 'AWS Region'); + +export function registerConfigCommands(program: Command, deps: Deps): Command { + const config = program.command('config').description('configuration options'); + + config + .command('generate') + .description('generate configuration') + .option('--values-file ', 'path to values.yml file') + .option('--config-dir ', 'path to where to create config directory') + .option('--force', 'Skip all confirmation prompts.') + .option('--existing-resources', 'Generate configuration using existing resources') + .option( + '--regenerate', + 'Regenerate configuration for an existing cluster. Enables skipping validations such as existing cluster name and CIDR block.', + ) + .action(async (options: GenerateOptions) => { + await configGenerate(deps, options); + }); + + clusterOptions(config.command('update')) + .description('update configuration from local file system to cluster settings db') + .option('--force', 'Skip all confirmation prompts.') + .option('--overwrite', 'Overwrite existing db config entries. Default behavior is to skip if the config entry exists.') + .option('--key-prefix ', 'Update configuration for the keys matching the given key prefix.') + .option('--config-dir ', 'Path to Config Directory; Uses default location if not provided') + .option('--module-set ', 'Name of the ModuleSet. Default: default', 'default') + .action(async (options: UpdateOptions) => { + await configUpdate(deps, options); + }); + + clusterOptions(config.command('set')) + .description('set config entries') + .option('--force', 'Skip confirmation prompts') + .argument('', 'Key=KEY_NAME,Type=[str|int|float|bool|list|list|list|list],Value=VALUE') + .action(async (entries: string[], options: { clusterName: string; awsRegion: string; awsProfile?: string; force?: boolean }) => { + const parsed = parseSetEntries(entries); + deps.out(renderTable(['Key', 'Value'], parsed.map((entry) => [entry.key, pyStr(entry.value)]))); + if (options.force !== true) { + const confirm = await deps.prompt({ message: 'Are you sure you want to update above config entries?' }); + if (confirm !== true && confirm !== 'Yes') { + deps.out('Abort!'); + throw new ExitWithCode(0); + } + } + const writer = await deps.configWriter({ + clusterName: options.clusterName, + awsRegion: options.awsRegion, + awsProfile: options.awsProfile, + }); + for (const entry of parsed) await writer.setConfigEntry(entry.key, entry.value); + }); + + clusterOptions(config.command('show')) + .description('show configuration for a cluster as yaml') + .option('-q, --query ', 'Search Query for configuration entries. Accepts a regular expression.') + .addOption(new Option('--format ', 'Output format. Default: table').choices(['table', 'yaml', 'raw'])) + .action(async (options: { clusterName: string; query?: string; format?: string }) => { + const rows = await scanSettings(deps, options.clusterName, options.query); + if (options.format === 'yaml') { + deps.out(toYaml(buildTree(rows))); + } else if (options.format === 'raw') { + for (const row of rows) if (row.value !== undefined && row.value !== null) deps.out(pyStr(row.value)); + } else { + deps.out( + renderTable( + ['Key', 'Value', 'Version'], + rows.map((row) => [row.key, pyStr(row.value), String(row.version ?? 0)]), + ), + ); + } + }); + + clusterOptions(config.command('export')) + .description('export configuration') + .option('--export-dir ', 'Export Directory. Defaults to the cluster config directory.') + .action(async (options: { clusterName: string; awsRegion: string; awsProfile?: string; exportDir?: string }) => { + await configExport(deps, options); + }); + + clusterOptions(config.command('delete')) + .description('delete all configuration entries for a given config key prefix') + .argument('', 'config key prefixes') + .action(async (prefixes: string[], options: { clusterName: string; awsRegion: string; awsProfile?: string }) => { + const writer = await deps.configWriter({ + clusterName: options.clusterName, + awsRegion: options.awsRegion, + awsProfile: options.awsProfile, + }); + for (const prefix of prefixes) await writer.deleteConfigEntries(prefix.trim()); + }); + + clusterOptions(config.command('diff')) + .description('diff configuration files between the latest config and the config in the db') + .option('--config-dir ', 'Path to local config folder; default location will be used if none provided') + .action(async (options: { clusterName: string; awsRegion: string; configDir?: string }) => { + const rows = await configDiff(deps, options); + deps.out(renderTable(['Key', 'Old Value', 'New Value', 'Status'], rows)); + }); + + clusterOptions(config.command("preview-upgrade")) + .description("preview configuration changes made by an upgrade") + .option("--base-os ", "Base OS to upgrade to.") + .option("--values-file ", "Path to values.yml. Uses the cluster copy by default.") + .option("--skip-global-settings-update", "Skip updating global settings.") + .argument("[modules...]", "module ids") + .action(async (modules: string[], options: ConfigUpgradePreviewOptions) => { + await configUpgradePreview(deps, { ...options, modules }); + }); + + clusterOptions(config.command('save-values')) + .description('save values file in s3 bucket') + .option('--values-file ', 'path to values.yml file') + .action(async (options: { clusterName: string; awsRegion: string; valuesFile?: string }) => { + const valuesFile = options.valuesFile ?? valuesFilePath(options.clusterName, options.awsRegion); + const bucket = await clusterBucketName(deps, options.clusterName, options.awsRegion); + await deps.s3.putObject({ Bucket: bucket, Key: VALUES_FILE_S3_KEY, Body: readFileSync(valuesFile) }); + deps.out(`saved ${valuesFile} to s3://${bucket}/${VALUES_FILE_S3_KEY}`); + }); + + clusterOptions(config.command('download-values')) + .description('download values.yml from s3 bucket to default or provided location') + .option('--values-dir ', 'Path to folder to save values.yml file') + .action(async (options: { clusterName: string; awsRegion: string; valuesDir?: string }) => { + const valuesFile = + options.valuesDir === undefined + ? valuesFilePath(options.clusterName, options.awsRegion) + : join(mkdirSync(options.valuesDir, { recursive: true }) ?? options.valuesDir, 'values.yml'); + const bucket = await clusterBucketName(deps, options.clusterName, options.awsRegion); + let body: string; + try { + body = await deps.s3.getObject({ Bucket: bucket, Key: VALUES_FILE_S3_KEY }); + } catch (error) { + deps.err( + `Values file not found at ${valuesFile} and could not be downloaded from ` + + `s3://${bucket}/${VALUES_FILE_S3_KEY}: ${(error as Error).message}. Restore values.yml to ` + + `${valuesFile} from a backup, then upload it with: ideactl config save-values`, + ); + throw new ExitWithCode(1); + } + writeFileSync(valuesFile, toYaml(yaml.load(body))); + deps.out(`downloaded s3://${bucket}/${VALUES_FILE_S3_KEY} to ${valuesFile}`); + }); + + return config; +} diff --git a/source/idea/ideactl/src/cli/commands/delete-cluster.ts b/source/idea/ideactl/src/cli/commands/delete-cluster.ts new file mode 100644 index 00000000..a8ab64a8 --- /dev/null +++ b/source/idea/ideactl/src/cli/commands/delete-cluster.ts @@ -0,0 +1,665 @@ +/** + * Destructive cluster removal commands. Effects are injected so the command can be replayed + * without credentials and each resource-discovery boundary remains explicit. + */ + +import type { Command } from "commander"; + +import type { ClusterConfig, ModuleInfo } from "../../config/cluster-config.ts"; + +export const CLUSTER_NAME_TAG = "idea:ClusterName"; +export const NODE_TYPE_TAG = "idea:NodeType"; +/** Set by the platform on every instance an auto scaling group launches. */ +export const AUTOSCALING_GROUP_TAG = "aws:autoscaling:groupName"; +export const MODULE_ID_TAG = "idea:ModuleId"; +const APP_NODE_TYPE = "app"; +const INFRA_NODE_TYPE = "infra"; + +export interface DeleteClusterOptions { + clusterName: string; + awsRegion: string; + awsProfile?: string; + deleteBootstrap?: boolean; + deleteDatabases?: boolean; + deleteBackups?: boolean; + deleteCloudwatchLogs?: boolean; + deleteAll?: boolean; + force?: boolean; +} + +export interface DeleteClusterInstance { + instanceId: string; + state: string; + nodeType?: string; + /** The auto scaling group that launched it, from the tag the platform sets on every member. */ + autoScalingGroupName?: string; +} + +export interface DeleteClusterStack { + stackName: string; + stackStatus?: string; + terminationProtection?: boolean; +} + +export interface DeleteClusterUserPool { + id: string; + name: string; +} + +export interface DescribedUserPool { + deletionProtection?: string; + tags?: Record; +} + +export interface RecoveryPoint { + arn: string; + status?: string; +} + +export interface ProjectRecord { + [key: string]: unknown; +} + +export interface DeletionTagFilter { + key: string; + values: string[]; +} + +export interface DeletionInstanceFilter { + name: string; + values: string[]; +} + +/** + * All deletion effects. The live CLI supplies adapters, while unit tests inject a recording + * implementation. Discovery inputs deliberately use the native filter shapes. + */ +export interface DeleteClusterDeps { + loadConfig(input: { clusterName: string; awsRegion: string; awsProfile?: string }): Promise; + findInstances(input: { filters: DeletionInstanceFilter[] }): Promise; + instanceTerminationProtection(instanceId: string): Promise; + disableInstanceTerminationProtection(instanceId: string): Promise; + terminateInstance(input: { instanceId: string; force: boolean; skipOsShutdown: boolean }): Promise; + getTaggedStacks(input: { + tagFilters: DeletionTagFilter[]; + resourceTypeFilters: string[]; + paginationToken?: string; + }): Promise<{ stacks: string[]; paginationToken?: string }>; + describeStack(stackName: string): Promise; + disableStackTerminationProtection(stackName: string): Promise; + /** `RetainResources` is passed only on a re-delete of a stack CloudFormation could not finish. */ + deleteStack(stackName: string, retainResources?: string[]): Promise; + /** Logical ids left in `DELETE_FAILED`. Optional: a replay implementation may not have them. */ + stackFailedResources?(stackName: string): Promise; + findAppInstance(input: { clusterName: string; moduleId: string }): Promise; + sendAppCleanup(input: { instanceIds: string[]; deleteDatabases: boolean }): Promise; + appCleanupStatus(commandId: string): Promise>; + findBedrockProjects(clusterName: string): Promise; + deleteBedrockProjectResources(input: { clusterName: string; projects: ProjectRecord[] }): Promise; + listUserPools(nextToken?: string): Promise<{ pools: DeleteClusterUserPool[]; nextToken?: string }>; + describeUserPool(userPoolId: string): Promise; + disableUserPoolDeletionProtection(userPoolId: string): Promise; + describeLambdaNetworkInterfaces(input: { clusterName: string }): Promise< + Array<{ networkInterfaceId: string; description?: string }> + >; + deleteNetworkInterface(networkInterfaceId: string): Promise; + describeBackupVault(backupVaultName: string): Promise; + listRecoveryPoints(backupVaultName: string): Promise; + deleteRecoveryPoint(input: { backupVaultName: string; recoveryPointArn: string }): Promise; + listTables(nextTableName?: string): Promise<{ tableNames: string[]; nextTableName?: string }>; + deleteTable(tableName: string): Promise; + listDynamoDbAlarms(clusterName: string): Promise>; + deleteAlarms(alarmNames: string[]): Promise; + listLogGroups(prefix: string): Promise>; + deleteLogGroup(name: string): Promise; + accountId(): Promise; + bucketExists(name: string): Promise; + deleteAllBucketObjectVersions(name: string): Promise; + deleteBucket(name: string): Promise; + prompt(message: string): Promise; + sleep(milliseconds: number): Promise; + out(line: string): void; + err(line: string): void; +} + +/** Builds dependencies for the profile selected by one deletion command. */ +export type DeleteClusterDepsFactory = ( + options: DeleteClusterOptions, + command: "delete-cluster" | "delete-backups", +) => Promise; + +type DeleteClusterDepsSource = DeleteClusterDeps | DeleteClusterDepsFactory; + +function isConfirmed(answer: boolean | string): boolean { + return answer === true || answer === "Yes"; +} + +function shouldDelete(options: DeleteClusterOptions, name: "Bootstrap" | "Databases" | "Backups" | "CloudwatchLogs"): boolean { + if (options.deleteAll === true) return true; + if (name === "Bootstrap") return options.deleteBootstrap === true; + if (name === "Databases") return options.deleteDatabases === true; + if (name === "Backups") return options.deleteBackups === true; + return options.deleteCloudwatchLogs === true; +} + +function stackNameMatches(moduleName: string, stack: DeleteClusterStack, clusterName: string, modules: ModuleInfo[]): boolean { + const configured = modules.find((module) => module.name === moduleName)?.stack_name; + return stack.stackName === configured || stack.stackName === `${clusterName}-${moduleName}`; +} + +/** + * A stack that is not there. CloudFormation answers a missing stack with a `ValidationError` whose + * message is `Stack with id does not exist`, so the code has to be matched as well as the + * message: matching the message alone classifies every disappeared stack as a hard failure. + */ +export function isMissingStackError(error: unknown): boolean { + if (!(error instanceof Error)) return false; + return /ValidationError|not found|does not exist/i.test(`${error.name}: ${error.message}`); +} + +/** + * Exact equivalent of the delete command's stack deletion helper. It discovers current stack + * protection before clearing it, and treats a disappeared stack as already deleted. + */ +async function deleteStack( + deps: DeleteClusterDeps, + options: DeleteClusterOptions, + stackName: string, + retainResources?: string[], +): Promise { + let stack: DeleteClusterStack; + try { + stack = await deps.describeStack(stackName); + } catch (error) { + if (isMissingStackError(error)) return; + throw error; + } + + if (stack.terminationProtection === true && options.force !== true) { + const confirmed = await deps.prompt(`Termination protection is enabled for stack: ${stackName}. Disable and terminate?`); + if (!isConfirmed(confirmed)) throw new DeleteClusterAbort("Cluster deletion cancelled. No stacks were deleted."); + } + if (stack.terminationProtection === true) { + deps.out(`disabling termination protection for stack: ${stackName}`); + await deps.disableStackTerminationProtection(stackName); + } + if (retainResources === undefined || retainResources.length === 0) { + deps.out(`terminating CloudFormation stack: ${stackName}`); + await deps.deleteStack(stackName); + return; + } + deps.out( + `terminating CloudFormation stack: ${stackName}, retaining ${retainResources.length} resource(s) ` + + `CloudFormation could not delete: ${retainResources.join(", ")}. These are left in the account and ` + + `have to be removed by hand.`, + ); + await deps.deleteStack(stackName, retainResources); +} + +/** + * Waits out one stack group, printing every status it reads so a slow delete is distinguishable + * from a wedged one. + * + * A stack in `DELETE_FAILED` is re-issued. The first re-issue is a plain delete, after the Lambda + * interface sweep, because an interface holding a security group is the common and recoverable + * cause. A stack that fails again is re-issued retaining the resources CloudFormation could not + * delete, which is the only way past a resource whose deletion never succeeds; the retained ids are + * printed because they stay in the account. The retry counter is shared by the whole invocation. + */ +async function waitForStackDeletion( + deps: DeleteClusterDeps, + options: DeleteClusterOptions, + stackNames: string[], + retryState: { attempts: number }, +): Promise { + let failed = 0; + const pending = [...stackNames]; + const maxAttempts = 3; + const failures = new Map(); + while (pending.length > 0) { + const deleted: string[] = []; + for (const stackName of pending) { + try { + const stack = await deps.describeStack(stackName); + if (stack.stackStatus === "DELETE_COMPLETE") { + deps.out(`stack: ${stackName}, status: ${stack.stackStatus}`); + deleted.push(stackName); + } else if (stack.stackStatus === "DELETE_FAILED") { + const previousFailures = (failures.get(stackName) ?? 0) + 1; + failures.set(stackName, previousFailures); + if (retryState.attempts < maxAttempts) { + deps.err( + `stack: ${stackName}, status: ${stack.stackStatus}, submitting a new delete request. ` + + `[Loop ${retryState.attempts}/${maxAttempts}]`, + ); + await deleteLambdaNetworkInterfaces(deps, options.clusterName); + const retain = previousFailures > 1 ? await failedResources(deps, stackName) : []; + await deleteStack(deps, options, stackName, retain); + retryState.attempts += 1; + } else { + deps.err(`stack: ${stackName}, status: ${stack.stackStatus}`); + deleted.push(stackName); + failed += 1; + } + } else { + deps.out(`stack: ${stackName}, status: ${stack.stackStatus ?? "unknown"}`); + // Sweep on every poll of a stack that is still deleting, not only the search stack: a + // function's interfaces become available part way through its own stack's delete, and the + // group that owns the network is the one that then waits out a timeout for them. + await deleteLambdaNetworkInterfaces(deps, options.clusterName); + } + } catch (error) { + if (isMissingStackError(error)) { + deps.out(`stack: ${stackName}, status: DELETE_COMPLETE`); + deleted.push(stackName); + } else { + throw error; + } + } + } + for (const stackName of deleted) pending.splice(pending.indexOf(stackName), 1); + if (pending.length > 0) { + deps.out(`waiting for ${pending.length} stack(s) to be deleted: ${pending.join(", ")} ...`); + await deps.sleep(15_000); + } + } + return failed === 0; +} + +/** Logical ids CloudFormation reports as `DELETE_FAILED`, empty when the read is unavailable. */ +async function failedResources(deps: DeleteClusterDeps, stackName: string): Promise { + if (deps.stackFailedResources === undefined) return []; + try { + return await deps.stackFailedResources(stackName); + } catch (error) { + deps.err(`could not read failed resources of ${stackName}: ${error instanceof Error ? error.message : String(error)}`); + return []; + } +} + +/** + * Removes the cluster's available Lambda network interfaces. + * + * A function in the cluster's network leaves its interfaces behind after the function is gone, and + * each one holds the security group it was launched with. CloudFormation cannot delete a group an + * interface still references, so it waits out its own timeout and then reports DELETE_FAILED. On a + * measured run that timeout was 57 minutes, and the same delete finished in 31 seconds once the + * interfaces were gone. + */ +async function deleteLambdaNetworkInterfaces(deps: DeleteClusterDeps, clusterName: string): Promise { + const interfaces = await deps.describeLambdaNetworkInterfaces({ clusterName }); + if (interfaces.length === 0) return; + deps.out(`found ${interfaces.length} available Lambda network interface(s) for the cluster. deleting ...`); + for (const networkInterface of interfaces) { + deps.out( + `deleting Lambda network interface: ${networkInterface.networkInterfaceId}` + + (networkInterface.description === undefined ? "" : `, description: ${networkInterface.description}`), + ); + await deps.deleteNetworkInterface(networkInterface.networkInterfaceId); + } +} + +/** + * Deletes one group of stacks and waits for the group to finish. + * + * The interface sweep runs before the first delete is issued, not only on a failure: interfaces + * left available by an earlier deploy already hold the groups these stacks own, so a sweep after + * the fact costs the operator the CloudFormation timeout first. + */ +async function deleteStackGroup( + deps: DeleteClusterDeps, + options: DeleteClusterOptions, + stacks: DeleteClusterStack[], + retryState: { attempts: number }, +): Promise { + const stackNames = stacks.map((stack) => stack.stackName); + if (stackNames.length === 0) return; + await deleteLambdaNetworkInterfaces(deps, options.clusterName); + for (const stackName of stackNames) await deleteStack(deps, options, stackName); + const successful = await waitForStackDeletion(deps, options, stackNames, retryState); + if (successful) return; + throw new DeleteClusterAbort( + `CloudFormation stacks for cluster ${options.clusterName} did not all delete. Later delete steps were not run. Check the stack events, then re-run ideactl delete-cluster --cluster-name ${options.clusterName} --aws-region ${options.awsRegion}.`, + ); +} + +async function cleanUpAppModules( + deps: DeleteClusterDeps, + options: DeleteClusterOptions, + modules: ModuleInfo[], +): Promise { + const instanceIds: string[] = []; + for (const module of modules) { + if (module.type !== "app") continue; + const instance = await deps.findAppInstance({ clusterName: options.clusterName, moduleId: module.module_id }); + if (instance?.state === "running") instanceIds.push(instance.instanceId); + } + const commandId = await deps.sendAppCleanup({ + instanceIds, + deleteDatabases: shouldDelete(options, "Databases"), + }); + for (;;) { + const statuses = await deps.appCleanupStatus(commandId); + const complete = statuses.filter((entry) => ["Success", "TimedOut", "Cancelled", "Failed"].includes(entry.status)); + if (complete.length === statuses.length) return; + await deps.sleep(10_000); + } +} + +async function deleteIdentityProviderStacks( + deps: DeleteClusterDeps, + options: DeleteClusterOptions, + identityStacks: DeleteClusterStack[], + retryState: { attempts: number }, +): Promise { + const userPoolIds: string[] = []; + let nextToken: string | undefined; + do { + const page = await deps.listUserPools(nextToken); + nextToken = page.nextToken; + if (options.force !== true) { + const confirmed = await deps.prompt( + `Are you sure you want to delete the User Pools associated with the cluster: ${options.clusterName}? This action is not reversible.`, + ); + if (!isConfirmed(confirmed)) throw new DeleteClusterAbort("Cluster deletion cancelled. No stacks were deleted."); + } + for (const pool of page.pools) { + if (pool.name === `${options.clusterName}-user-pool`) userPoolIds.push(pool.id); + } + for (const userPoolId of userPoolIds) { + const pool = await deps.describeUserPool(userPoolId); + if (pool.deletionProtection?.toUpperCase() !== "ACTIVE") continue; + const tags = pool.tags ?? {}; + if (Object.keys(tags).length === 0 || tags[CLUSTER_NAME_TAG] === options.clusterName) { + await deps.disableUserPoolDeletionProtection(userPoolId); + await deps.sleep(500); + } + } + } while (nextToken !== undefined && nextToken !== ""); + + await deleteStackGroup(deps, options, identityStacks, retryState); +} + +/** + * Purges the cluster's backup vault. + * + * The existence probe is allowed to fail for any reason and the purge is then skipped: a cluster + * installed without backups has no vault, and the service answers a vault that does not exist with + * an access denial rather than a not-found, so the two are indistinguishable from here and neither + * is a reason to stop a teardown. + * + * A failure once the vault is known to exist is raised. A recovery point that is not deleted keeps + * the vault, and the vault keeps its stack, so swallowing that error turns a loud failure into a + * stack delete that fails later for a reason nothing printed. + */ +async function deleteBackups(deps: DeleteClusterDeps, clusterName: string): Promise { + const vaultName = `${clusterName}-cluster-backup-vault`; + try { + await deps.describeBackupVault(vaultName); + } catch (error) { + deps.out( + `backup vault ${vaultName} is not readable (${error instanceof Error ? error.message : String(error)}). skip.`, + ); + return; + } + const recoveryPoints = await deps.listRecoveryPoints(vaultName); + deps.out(`${recoveryPoints.length} recovery point(s) in vault ${vaultName}`); + for (const point of recoveryPoints) { + const status = (point.status ?? "UNKNOWN").toUpperCase(); + if (status !== "COMPLETED" && status !== "EXPIRED") { + deps.err(`cannot delete recovery point ${point.arn}, status: ${status}. this may block the stack delete.`); + continue; + } + deps.out(`deleting recovery point: ${point.arn} ...`); + await deps.deleteRecoveryPoint({ backupVaultName: vaultName, recoveryPointArn: point.arn }); + await deps.sleep(100); + } +} + +async function deleteBootstrapAndBucket( + deps: DeleteClusterDeps, + options: DeleteClusterOptions, + config: ClusterConfig | undefined, +): Promise { + await deleteStack(deps, options, `${options.clusterName}-bootstrap`); + const configuredBucket = config?.getString("cluster.cluster_s3_bucket", undefined, { moduleId: "cluster" }); + const bucketName = configuredBucket ?? `${options.clusterName}-cluster-${options.awsRegion}-${await deps.accountId()}`; + if (!(await deps.bucketExists(bucketName))) { + deps.out(`cluster bucket not found: ${bucketName}. skip.`); + return; + } + deps.out(`deleting S3 bucket: ${bucketName} ...`); + await deps.deleteAllBucketObjectVersions(bucketName); + await deps.sleep(5_000); + await deps.deleteBucket(bucketName); +} + +async function deleteDatabases(deps: DeleteClusterDeps, options: DeleteClusterOptions): Promise { + const tables: string[] = []; + let nextTableName: string | undefined; + do { + const page = await deps.listTables(nextTableName); + nextTableName = page.nextTableName; + tables.push(...page.tableNames.filter((name) => name.startsWith(`${options.clusterName}.`))); + } while (nextTableName !== undefined && nextTableName !== ""); + if (tables.length === 0) return; + + if (options.force !== true) { + const confirmed = await deps.prompt(`Are you sure you want to delete all dynamodb tables associated with the cluster: ${options.clusterName}?`); + if (!isConfirmed(confirmed)) return; + } + for (const tableName of tables) { + deps.out(`deleting table: ${tableName} ...`); + await deps.deleteTable(tableName); + } + + const alarmNames = (await deps.listDynamoDbAlarms(options.clusterName)) + .filter((alarm) => alarm.namespace === "AWS/DynamoDB" && alarm.tableName !== undefined && tables.includes(alarm.tableName)) + .map((alarm) => alarm.name); + if (alarmNames.length > 0) deps.out(`deleting ${alarmNames.length} cloudwatch alarm(s) ...`); + for (let index = 0; index < alarmNames.length; index += 100) { + await deps.deleteAlarms(alarmNames.slice(index, index + 100)); + } +} + +async function deleteCloudwatchLogs(deps: DeleteClusterDeps, options: DeleteClusterOptions): Promise { + const logs: Array<{ name: string; size: number }> = []; + for (const prefix of [options.clusterName, `/${options.clusterName}`, `/aws/lambda/${options.clusterName}`]) { + logs.push(...(await deps.listLogGroups(prefix))); + } + if (logs.length === 0) return; + if (options.force !== true) { + const confirmed = await deps.prompt(`Are you sure you want to delete all cloudwatch logs associated with the cluster: ${options.clusterName}?`); + if (!isConfirmed(confirmed)) return; + } + for (const log of logs) { + deps.out(`deleting cloudwatch log group: ${log.name} ...`); + await deps.deleteLogGroup(log.name); + await deps.sleep(100); + } +} + +export class DeleteClusterAbort extends Error {} + +/** + * Runs the documented 13-step removal sequence. Every discovered stack is scoped by the cluster + * tag, and the retained bootstrap bucket is reached only in the explicit bootstrap branch. + */ +export async function deleteCluster(deps: DeleteClusterDeps, options: DeleteClusterOptions): Promise { + deps.out(`deleting cluster: ${options.clusterName}, region: ${options.awsRegion}`); + const config = await deps.loadConfig(options); + const modules = config?.modules() ?? []; + if (config === undefined) deps.out("no cluster settings tables found. using generated resource names."); + deps.out(`searching for EC2 instances tagged ${CLUSTER_NAME_TAG}=${options.clusterName} ...`); + const instances = await deps.findInstances({ + filters: [{ name: `tag:${CLUSTER_NAME_TAG}`, values: [options.clusterName] }], + }); + const ec2Instances: DeleteClusterInstance[] = []; + const protectedInstances: DeleteClusterInstance[] = []; + const groupInstances: DeleteClusterInstance[] = []; + for (const instance of instances) { + if (instance.state === "terminated") continue; + // A member of an auto scaling group is not ours to terminate. Terminating one while its group + // still wants that many members has the group replace it, which is the group doing its job + // and this command losing a race it started. The stack delete removes the group, and the + // group removes its members, once the container stack's release has cleared their scale-in + // protection. Terminating early is a harmless head start only for a standalone instance, + // which is what every instance in a cluster used to be. + if (instance.autoScalingGroupName !== undefined) { + groupInstances.push(instance); + continue; + } + if (await deps.instanceTerminationProtection(instance.instanceId)) protectedInstances.push(instance); + await deps.sleep(100); + if (instance.nodeType !== APP_NODE_TYPE && instance.nodeType !== INFRA_NODE_TYPE) ec2Instances.push(instance); + } + for (const instance of groupInstances) { + deps.out( + `instance left to its auto scaling group: ${instance.instanceId}, group: ${instance.autoScalingGroupName ?? "unknown"}`, + ); + } + + for (const instance of ec2Instances) deps.out(`instance to terminate: ${instance.instanceId}, state: ${instance.state}`); + for (const instance of protectedInstances) deps.out(`instance with termination protection: ${instance.instanceId}`); + + deps.out(`searching for CloudFormation stacks tagged ${CLUSTER_NAME_TAG}=${options.clusterName} ...`); + const regularStacks: DeleteClusterStack[] = []; + const clusterStacks: DeleteClusterStack[] = []; + const identityStacks: DeleteClusterStack[] = []; + let paginationToken: string | undefined; + do { + const page = await deps.getTaggedStacks({ + tagFilters: [{ key: CLUSTER_NAME_TAG, values: [options.clusterName] }], + resourceTypeFilters: ["cloudformation"], + paginationToken, + }); + paginationToken = page.paginationToken; + for (const stackId of page.stacks) { + try { + const stack = await deps.describeStack(stackId); + if (stack.stackName === `${options.clusterName}-bootstrap`) continue; + if (stackNameMatches("cluster", stack, options.clusterName, modules)) { + clusterStacks.push(stack); + } else if (stackNameMatches("identity-provider", stack, options.clusterName, modules)) { + identityStacks.push(stack); + } else { + regularStacks.push(stack); + } + await deps.sleep(500); + } catch (error) { + if (!isMissingStackError(error)) throw error; + } + } + } while (paginationToken !== undefined && paginationToken !== ""); + for (const stack of [...regularStacks, ...identityStacks, ...clusterStacks]) { + deps.out( + `stack to delete: ${stack.stackName}, status: ${stack.stackStatus ?? "unknown"}, ` + + `termination protection: ${stack.terminationProtection === true}`, + ); + } + deps.out(`${regularStacks.length + identityStacks.length + clusterStacks.length} stack(s) will be terminated.`); + + if (options.force !== true) { + const confirmed = await deps.prompt(`Are you sure you want to delete cluster: ${options.clusterName}, region: ${options.awsRegion} ?`); + if (!isConfirmed(confirmed)) return; + } + + deps.out("running the application module clean-up ..."); + await cleanUpAppModules(deps, options, modules); + if (protectedInstances.length > 0 && options.force !== true) { + const confirmed = await deps.prompt("Are you sure you want to disable termination protection for above instances ?"); + if (!isConfirmed(confirmed)) return; + } + for (const instance of protectedInstances) { + deps.out(`disabling termination protection for EC2 instance: ${instance.instanceId} ...`); + await deps.disableInstanceTerminationProtection(instance.instanceId); + await deps.sleep(1_000); + } + for (const instance of ec2Instances) { + deps.out(`terminating EC2 instance: ${instance.instanceId}`); + await deps.terminateInstance({ + instanceId: instance.instanceId, + force: options.force === true, + skipOsShutdown: options.force === true, + }); + await deps.sleep(1_000); + } + + deps.out("searching for project resources CloudFormation does not own ..."); + const projects = await deps.findBedrockProjects(options.clusterName); + try { + if (projects.length > 0) await deps.deleteBedrockProjectResources({ clusterName: options.clusterName, projects }); + } catch (error) { + deps.err(`failed to delete bedrock project resources: ${error instanceof Error ? error.message : String(error)}`); + } + + const retryState = { attempts: 0 }; + deps.out(`deleting ${regularStacks.length} module stack(s) ...`); + await deleteStackGroup(deps, options, regularStacks, retryState); + + deps.out(`deleting ${identityStacks.length} identity-provider stack(s) ...`); + await deleteIdentityProviderStacks(deps, options, identityStacks, retryState); + + if (shouldDelete(options, "Backups")) { + if (options.force === true || isConfirmed(await deps.prompt(`Are you sure you want to delete all the backup recovery points associated with the cluster: ${options.clusterName}?`))) { + await deleteBackups(deps, options.clusterName); + } + } + + deps.out(`deleting ${clusterStacks.length} cluster stack(s) ...`); + await deleteStackGroup(deps, options, clusterStacks, retryState); + + if (shouldDelete(options, "Bootstrap")) { + if (options.force === true || isConfirmed(await deps.prompt(`Are you sure you want to delete the bootstrap stack and S3 Bucket associated with the cluster: ${options.clusterName}? This action is not reversible.`))) { + await deleteBootstrapAndBucket(deps, options, config); + } + } + if (shouldDelete(options, "Databases")) await deleteDatabases(deps, options); + if (shouldDelete(options, "CloudwatchLogs")) await deleteCloudwatchLogs(deps, options); +} + +/** `delete-backups` is the same vault purge, preceded by its own confirmation. */ +export async function deleteBackupsCommand( + deps: DeleteClusterDeps, + options: Pick, +): Promise { + if (options.force !== true) { + const confirmed = await deps.prompt(`Are you sure you want to delete all the backup recovery points for cluster ${options.clusterName} ?`); + if (!isConfirmed(confirmed)) return; + } + await deleteBackups(deps, options.clusterName); +} + +/** Registers the two destructive commands on the command program supplied by CLI core. */ +export function registerDeleteClusterCommands(program: Command, deps: DeleteClusterDepsSource): void { + const resolveDeps = async ( + options: DeleteClusterOptions, + command: "delete-cluster" | "delete-backups", + ): Promise => typeof deps === "function" ? deps(options, command) : deps; + program + .command("delete-cluster") + .description("delete cluster") + .requiredOption("--cluster-name ", "Cluster Name") + .requiredOption("--aws-region ", "AWS Region") + .option("--aws-profile ", "AWS Profile Name") + .option("--delete-bootstrap", "Delete Bootstrap and S3 bucket") + .option("--delete-databases", "Delete Databases") + .option("--delete-backups", "Delete Backups") + .option("--delete-cloudwatch-logs", "Delete CloudWatch Logs") + .option("--delete-all", "Delete all") + .option("--force", "Skip confirmation prompts") + .action(async (options: DeleteClusterOptions) => { + await deleteCluster(await resolveDeps(options, "delete-cluster"), options); + }); + + program + .command("delete-backups") + .description("delete all recovery points in the cluster's backup vault") + .requiredOption("--cluster-name ", "Cluster Name") + .requiredOption("--aws-region ", "AWS Region") + .option("--aws-profile ", "AWS Profile Name") + .option("--force", "Skip confirmation prompts") + .action(async (options: DeleteClusterOptions) => { + await deleteBackupsCommand(await resolveDeps(options, "delete-backups"), options); + }); +} diff --git a/source/idea/ideactl/src/cli/commands/deploy.ts b/source/idea/ideactl/src/cli/commands/deploy.ts new file mode 100644 index 00000000..4135659f --- /dev/null +++ b/source/idea/ideactl/src/cli/commands/deploy.ts @@ -0,0 +1,220 @@ +/** + * Port of `deploy` (`app_main.py:1043-1133`) and of `bootstrap`'s CLI half. + * + * `MODULES...` are module ids; `all` may only appear on its own. The ordering, the batching and the + * per-module CDK invocation live in `deployment-helper.ts` and `cdk-invoker.ts`. + */ + +import { writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import type { Command } from 'commander'; + +import { ClusterConfig } from '../../config/cluster-config.ts'; +import { InvalidParams } from '../../config/values.ts'; +import { buildBootstrapArgv, bootstrapTags, bootstrapStackName } from './bootstrap.ts'; +import { CdkInvoker, cdkBin, clusterCdkDir, ExitWithCode, setupClusterCdkDir, type Deps } from '../cdk-invoker.ts'; +import { DeploymentHelper } from '../deployment-helper.ts'; +import { checkAwsvpcTrunking } from './upgrade.ts'; + +export interface DeployCommandOptions { + clusterName: string; + awsRegion: string; + awsProfile?: string; + terminationProtection?: string; + deploymentId?: string; + upgrade?: boolean; + forceBuildBootstrap?: boolean; + rollback?: boolean; + optimizeDeployment?: boolean; + moduleSet: string; + allowReplacement?: string[]; +} + +/** `deploy`'s argument handling: dedupe, and `all` only as the only module. */ +export function resolveRequestedModules(modules: readonly string[]): { + allModules: boolean; + moduleIds: string[] | undefined; +} { + const moduleIds: string[] = []; + let allModules = false; + for (const moduleId of modules) { + if (moduleId === 'all') allModules = true; + if (moduleIds.includes(moduleId)) continue; + moduleIds.push(moduleId); + } + if (allModules) { + if (moduleIds.length > 1) { + throw new InvalidParams('fatal error - use of "all" deployment must be the only requested module'); + } + return { allModules, moduleIds: undefined }; + } + return { allModules, moduleIds }; +} + +export function asBoolFlag(value: string | boolean | undefined, defaultValue: boolean): boolean { + if (value === undefined) return defaultValue; + if (typeof value === 'boolean') return value; + return ['true', 'yes', 'y', '1', 'on'].includes(value.toLowerCase()); +} + +export async function runDeploy(deps: Deps, modules: readonly string[], options: DeployCommandOptions): Promise { + const { allModules, moduleIds } = resolveRequestedModules(modules); + const helper = await DeploymentHelper.open({ + clusterName: options.clusterName, + awsRegion: options.awsRegion, + moduleSet: options.moduleSet, + awsProfile: options.awsProfile, + terminationProtection: asBoolFlag(options.terminationProtection, true), + deploymentId: options.deploymentId, + upgrade: options.upgrade === true, + allModules, + forceBuildBootstrap: options.forceBuildBootstrap === true, + optimizeDeployment: options.optimizeDeployment === true, + rollback: options.rollback !== false, + moduleIds, + allowReplacement: options.allowReplacement ?? [], + deps, + }); + if (helper.getDeploymentModuleNames().includes('ecs')) { + await checkAwsvpcTrunking(deps, options); + } + await helper.invoke(); +} + +/** + * `bootstrap`: renders the toolkit stack template into the cluster `_cdk` directory and runs + * `cdk bootstrap`. The rendering half is `src/cdk/stacks/bootstrap.ts`; this wires it up. + */ +export async function runBootstrap( + deps: Deps, + options: { + clusterName: string; + awsRegion: string; + awsProfile?: string; + terminationProtection?: string; + customPermissionsBoundary?: string; + cloudformationExecutionPolicies?: string; + publicAccessBlockConfiguration?: string; + moduleSet: string; + }, +): Promise { + const config = await ClusterConfig.fromDynamoDb(options.clusterName, options.awsRegion, { + moduleSet: options.moduleSet, + scan: deps.scan, + }); + const clusterBucket = config.getString('cluster.cluster_s3_bucket', undefined, { required: true }) as string; + const cdkHome = setupClusterCdkDir(options.clusterName, options.awsRegion); + const templatePath = join(cdkHome, 'cdk_toolkit_stack.yml'); + + // Re-rendered on every bootstrap, as `bootstrap_cluster` does: the permissions boundary and the + // region's ELB account id are command-line and release inputs, not stack state. + const { elbAccountIdForRegion, renderBootstrapStack } = await import('../../cdk/stacks/bootstrap.ts'); + writeFileSync( + templatePath, + renderBootstrapStack({ + clusterName: options.clusterName, + awsDnsSuffix: config.getString('cluster.aws.dns_suffix', 'amazonaws.com'), + awsElbAccountId: elbAccountIdForRegion(options.awsRegion), + inputPermissionsBoundary: options.customPermissionsBoundary, + }), + ); + deps.out( + `rendered cdk toolkit stack template for cluster: ${options.clusterName}, template: ${templatePath}`, + ); + + const invoker = new CdkInvoker({ + clusterName: options.clusterName, + awsRegion: options.awsRegion, + moduleId: 'bootstrap', + moduleName: 'bootstrap', + moduleSet: options.moduleSet, + awsProfile: options.awsProfile, + terminationProtection: asBoolFlag(options.terminationProtection, true), + deps, + }); + + const { shake256Hex } = await import('../../util/shake256.ts'); + const argv = buildBootstrapArgv({ + cdkBin: cdkBin(), + cdkAppCmd: invoker.getCdkAppCmd(), + clusterName: options.clusterName, + clusterBucket, + terminationProtection: asBoolFlag(options.terminationProtection, true), + qualifier: shake256Hex(options.clusterName, 5), + templatePath, + customPermissionsBoundary: options.customPermissionsBoundary, + cloudformationExecutionPolicies: options.cloudformationExecutionPolicies, + publicAccessBlockConfiguration: asBoolFlag(options.publicAccessBlockConfiguration, true), + // Custom tags come from `global-settings.custom_tags`, never from a flag; the cluster tag is + // added last so it wins a key collision. + tags: bootstrapTags(options.clusterName, config.getList('global-settings.custom_tags', [])), + awsProfile: options.awsProfile, + }); + + deps.out(`bootstrapping cluster CDK stack and S3 bucket: ${clusterBucket} ...`); + deps.out(`shell> ${argv.join(' ')}`); + const code = await deps.spawn(argv, { + cwd: clusterCdkDir(options.clusterName, options.awsRegion), + env: { + ...process.env, + AWS_DEFAULT_REGION: options.awsRegion, + ...(options.awsProfile === undefined + ? {} + : { + AWS_PROFILE: options.awsProfile, + AWS_DEFAULT_PROFILE: options.awsProfile, + }), + }, + }); + if (code !== 0) throw new ExitWithCode(code); + deps.out(`bootstrapped ${bootstrapStackName(options.clusterName)}`); +} + +export function registerDeployCommands(program: Command, deps: Deps): void { + program + .command('bootstrap') + .description('bootstrap cluster') + .requiredOption('--cluster-name ', 'Cluster Name') + .option('--aws-profile ', 'AWS Profile Name') + .requiredOption('--aws-region ', 'AWS Region') + .option('--termination-protection ', 'Set termination protection to true or false. Default: true', 'true') + .option('--custom-permissions-boundary ', 'Name of a custom permissions boundary to pass to CDK (Default to none)', '') + .option('--cloudformation-execution-policies ', 'Customize CDK CloudFormation execution policies', '') + .option( + '--public-access-block-configuration ', + 'Include S3 Block Public Access configuration for CDK staging bucket. Set to false for restricted S3 environments.', + 'true', + ) + .option('--module-set ', 'Name of the ModuleSet. Default: default', 'default') + .action(async (options: Parameters[1]) => { + await runBootstrap(deps, options); + }); + + program + .command('deploy') + .description('deploy modules. Use `all` as the module id to deploy all modules.') + .requiredOption('--cluster-name ', 'Cluster Name') + .requiredOption('--aws-region ', 'AWS Region') + .option('--aws-profile ', 'AWS Profile Name') + .option('--termination-protection ', 'Set termination protection to true or false. Default: true', 'true') + .option('--deployment-id ', 'A UUID to identify the deployment.') + .option('--upgrade', 'Upgrade the module by re-running the CDK stack if the module has already been deployed.') + .option( + '--force-build-bootstrap', + 'If the bootstrap package directory for a given DeploymentId already exists, the directory will be deleted and rendered again.', + ) + .option('--rollback', 'Rollback stack to stable state on failure. Default.', true) + .option('--no-rollback', 'Do not roll back on failure, to iterate more rapidly.') + .option('--optimize-deployment', 'Deploy applicable stacks in parallel.') + .option('--module-set ', 'Name of the ModuleSet. Default: default', 'default') + .option( + '--allow-replacement ', + 'Accept a change-set entry the deploy guard would refuse, by logical ID. Repeatable.', + (value: string, previous: string[] = []) => [...previous, value], + ) + .argument('', 'module ids, or `all`') + .action(async (modules: string[], options: DeployCommandOptions) => { + await runDeploy(deps, modules, options); + }); +} diff --git a/source/idea/ideactl/src/cli/commands/directoryservice.ts b/source/idea/ideactl/src/cli/commands/directoryservice.ts new file mode 100644 index 00000000..e8003960 --- /dev/null +++ b/source/idea/ideactl/src/cli/commands/directoryservice.ts @@ -0,0 +1,117 @@ +/** + * Directory service operator commands. + * + * Secret creation deliberately does not read the cluster configuration because + * it is used before a cluster has been initialized. + */ + +import { Command } from "commander"; + +import { ClusterConfigError, isEmpty } from "../../config/cluster-config.ts"; + +export interface DirectorySecretsApi { + createSecret(input: { + Name: string; + Description: string; + SecretString: string; + Tags: Array<{ Key: string; Value: string }>; + KmsKeyId?: string; + }): Promise<{ ARN?: string }>; +} + +export interface DirectoryServiceDeps { + secrets: DirectorySecretsApi; + promptCredentials?: (defaults: { purpose?: string; username?: string; password?: string }) => Promise<{ + purpose: string; + username: string; + password: string; + }>; + out(line: string): void; +} + +/** Builds dependencies for the profile selected by one directory-service command. */ +export type DirectoryServiceDepsFactory = (options: { + clusterName: string; + awsRegion: string; + awsProfile?: string; +}) => Promise; + +type DirectoryServiceDepsSource = DirectoryServiceDeps | DirectoryServiceDepsFactory; + +export interface CreateServiceAccountSecretsOptions { + clusterName: string; + username?: string; + password?: string; + kmsKeyId?: string; + purpose?: string; +} + +/** Create the paired username and password secrets in a fixed order. */ +export async function createServiceAccountSecrets( + deps: DirectoryServiceDeps, + options: CreateServiceAccountSecretsOptions, +): Promise<{ purpose: string; usernameSecretArn: string | undefined; passwordSecretArn: string | undefined }> { + let { purpose, username, password } = options; + if (isEmpty(username) || isEmpty(password)) { + if (deps.promptCredentials === undefined) { + throw new ClusterConfigError("username and password are required when interactive credentials are unavailable"); + } + const entered = await deps.promptCredentials({ purpose, username, password }); + purpose = entered.purpose; + username = entered.username; + password = entered.password; + } + // A supplied credential pair with no purpose uses this literal in its secret name. + const resolvedPurpose = purpose === undefined ? "None" : purpose; + if (isEmpty(username) || isEmpty(password)) throw new ClusterConfigError("username and password are required"); + const resolvedUsername = username as string; + const resolvedPassword = password as string; + const tags = [ + { Key: "idea:ClusterName", Value: options.clusterName }, + { Key: "idea:ModuleName", Value: "directoryservice" }, + { Key: "idea:ModuleId", Value: "directoryservice" }, + ]; + const common = { + Tags: tags, + ...(isEmpty(options.kmsKeyId) ? {} : { KmsKeyId: options.kmsKeyId as string }), + }; + const usernameSecret = await deps.secrets.createSecret({ + Name: `${options.clusterName}-directoryservice-${resolvedPurpose}-username`, + Description: `DirectoryService ${resolvedPurpose} username, Cluster: ${options.clusterName}`, + SecretString: resolvedUsername, + ...common, + }); + const passwordSecret = await deps.secrets.createSecret({ + Name: `${options.clusterName}-directoryservice-${resolvedPurpose}-password`, + Description: `DirectoryService ${resolvedPurpose} password, Cluster: ${options.clusterName}`, + SecretString: resolvedPassword, + ...common, + }); + return { purpose: resolvedPurpose, usernameSecretArn: usernameSecret.ARN, passwordSecretArn: passwordSecret.ARN }; +} + +/** Register the `directoryservice` command group. */ +export function registerDirectoryServiceCommands(program: Command, deps: DirectoryServiceDepsSource): Command { + const resolveDeps = async (options: CreateServiceAccountSecretsOptions & { + awsRegion: string; + awsProfile?: string; + }): Promise => typeof deps === "function" ? deps(options) : deps; + const group = program.command("directoryservice").description("directory service commands"); + group.command("create-service-account-secrets") + .requiredOption("--cluster-name ") + .requiredOption("--aws-region ") + .option("--aws-profile ") + .option("--username ") + .option("--password ") + .option("--kms-key-id ") + .option("--purpose ") + .action(async (options: CreateServiceAccountSecretsOptions & { awsRegion: string; awsProfile?: string }) => { + const actionDeps = await resolveDeps(options); + const result = await createServiceAccountSecrets(actionDeps, options); + actionDeps.out(`directory service ${result.purpose} secrets created successfully: `); + actionDeps.out(`Account Purpose: ${result.purpose}`); + actionDeps.out(`Username Secret ARN: ${result.usernameSecretArn}`); + actionDeps.out(`Password Secret ARN: ${result.passwordSecretArn}`); + }); + return group; +} diff --git a/source/idea/ideactl/src/cli/commands/migrate.ts b/source/idea/ideactl/src/cli/commands/migrate.ts new file mode 100644 index 00000000..9135b097 --- /dev/null +++ b/source/idea/ideactl/src/cli/commands/migrate.ts @@ -0,0 +1,716 @@ +/** + * Drive the one-phase control-plane migration in its fixed serial order. + * + * Cluster reads and writes are injected. Durable progress uses the shared + * upgrade journal, with exact migration step markers stored as snapshots. + */ + +import type { Command } from "commander"; + +import { + UpgradeStateJournal, + readUpgradeState, + type UpgradeBoundary, + type UpgradeOperationRecord, + type UpgradeStateObjectApi, +} from "../../config/upgrade-state.ts"; + +/** The durable boundaries and operator-facing actions from the migration plan. */ +export const MIGRATION_STEPS = [ + { + id: "PREFLIGHT_PASSED", + precondition: "Every read-only migration predicate is known and green", + action: "Record the target-bound pre-flight manifest", + }, + { + id: "OPERATION_STARTED", + precondition: "The pre-flight fingerprints are current and the operation lock is held", + action: "Capture exact rollback inputs and identify every required package", + }, + { + id: "ADMISSION_CLOSED", + precondition: "Maintenance and PBS administration paths are reachable", + action: "Announce maintenance, close scheduler admission, and wait for every job inventory to drain", + }, + { + id: "WORKLOAD_DRAINED", + precondition: "Maintenance is enabled, scheduling is disabled, and all submission queues are disabled", + action: "Confirm empty job inventories and retire compute nodes carrying the legacy scheduler address", + }, + { + id: "LEGACY_SCHEDULER_CAPTURED", + precondition: "Workload inventories are empty and scheduler stop, restart, archive, and restore paths are ready", + action: "Stop the legacy scheduler and capture a read-back-verified spool archive", + }, + { + id: "SOFTWARE_STACKS_RECONCILED", + precondition: "The legacy scheduler is stopped and its verified archive and host are retained", + action: "Apply the recorded desktop software-stack reconciliation plan one row at a time", + }, + { + id: "CONFIGURATION_STAGED", + precondition: "Software-stack rows and any required search index changes are reconciled", + action: "Stage target configuration with container routing disabled and scheduler desired count zero", + }, + { + id: "PROVIDERS_COMMITTED", + precondition: "Configuration snapshots and row journals are complete and admission remains closed", + action: "Deploy provider stacks serially in the recorded dependency order", + }, + { + id: "SCHEDULER_DNS_RETAINED", + precondition: "Provider stacks are stable and the retain-only scheduler change set has no other effect", + action: "Commit and verify retention of the existing scheduler DNS record", + }, + { + id: "ECS_CONFIGURATION_ACTIVE", + precondition: "The scheduler DNS record is retained and the legacy scheduler remains stopped", + action: "Enable container configuration with the stable scheduler name and scheduler desired count zero", + }, + { + id: "ECS_STAGED", + precondition: "Container configuration is active and the scheduler desired and running counts are zero", + action: "Deploy and prove container services while production routes remain on legacy targets", + }, + { + id: "PBS_STATE_SEEDED", + precondition: "Container targets are healthy, the scheduler is at zero, and the source archive is verified", + action: "Restore and verify the scheduler state in the target shared directory", + }, + { + id: "ECS_SCHEDULER_READY", + precondition: "Shared scheduler state is verified and the legacy scheduler remains stopped", + action: "Start and prove exactly one container scheduler against the restored state", + }, + { + id: "CLUSTER_MANAGER_ROUTED", + precondition: "The container cluster-manager targets are healthy and the route-only change is safe", + action: "Route cluster-manager production endpoints to containers and prove the production path", + }, + { + id: "CLUSTER_MANAGER_LEGACY_REMOVED", + precondition: "Cluster-manager production routing is proved on containers and removals match the allow-list", + action: "Remove only legacy cluster-manager host resources and prove the production path again", + }, + { + id: "VDC_ROUTED", + precondition: "Cluster-manager removal is stable, VDC container targets are healthy, and route-only changes are safe", + action: "Route controller, broker, and gateway endpoints to containers and prove desktop reconnect", + }, + { + id: "VDC_LEGACY_REMOVED", + precondition: "VDC production routing is proved on containers and removals preserve session infrastructure", + action: "Remove only legacy VDC control-plane host resources and prove the production paths again", + }, + { + id: "SCHEDULER_ROUTED", + precondition: "VDC removal is stable and container scheduler DNS, targets, and API health are proved", + action: "Route scheduler endpoints to the container scheduler while retaining the stopped legacy host", + }, + { + id: "SCHEDULER_LEGACY_REMOVED", + precondition: "Scheduler routing is proved, the archive is verified, and removals match the allow-list", + action: "Remove legacy scheduler host resources while preserving task-owned DNS and shared scheduler state", + }, + { + id: "TARGET_PROVED", + precondition: "All target stacks and routes are stable, and admission remains closed", + action: "Reconcile protection and values, then run every application and replacement proof", + }, + { + id: "ADMISSION_REOPENED", + precondition: "The complete target proof is retained and every admission control has a recorded prior value", + action: "Restore PBS controls, clear maintenance, and verify both admission paths", + }, + { + id: "OPERATION_COMPLETED", + precondition: "Admission is verified open and final cluster health is green", + action: "Record final fingerprints, complete the operation, and release its lock", + }, +] as const; + +export type MigrationStepId = (typeof MIGRATION_STEPS)[number]["id"]; +export type ExecutableMigrationStepId = Exclude; + +/** Values fixed for one new or resumed migration operation. */ +export interface MigrationContext { + clusterName: string; + awsRegion: string; + awsProfile?: string; + targetVersion: string; + targetBaseOs: string; + imageDigest: string; + moduleSet: string; + selectedModules: readonly string[]; + deploymentId: string; + resuming: boolean; + /** Operator acceptance of the pre-flight target-template comparison. */ + acceptTemplateComparison?: string; + /** Operator acceptance of the configuration rows the run would overwrite. */ + acceptDrift?: string; +} + +/** A safe, printable observation. It must not contain secret values. */ +export interface MigrationObservation { + ok: boolean; + detail: string; +} + +/** Read-back state required before the scheduler closure can commit. */ +export interface SchedulerClosureObservation { + detail: string; + maintenanceEnabled: boolean; + schedulingEnabled: boolean; + enabledQueues: readonly string[]; + queuedJobs: number; + provisioningJobs: number; + runningJobs: number; +} + +/** + * Result of reconciling a step that has a durable started marker but no + * committed marker. + */ +export interface MigrationReconciliation { + state: "committed" | "retryable" | "uncertain"; + detail: string; + schedulerClosure?: SchedulerClosureObservation; +} + +/** Injected cluster operations used by the serial driver. */ +export interface MigrationStepExecutor { + checkPrecondition( + step: MigrationStepId, + context: Readonly, + ): Promise; + execute( + step: ExecutableMigrationStepId, + context: Readonly, + ): Promise; + closeScheduler(context: Readonly): Promise; + reconcile( + step: MigrationStepId, + context: Readonly, + ): Promise; +} + +/** Dependencies whose live implementations are supplied by the command tree. */ +export interface MigrateDeps { + stateObjects: UpgradeStateObjectApi; + steps: MigrationStepExecutor; + uuid(): string; + targetVersion(): string; + out(line: string): void; + now?: () => number; +} + +/** Command options for a new operation or a durable resume. */ +export interface MigrateOptions { + clusterName: string; + awsRegion: string; + awsProfile?: string; + stateBucket: string; + targetBaseOs?: string; + imageDigest?: string; + moduleSet: string; + selectedModules?: readonly string[]; + deploymentId?: string; + resume?: string; + acceptTemplateComparison?: string; + acceptDrift?: string; +} + +/** A refusal is safe to show as a single operator-facing command error. */ +export class MigrationRefusedError extends Error {} + +interface MigrationBoundaryGroup { + boundary: UpgradeBoundary; + steps: readonly MigrationStepId[]; +} + +const MIGRATION_BOUNDARY_GROUPS: readonly MigrationBoundaryGroup[] = [ + { boundary: "preflight", steps: ["PREFLIGHT_PASSED", "OPERATION_STARTED"] }, + { + boundary: "eol-software-stacks", + steps: [ + "ADMISSION_CLOSED", + "WORKLOAD_DRAINED", + "LEGACY_SCHEDULER_CAPTURED", + "SOFTWARE_STACKS_RECONCILED", + ], + }, + { boundary: "values-file", steps: ["CONFIGURATION_STAGED"] }, + { boundary: "global-settings", steps: ["PROVIDERS_COMMITTED"] }, + { boundary: "full-configuration", steps: ["SCHEDULER_DNS_RETAINED", "ECS_CONFIGURATION_ACTIVE"] }, + { boundary: "host-settings", steps: ["ECS_STAGED", "PBS_STATE_SEEDED", "ECS_SCHEDULER_READY"] }, + { + boundary: "protection-sweep", + steps: [ + "CLUSTER_MANAGER_ROUTED", + "CLUSTER_MANAGER_LEGACY_REMOVED", + "VDC_ROUTED", + "VDC_LEGACY_REMOVED", + "SCHEDULER_ROUTED", + "SCHEDULER_LEGACY_REMOVED", + ], + }, + { boundary: "module-deployments", steps: ["TARGET_PROVED"] }, + { boundary: "finalization", steps: ["ADMISSION_REOPENED", "OPERATION_COMPLETED"] }, +]; + +const INPUTS_SNAPSHOT = "migration:inputs"; +const INPUTS_SOURCE = "migration-command"; +const IMAGE_DIGEST = /^[^\s]+@sha256:[0-9a-f]{64}$/; + +function requireText(value: string | undefined, label: string): string { + if (value === undefined || value.trim() === "") { + throw new TypeError(`${label} must be a non-empty string`); + } + return value; +} + +function requireUniqueModules(modules: readonly string[] | undefined): string[] { + if (modules === undefined || modules.length === 0) { + throw new TypeError("At least one selected module is required for a new migration"); + } + const result = modules.map((moduleId) => requireText(moduleId, "Selected module")); + if (new Set(result).size !== result.length) { + throw new TypeError("Selected modules must not contain duplicates"); + } + return result; +} + +function requireImageDigest(value: string | undefined): string { + const image = requireText(value, "Image digest"); + if (!IMAGE_DIGEST.test(image)) { + throw new TypeError("Image must be an immutable reference ending in @sha256 followed by 64 lowercase hexadecimal characters"); + } + return image; +} + +function startedMarker(step: MigrationStepId): string { + return `migration:${step}:started`; +} + +function committedMarker(step: MigrationStepId): string { + return `migration:${step}:committed`; +} + +function hasSnapshot(record: UpgradeOperationRecord, name: string): boolean { + return record.snapshots.some((snapshot) => snapshot.name === name); +} + +function committedSteps(record: UpgradeOperationRecord): MigrationStepId[] { + return MIGRATION_STEPS + .map((step) => step.id) + .filter((step) => hasSnapshot(record, committedMarker(step))); +} + +function stepById(stepId: MigrationStepId): (typeof MIGRATION_STEPS)[number] { + const step = MIGRATION_STEPS.find((candidate) => candidate.id === stepId); + if (step === undefined) throw new TypeError(`Unknown migration step: ${stepId}`); + return step; +} + +function validateObservation(observation: MigrationObservation, label: string): void { + if (typeof observation.ok !== "boolean") { + throw new TypeError(`${label} must include a boolean ok value`); + } + requireText(observation.detail, `${label} detail`); +} + +function validateCount(value: number, label: string): void { + if (!Number.isSafeInteger(value) || value < 0) { + throw new TypeError(`${label} must be a non-negative integer`); + } +} + +function schedulerClosureFailure(observation: SchedulerClosureObservation): string | undefined { + requireText(observation.detail, "Scheduler closure detail"); + if (typeof observation.maintenanceEnabled !== "boolean" || typeof observation.schedulingEnabled !== "boolean") { + throw new TypeError("Scheduler closure state must include boolean maintenance and scheduling values"); + } + if (!Array.isArray(observation.enabledQueues) || observation.enabledQueues.some((queue) => typeof queue !== "string" || queue === "")) { + throw new TypeError("Scheduler closure enabledQueues must contain non-empty strings"); + } + validateCount(observation.queuedJobs, "Queued job count"); + validateCount(observation.provisioningJobs, "Provisioning job count"); + validateCount(observation.runningJobs, "Running job count"); + if (!observation.maintenanceEnabled) return "maintenance is not enabled"; + if (observation.schedulingEnabled) return "PBS scheduling is still enabled"; + if (observation.enabledQueues.length > 0) { + return `submission queues are still enabled: ${observation.enabledQueues.join(", ")}`; + } + const activeJobs = observation.queuedJobs + observation.provisioningJobs + observation.runningJobs; + if (activeJobs > 0) { + return `the queue is not drained: ${observation.queuedJobs} queued, ${observation.provisioningJobs} provisioning, ${observation.runningJobs} running`; + } + return undefined; +} + +function renderSchedulerObservation(observation: SchedulerClosureObservation): string { + return [ + observation.detail, + `maintenance=${observation.maintenanceEnabled}`, + `scheduling=${observation.schedulingEnabled}`, + `enabledQueues=${observation.enabledQueues.length}`, + `queued=${observation.queuedJobs}`, + `provisioning=${observation.provisioningJobs}`, + `running=${observation.runningJobs}`, + ].join("; "); +} + +async function checkPrecondition( + deps: MigrateDeps, + context: Readonly, + stepId: MigrationStepId, +): Promise { + const step = stepById(stepId); + deps.out(`CHECK [${step.id}] ${step.precondition}`); + const observation = await deps.steps.checkPrecondition(step.id, context); + validateObservation(observation, `Precondition ${step.id}`); + deps.out(`OBSERVED [${step.id}] ${observation.ok ? "PASS" : "FAIL"}: ${observation.detail}`); + if (!observation.ok) { + throw new MigrationRefusedError(`${step.id} refused: ${observation.detail}`); + } + return observation; +} + +async function recordMarker( + journal: UpgradeStateJournal, + name: string, + detail: string, +): Promise { + await journal.recordSnapshot({ + name, + source: INPUTS_SOURCE, + body: JSON.stringify({ detail }), + }); +} + +async function markProvedModules( + deps: MigrateDeps, + journal: UpgradeStateJournal, +): Promise { + for (const moduleId of journal.record().operation.selectedModules) { + await journal.runModule(moduleId, async () => { + deps.out(`OBSERVED [TARGET_PROVED] module ${moduleId} is included in the retained target proof`); + }); + } +} + +async function commitReconciledStep( + deps: MigrateDeps, + journal: UpgradeStateJournal, + stepId: MigrationStepId, + reconciliation: MigrationReconciliation, +): Promise { + requireText(reconciliation.detail, `Reconciliation ${stepId} detail`); + if ( + reconciliation.state !== "committed" && + reconciliation.state !== "retryable" && + reconciliation.state !== "uncertain" + ) { + throw new TypeError(`Reconciliation ${stepId} has an invalid state`); + } + deps.out(`OBSERVED [${stepId}] reconciliation ${reconciliation.state}: ${reconciliation.detail}`); + if (reconciliation.state === "uncertain") { + throw new MigrationRefusedError( + `${stepId} has a started marker but its external result is uncertain: ${reconciliation.detail}`, + ); + } + if (reconciliation.state === "retryable") return false; + if (stepId === "ADMISSION_CLOSED") { + if (reconciliation.schedulerClosure === undefined) { + throw new MigrationRefusedError("ADMISSION_CLOSED reconciliation did not include scheduler read-back state"); + } + const failure = schedulerClosureFailure(reconciliation.schedulerClosure); + deps.out(`OBSERVED [ADMISSION_CLOSED] ${renderSchedulerObservation(reconciliation.schedulerClosure)}`); + if (failure !== undefined) throw new MigrationRefusedError(`ADMISSION_CLOSED refused: ${failure}`); + } + if (stepId === "TARGET_PROVED") await markProvedModules(deps, journal); + await recordMarker(journal, committedMarker(stepId), reconciliation.detail); + return true; +} + +async function executeStep( + deps: MigrateDeps, + journal: UpgradeStateJournal, + context: Readonly, + stepId: MigrationStepId, + prechecked?: MigrationObservation, +): Promise { + if (hasSnapshot(journal.record(), committedMarker(stepId))) return; + + const precondition = prechecked ?? await checkPrecondition(deps, context, stepId); + const started = hasSnapshot(journal.record(), startedMarker(stepId)); + if (started) { + deps.out(`RECONCILE [${stepId}] A started marker exists without a committed marker`); + const reconciliation = await deps.steps.reconcile(stepId, context); + if (await commitReconciledStep(deps, journal, stepId, reconciliation)) return; + } else { + await recordMarker(journal, startedMarker(stepId), precondition.detail); + } + + const step = stepById(stepId); + if (stepId === "PREFLIGHT_PASSED") { + deps.out(`RUN [${step.id}] ${step.action}`); + await recordMarker(journal, committedMarker(stepId), precondition.detail); + return; + } + + if (stepId === "ADMISSION_CLOSED") { + deps.out( + "ANNOUNCE [ADMISSION_CLOSED] The scheduler is closing. Maintenance, PBS scheduling, and every submission queue must remain closed until the migration is proved.", + ); + deps.out(`RUN [${step.id}] ${step.action}`); + const observation = await deps.steps.closeScheduler(context); + const failure = schedulerClosureFailure(observation); + deps.out(`OBSERVED [${step.id}] ${renderSchedulerObservation(observation)}`); + if (failure !== undefined) throw new MigrationRefusedError(`${step.id} refused: ${failure}`); + await recordMarker(journal, committedMarker(stepId), observation.detail); + return; + } + + deps.out(`RUN [${step.id}] ${step.action}`); + const observation = await deps.steps.execute(stepId, context); + validateObservation(observation, `Step ${step.id}`); + deps.out(`OBSERVED [${step.id}] ${observation.ok ? "PASS" : "FAIL"}: ${observation.detail}`); + if (!observation.ok) { + throw new MigrationRefusedError(`${step.id} refused after execution: ${observation.detail}`); + } + if (stepId === "TARGET_PROVED") await markProvedModules(deps, journal); + await recordMarker(journal, committedMarker(stepId), observation.detail); +} + +function parseInputs(record: UpgradeOperationRecord): { imageDigest: string } | undefined { + const snapshot = record.snapshots.find((candidate) => candidate.name === INPUTS_SNAPSHOT); + if (snapshot === undefined) return undefined; + let value: unknown; + try { + value = JSON.parse(snapshot.body); + } catch (error) { + throw new MigrationRefusedError( + `The migration inputs record is invalid JSON: ${error instanceof Error ? error.message : String(error)}`, + ); + } + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new MigrationRefusedError("The migration inputs record must be an object"); + } + const imageDigest = (value as Record)["imageDigest"]; + if (typeof imageDigest !== "string") { + throw new MigrationRefusedError("The migration inputs record has no image digest"); + } + return { imageDigest: requireImageDigest(imageDigest) }; +} + +async function recordInputs( + journal: UpgradeStateJournal, + context: Readonly, +): Promise { + await journal.recordSnapshot({ + name: INPUTS_SNAPSHOT, + source: INPUTS_SOURCE, + body: JSON.stringify({ imageDigest: context.imageDigest }), + }); +} + +function newContext( + deps: MigrateDeps, + options: MigrateOptions, +): MigrationContext { + if (options.resume !== undefined) { + throw new TypeError("newContext cannot create a resumed operation"); + } + if (options.deploymentId !== undefined && options.deploymentId.trim() === "") { + throw new TypeError("Deployment ID must be a non-empty string"); + } + return { + clusterName: requireText(options.clusterName, "Cluster name"), + awsRegion: requireText(options.awsRegion, "AWS region"), + awsProfile: options.awsProfile, + targetVersion: requireText(deps.targetVersion(), "Target version"), + targetBaseOs: requireText(options.targetBaseOs, "Target base OS"), + imageDigest: requireImageDigest(options.imageDigest), + moduleSet: requireText(options.moduleSet, "Module set"), + selectedModules: requireUniqueModules(options.selectedModules), + deploymentId: options.deploymentId ?? requireText(deps.uuid(), "Generated deployment ID"), + resuming: false, + ...acceptances(options), + }; +} + +/** Acceptances are per invocation: each one names the exact report it accepts. */ +function acceptances(options: MigrateOptions): Pick { + return { + ...(options.acceptTemplateComparison === undefined + ? {} + : { acceptTemplateComparison: requireText(options.acceptTemplateComparison, "Accepted template comparison") }), + ...(options.acceptDrift === undefined + ? {} + : { acceptDrift: requireText(options.acceptDrift, "Accepted drift report") }), + }; +} + +function resumedContext( + deps: MigrateDeps, + options: MigrateOptions, + record: UpgradeOperationRecord, +): MigrationContext { + const installedVersion = requireText(deps.targetVersion(), "Target version"); + if (record.operation.targetVersion !== installedVersion) { + throw new MigrationRefusedError( + `Migration ${record.operation.deploymentId} targets release ${record.operation.targetVersion}, but this release is ${installedVersion}`, + ); + } + const inputs = parseInputs(record); + const imageDigest = inputs?.imageDigest ?? requireImageDigest(options.imageDigest); + if (options.imageDigest !== undefined && requireImageDigest(options.imageDigest) !== imageDigest) { + throw new MigrationRefusedError("The supplied image digest does not match the durable migration record"); + } + return { + clusterName: record.operation.clusterName, + awsRegion: record.operation.awsRegion, + awsProfile: options.awsProfile, + targetVersion: record.operation.targetVersion, + targetBaseOs: record.operation.targetBaseOs, + imageDigest, + moduleSet: record.operation.moduleSet, + selectedModules: [...record.operation.selectedModules], + deploymentId: record.operation.deploymentId, + resuming: true, + ...acceptances(options), + }; +} + +function validateCommandIdentity(options: MigrateOptions): void { + requireText(options.clusterName, "Cluster name"); + requireText(options.awsRegion, "AWS region"); + requireText(options.stateBucket, "State bucket"); + if (options.awsProfile !== undefined) requireText(options.awsProfile, "AWS profile"); + if (options.resume !== undefined) { + requireText(options.resume, "Resume deployment ID"); + if (options.deploymentId !== undefined) { + throw new TypeError("--resume and --deployment-id cannot be used together"); + } + } +} + +function renderProgress(deps: MigrateDeps, journal: UpgradeStateJournal): void { + const report = journal.report(); + const committed = committedSteps(journal.record()); + deps.out( + `PROGRESS deployment=${report.deploymentId ?? "unknown"} status=${report.status ?? "unknown"} next=${report.stoppedAt ?? "completion"}`, + ); + deps.out(`PROGRESS committed migration steps: ${committed.length === 0 ? "none" : committed.join(", ")}`); +} + +/** + * Run or resume the complete one-phase migration. + * + * Every mutating action gets a started marker before execution and a committed + * marker only after the executor returns its owning-service observation. + */ +export async function migrateCluster(deps: MigrateDeps, options: MigrateOptions): Promise { + validateCommandIdentity(options); + const location = { bucket: options.stateBucket }; + const holderId = requireText(deps.uuid(), "Journal holder ID"); + let journal: UpgradeStateJournal; + let context: MigrationContext; + let initialPreflight: MigrationObservation | undefined; + + if (options.resume === undefined) { + context = newContext(deps, options); + initialPreflight = await checkPrecondition(deps, context, "PREFLIGHT_PASSED"); + journal = await UpgradeStateJournal.start( + deps.stateObjects, + location, + { + clusterName: context.clusterName, + awsRegion: context.awsRegion, + targetVersion: context.targetVersion, + targetBaseOs: context.targetBaseOs, + moduleSet: context.moduleSet, + selectedModules: context.selectedModules, + deploymentId: context.deploymentId, + }, + { holderId, now: deps.now }, + ); + await recordInputs(journal, context); + } else { + const record = await readUpgradeState(deps.stateObjects, location); + if (record === undefined) { + throw new MigrationRefusedError(`No durable record exists for migration ${options.resume}`); + } + if ( + record.operation.clusterName !== options.clusterName || + record.operation.awsRegion !== options.awsRegion || + record.operation.deploymentId !== options.resume + ) { + throw new MigrationRefusedError( + `The durable record does not belong to ${options.clusterName}/${options.awsRegion}/${options.resume}`, + ); + } + context = resumedContext(deps, options, record); + journal = await UpgradeStateJournal.resume( + deps.stateObjects, + location, + { + clusterName: context.clusterName, + awsRegion: context.awsRegion, + deploymentId: context.deploymentId, + }, + { holderId, now: deps.now }, + ); + if (parseInputs(journal.record()) === undefined) await recordInputs(journal, context); + } + + renderProgress(deps, journal); + for (const group of MIGRATION_BOUNDARY_GROUPS) { + await journal.runBoundary(group.boundary, async () => { + for (const step of group.steps) { + const prechecked = step === "PREFLIGHT_PASSED" ? initialPreflight : undefined; + await executeStep(deps, journal, context, step, prechecked); + } + }); + } + await journal.complete(); + deps.out(`COMPLETE migration ${context.deploymentId}`); +} + +function collectModule(value: string, previous: string[] = []): string[] { + return [...previous, value]; +} + +interface RegisteredMigrateOptions extends Omit { + selectedModule?: string[]; +} + +/** Register the migration command with replayable dependencies. */ +export function registerMigrateCommands(program: Command, deps: MigrateDeps): void { + program + .command("migrate") + .description("run or resume the one-phase control-plane migration") + .requiredOption("--cluster-name ", "Cluster Name") + .requiredOption("--aws-region ", "AWS Region") + .option("--aws-profile ", "AWS Profile Name") + .requiredOption("--state-bucket ", "Bucket containing the durable operation record") + .option("--target-base-os ", "Target Base OS for a new migration") + .option("--image-digest ", "Immutable control-plane image digest") + .option("--module-set ", "Name of the ModuleSet", "default") + .option("--selected-module ", "Deployable module ID, repeat for each module", collectModule) + .option("--deployment-id ", "Deployment ID for a new migration") + .option("--resume ", "Resume an incomplete migration") + .option( + "--accept-template-comparison ", + "Accept the target-template comparison named by this fingerprint", + ) + .option("--accept-drift ", "Accept overwriting the configuration rows named by this fingerprint") + .action(async (commandOptions: RegisteredMigrateOptions) => { + await migrateCluster(deps, { + ...commandOptions, + imageDigest: commandOptions.imageDigest, + selectedModules: commandOptions.selectedModule, + }); + }); +} diff --git a/source/idea/ideactl/src/cli/commands/replace.ts b/source/idea/ideactl/src/cli/commands/replace.ts new file mode 100644 index 00000000..a6817af6 --- /dev/null +++ b/source/idea/ideactl/src/cli/commands/replace.ts @@ -0,0 +1,213 @@ +/** + * `replace`: the deliberate path to a new stateful component. + * + * Replacing something that holds state is an intentional act with a consequence an operator has to + * accept in advance. It is never something an upgrade or a migration does on the way past, so it + * has its own verb rather than a flag on theirs. `upgrade` and `deploy` keep refusing every + * replacement through the change-set guard, and the `UpdateReplacePolicy: Retain` on the resources + * an upgrade has never replaced is the backstop under that, for a deploy that reaches + * CloudFormation by some other route. + * + * Two components here, the jump host and the scheduler host, are replaced by an ordinary upgrade + * already and carry no retain policy. They are listed because an operator may want to replace one + * on purpose, and the consequence is worth reading either way. + * + * This command does not invent a replacement. CloudFormation replaces a resource when a property + * that cannot be changed in place changes, and `forcedBy` below names those properties for each + * component. The operator changes the setting, then runs this, and this permits the replacement the + * new configuration already implies, for that one component, once. + */ + +import type { Command } from 'commander'; + +import { DeploymentHelper } from '../deployment-helper.ts'; +import { ExitWithCode, type Deps } from '../cdk-invoker.ts'; + +export interface ReplaceableComponent { + /** What an operator calls it. */ + component: string; + /** The module whose stack builds it. */ + moduleName: string; + /** The CloudFormation type, which is how the allowance is scoped. */ + resourceType: string; + /** What is lost. Written for the consequence, not for the mechanism. */ + consequence: string; + /** The properties whose change makes CloudFormation replace it, and where they come from. */ + forcedBy: string; + /** Set when there is no version of this that ends well. The command refuses and explains. */ + refuse?: string; +} + +export const REPLACEABLE_COMPONENTS: readonly ReplaceableComponent[] = [ + { + component: 'jump-host', + moduleName: 'bastion-host', + resourceType: 'AWS::EC2::Instance', + consequence: + 'The jump host is rebuilt with a new host identity, so every operator who has connected to it ' + + 'before finds their stored host key no longer matches and is asked to accept a new one, and ' + + 'automation that pins the old key stops working until it is updated. An ordinary upgrade ' + + 'already replaces this host, about twenty times over the life of a cluster, so the new key ' + + 'is a familiar interruption rather than a rare one. Keeping the identity across a ' + + 'replacement is being designed separately; until that lands, expect the prompt.', + forcedBy: 'ImageId, KeyName, LaunchTemplate or NetworkInterfaces, from bastion-host.instance_ami and cluster.network.ssh_key_pair', + }, + { + component: 'search-domain', + moduleName: 'analytics', + resourceType: 'AWS::OpenSearchService::Domain', + consequence: + 'The indexed history is destroyed. Every job record, desktop session record and application ' + + 'document already indexed is gone, and none of it is rebuilt from the cluster settings: the ' + + 'new domain starts empty. Unless the intent is to start the index over from nothing, this is ' + + 'almost certainly the wrong action.', + forcedBy: 'DomainName or EngineMode, from analytics.opensearch.domain_name', + }, + { + component: 'directory', + moduleName: 'directoryservice', + resourceType: 'AWS::DirectoryService::MicrosoftAD', + consequence: + 'Every machine joined to the directory is severed from it. Compute nodes, virtual desktops ' + + 'and the control-plane hosts lose their domain membership and have to be rejoined, and the ' + + 'user and group objects that live in the directory rather than in the cluster settings are ' + + 'gone with it. Nobody authenticates against the cluster until the rejoin is finished.', + forcedBy: 'Name, ShortName, Edition, Password, CreateAlias or VpcSettings, from directoryservice.name, .ad_short_name, .ad_edition and the cluster network settings', + }, + { + component: 'user-pool', + moduleName: 'identity-provider', + resourceType: 'AWS::Cognito::UserPool', + consequence: + 'Every account in the pool is lost, along with every multi-factor enrolment. Each person has ' + + 'to be created again and enrol again before they can sign in, and the sign-in integrations ' + + 'that name the old pool stop working until they are pointed at the new one.', + forcedBy: + 'no property of a user pool is marked replacing in the CloudFormation resource specification, ' + + 'so a replacement here comes from the resource moving or being rebuilt rather than from a setting', + }, + { + component: 'scheduler-host', + moduleName: 'scheduler', + resourceType: 'AWS::EC2::Instance', + consequence: + 'The machine the batch server runs on is rebuilt, and whatever the server holds locally goes ' + + 'with it. Execution nodes configured with the current server name stop finding the server, ' + + 'so work in flight is lost rather than requeued. An ordinary upgrade already replaces this ' + + 'host, which is why an upgrade is drained and announced; running it on its own needs the ' + + 'same drain and the same announcement.', + forcedBy: 'ImageId, KeyName, LaunchTemplate or NetworkInterfaces, from scheduler.instance_ami and cluster.network.ssh_key_pair', + }, + { + component: 'shared-file-system', + moduleName: 'shared-storage', + resourceType: 'AWS::EFS::FileSystem', + consequence: + 'The applications and data file systems hold the only copy of what the cluster stores. A ' + + 'replacement is an empty file system beside a full one.', + forcedBy: 'Encrypted, PerformanceMode, KmsKeyId or AvailabilityZoneName, from shared-storage.apps.efs.* and shared-storage.data.efs.*', + refuse: + 'Moving to a new file system is a migration, not a replacement: the data has to be copied ' + + 'while both exist, and no ordering of a single deploy does that. Create the new file system, ' + + 'copy the data across, then point the cluster setting at it.', + }, + { + component: 'backup-vault', + moduleName: 'cluster', + resourceType: 'AWS::Backup::BackupVault', + consequence: 'The vault holds every recovery point taken from this cluster.', + forcedBy: 'EncryptionKeyArn, from cluster.backups.backup_vault.kms_key_id; the vault name is derived and not settable', + refuse: + 'A replacement vault is empty and the recovery points in the old one cannot be moved into ' + + 'it, so there is no version of this that ends with the backups intact. Copy what is needed ' + + 'to a vault you create separately first.', + }, +]; + +export function componentByName(name: string): ReplaceableComponent | undefined { + return REPLACEABLE_COMPONENTS.find((entry) => entry.component === name); +} + +/** The block an operator reads before they can proceed. */ +export function warningFor(component: ReplaceableComponent): string { + return [ + '', + `REPLACING ${component.component.toUpperCase()}`, + '', + component.consequence, + '', + `Forced by a change to: ${component.forcedBy}`, + `Stack: ${component.moduleName}. Resource type: ${component.resourceType}.`, + '', + ].join('\n'); +} + +export interface ReplaceCommandOptions { + clusterName: string; + awsRegion: string; + awsProfile?: string; + moduleSet: string; + deploymentId?: string; + /** The component name, typed again. Anything else does not proceed. */ + confirm?: string; +} + +export async function runReplace(deps: Deps, componentName: string, options: ReplaceCommandOptions): Promise { + const component = componentByName(componentName); + if (component === undefined) { + deps.err(`unknown component: ${componentName}`); + deps.err(`components: ${REPLACEABLE_COMPONENTS.map((entry) => entry.component).join(', ')}`); + throw new ExitWithCode(1); + } + + deps.out(warningFor(component)); + + if (component.refuse !== undefined) { + deps.err(`${component.component} cannot be replaced by this command.`); + deps.err(component.refuse); + throw new ExitWithCode(1); + } + + if (options.confirm !== component.component) { + deps.err('Nothing was changed.'); + deps.err(`Read the consequence above. To proceed, run the same command again with --confirm ${component.component}`); + throw new ExitWithCode(1); + } + + const helper = await DeploymentHelper.open({ + clusterName: options.clusterName, + awsRegion: options.awsRegion, + moduleSet: options.moduleSet, + awsProfile: options.awsProfile, + deploymentId: options.deploymentId, + upgrade: true, + allModules: false, + moduleIds: [component.moduleName], + // Scoped to this one type in this one stack, and only to the replacement class. A removal is a + // different intent and is still refused. + allowReplacementOfType: new Map([[component.resourceType, component.component]]), + deps, + }); + await helper.invoke(); + + deps.out( + `If the change set held no replacement of ${component.resourceType}, nothing was replaced. ` + + `CloudFormation only replaces when one of these changes: ${component.forcedBy}.`, + ); +} + +export function registerReplaceCommands(program: Command, deps: Deps): void { + program + .command('replace') + .description('replace one stateful component, deliberately. Upgrade and migrate never do this.') + .requiredOption('--cluster-name ', 'Cluster Name') + .requiredOption('--aws-region ', 'AWS Region') + .option('--aws-profile ', 'AWS Profile Name') + .option('--module-set ', 'Name of the ModuleSet. Default: default', 'default') + .option('--deployment-id ', 'A UUID to identify the deployment.') + .option('--confirm ', 'The component name again. Required, and it is the only thing that lets the run proceed.') + .argument('', `one of: ${REPLACEABLE_COMPONENTS.map((entry) => entry.component).join(', ')}`) + .action(async (component: string, options: ReplaceCommandOptions) => { + await runReplace(deps, component, options); + }); +} diff --git a/source/idea/ideactl/src/cli/commands/shared-storage.ts b/source/idea/ideactl/src/cli/commands/shared-storage.ts new file mode 100644 index 00000000..f6e9dee6 --- /dev/null +++ b/source/idea/ideactl/src/cli/commands/shared-storage.ts @@ -0,0 +1,212 @@ +/** + * Shared-storage attach and create command workflow. + * + * The interactive questionnaire is represented by an injected prompt. Its + * result is converted to the same settings map the administrator stores. + */ + +import { Command } from "commander"; + +import { ClusterConfig, ClusterConfigError, isEmpty } from "../../config/cluster-config.ts"; + +export const NEXT_STEP_EXIT = "Exit"; +export const NEXT_STEP_UPDATE_SETTINGS = "Update Cluster Settings and Exit"; +export const NEXT_STEP_UPGRADE_MODULE = "Deploy Module: Shared Storage"; +export const NEXT_STEP_DEPLOY_MODULE = "Upgrade Module: Shared Storage"; + +export interface SharedStorageApi { + describeFileSystems(input: { FileSystemId?: string; FileSystemIds?: string[] }): Promise<{ FileSystems?: Array> }>; + describeFileCaches(input: { FileCacheIds: string[] }): Promise<{ FileCaches?: Array> }>; + describeStorageVirtualMachines(input: { StorageVirtualMachineIds: string[] }): Promise<{ StorageVirtualMachines?: Array> }>; + describeVolumes(input: { VolumeIds: string[] }): Promise<{ Volumes?: Array> }>; +} + +export interface SharedStorageDeps { + config?: ClusterConfig; + storage: SharedStorageApi; + awsDnsSuffix(): Promise; + prompt(useExistingFs: boolean, existingCluster: boolean): Promise>; + promptNextStep(choices: string[]): Promise; + syncSettings(entries: Array<{ key: string; value: unknown }>): Promise; + deploy(moduleId: string, upgrade: boolean): Promise; + out(line: string): void; +} + +/** Builds dependencies for the profile selected by one shared-storage command. */ +export type SharedStorageDepsFactory = (options: SharedStorageOptions & { + awsProfile?: string; +}) => Promise; + +type SharedStorageDepsSource = SharedStorageDeps | SharedStorageDepsFactory; + +export interface SharedStorageOptions { + clusterName?: string; + awsRegion: string; + kmsKeyId?: string; +} + +function text(params: Record, key: string, fallback = ""): string { + const value = params[key]; + return typeof value === "string" && value.trim() !== "" ? value : fallback; +} + +function bool(params: Record, key: string, fallback = false): boolean { + const value = params[key]; + if (typeof value === "boolean") return value; + if (typeof value === "string") return ["true", "yes", "y", "1", "on"].includes(value.toLowerCase()); + return fallback; +} + +function list(params: Record, key: string): unknown[] { + return Array.isArray(params[key]) ? params[key] as unknown[] : []; +} + +function object(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; +} + +function delimited(value: string, fallback: string[]): string[] { + const entries = value.split(",").map((entry) => entry.trim().toLowerCase()).filter((entry) => entry !== ""); + return entries.length === 0 ? fallback : entries; +} + +function common(params: Record): Record { + const provider = text(params, "shared_storage_provider"); + const result: Record = { + title: text(params, "shared_storage_title"), + provider, + scope: delimited(text(params, "shared_storage_scope"), ["cluster"]), + }; + const scope = result.scope as string[]; + if (scope.includes("module")) result.modules = list(params, "shared_storage_scope_modules"); + if (scope.includes("project")) result.projects = delimited(text(params, "shared_storage_scope_projects"), ["default"]); + if (scope.includes("scheduler:queue-profile")) result.queue_profiles = delimited(text(params, "shared_storage_scope_queue_profiles"), ["compute"]); + if (["fsx_netapp_ontap", "fsx_windows_file_server"].includes(provider)) result.mount_drive = text(params, "shared_storage_mount_drive", "Z:"); + if (provider !== "fsx_windows_file_server") { + result.mount_dir = text(params, "shared_storage_mount_dir"); + result.mount_options = text(params, `${provider}.mount_options`); + } + return result; +} + +function requireValue(value: string, name: string): string { + if (isEmpty(value)) throw new ClusterConfigError(`${name} is required`); + return value; +} + +/** Build the precise provider map from questionnaire values and describe responses. */ +export async function buildSharedStorageConfig( + deps: Pick, + options: SharedStorageOptions, + params: Record, + useExistingFs: boolean, +): Promise> { + const name = requireValue(text(params, "shared_storage_name"), "shared_storage_name"); + const provider = text(params, "shared_storage_provider"); + const base = common(params); + if (provider === "efs") { + if (!useExistingFs) { + const transition = text(params, "efs.transition_to_ia"); + return { [name]: { ...base, efs: { + kms_key_id: options.kmsKeyId, + encrypted: true, + throughput_mode: text(params, "efs.throughput_mode"), + performance_mode: text(params, "efs.performance_mode"), + removal_policy: text(params, "efs.removal_policy", "DESTROY"), + cloudwatch_monitoring: bool(params, "efs.cloudwatch_monitoring"), + transition_to_ia: transition === "DISABLED" ? null : transition, + } } }; + } + const fileSystemId = requireValue(text(params, "efs.file_system_id"), "efs.file_system_id"); + const fileSystem = (await deps.storage.describeFileSystems({ FileSystemId: fileSystemId })).FileSystems?.[0] ?? {}; + return { [name]: { ...base, efs: { + use_existing_fs: true, + file_system_id: fileSystemId, + dns: `${fileSystemId}.efs.${options.awsRegion}.${await deps.awsDnsSuffix()}`, + encrypted: bool(fileSystem, "Encrypted"), + } } }; + } + if (provider === "fsx_cache") { + const id = requireValue(text(params, "fsx_cache.file_system_id"), "fsx_cache.file_system_id"); + const fs = (await deps.storage.describeFileCaches({ FileCacheIds: [id] })).FileCaches?.[0] ?? {}; + const lustre = object(fs.LustreConfiguration); + return { [name]: { ...base, fsx_cache: { use_existing_fs: true, file_system_id: id, dns: text(fs, "DNSName"), mount_name: text(lustre, "MountName"), version: text(fs, "FileCacheTypeVersion") } } }; + } + if (provider === "fsx_lustre") { + const id = requireValue(text(params, "fsx_lustre.file_system_id"), "fsx_lustre.file_system_id"); + const fs = (await deps.storage.describeFileSystems({ FileSystemIds: [id] })).FileSystems?.[0] ?? {}; + const lustre = object(fs.LustreConfiguration); + return { [name]: { ...base, fsx_lustre: { use_existing_fs: true, file_system_id: id, dns: text(fs, "DNSName"), mount_name: text(lustre, "MountName"), version: text(fs, "FileSystemTypeVersion") } } }; + } + if (provider === "fsx_netapp_ontap") { + const fileSystemId = requireValue(text(params, "fsx_netapp_ontap.file_system_id"), "fsx_netapp_ontap.file_system_id"); + const svmId = requireValue(text(params, "fsx_netapp_ontap.svm_id"), "fsx_netapp_ontap.svm_id"); + const volumeId = requireValue(text(params, "fsx_netapp_ontap.volume_id"), "fsx_netapp_ontap.volume_id"); + const svm = (await deps.storage.describeStorageVirtualMachines({ StorageVirtualMachineIds: [svmId] })).StorageVirtualMachines?.[0] ?? {}; + const endpoints = object(svm.Endpoints); + const volume = (await deps.storage.describeVolumes({ VolumeIds: [volumeId] })).Volumes?.[0] ?? {}; + const ontap = object(volume.OntapConfiguration); + return { [name]: { ...base, fsx_netapp_ontap: { use_existing_fs: true, file_system_id: fileSystemId, svm: { + svm_id: svmId, smb_dns: text(object(endpoints.Smb), "DNSName"), nfs_dns: text(object(endpoints.Nfs), "DNSName"), + management_dns: text(object(endpoints.Management), "DNSName"), iscsi_dns: text(object(endpoints.Iscsi), "DNSName"), + }, volume: { volume_id: volumeId, volume_path: text(ontap, "JunctionPath"), security_style: text(ontap, "SecurityStyle"), cifs_share_name: text(params, "fsx_netapp_ontap.cifs_share_name") } } } }; + } + if (provider === "fsx_openzfs") { + const fsId = requireValue(text(params, "fsx_openzfs.file_system_id"), "fsx_openzfs.file_system_id"); + const volumeId = requireValue(text(params, "fsx_openzfs.volume_id"), "fsx_openzfs.volume_id"); + const fs = (await deps.storage.describeFileSystems({ FileSystemIds: [fsId] })).FileSystems?.[0] ?? {}; + const volume = (await deps.storage.describeVolumes({ VolumeIds: [volumeId] })).Volumes?.[0] ?? {}; + return { [name]: { ...base, fsx_openzfs: { use_existing_fs: true, file_system_id: fsId, dns: text(fs, "DNSName"), volume_id: volumeId, volume_path: text(object(volume.OpenZFSConfiguration), "VolumePath") } } }; + } + if (provider === "fsx_windows_file_server") { + const id = requireValue(text(params, "fsx_windows_file_server.file_system_id"), "fsx_windows_file_server.file_system_id"); + const fs = (await deps.storage.describeFileSystems({ FileSystemIds: [id] })).FileSystems?.[0] ?? {}; + return { [name]: { ...base, fsx_windows_file_server: { use_existing_fs: true, file_system_id: id, dns: text(fs, "DNSName"), preferred_file_server_ip: text(object(fs.WindowsConfiguration), "PreferredFileServerIp") } } }; + } + throw new ClusterConfigError(`shared storage provider: ${provider} not supported`); +} + +function flatten(value: Record, prefix = ""): Array<{ key: string; value: unknown }> { + const entries: Array<{ key: string; value: unknown }> = []; + for (const [key, item] of Object.entries(value)) { + const path = prefix === "" ? key : `${prefix}.${key}`; + if (typeof item === "object" && item !== null && !Array.isArray(item)) entries.push(...flatten(item as Record, path)); + else entries.push({ key: path, value: item }); + } + return entries; +} + +/** Run the prompt, write selected settings, and optionally deploy the shared-storage module. */ +export async function manageSharedStorage(deps: SharedStorageDeps, options: SharedStorageOptions, useExistingFs: boolean): Promise { + const config = deps.config; + const clusterModule = config?.moduleInfoById(config.moduleId("cluster")); + const sharedModuleId = config?.moduleId("shared-storage"); + const sharedModule = sharedModuleId === undefined ? undefined : config?.moduleInfoById(sharedModuleId); + const existingCluster = clusterModule?.status === "deployed"; + const params = await deps.prompt(useExistingFs, existingCluster === true); + const storage = await buildSharedStorageConfig(deps, options, params, useExistingFs); + deps.out(JSON.stringify(storage, null, 2).replaceAll(": null", ": ~")); + const choices = [NEXT_STEP_EXIT]; + if (config !== undefined) choices.push(NEXT_STEP_UPDATE_SETTINGS); + if (!useExistingFs && existingCluster === true) choices.push(sharedModule?.status === "deployed" ? NEXT_STEP_UPGRADE_MODULE : NEXT_STEP_DEPLOY_MODULE); + const next = choices.length === 1 ? NEXT_STEP_EXIT : await deps.promptNextStep(choices); + if (next === NEXT_STEP_EXIT) return; + if (sharedModuleId === undefined) throw new ClusterConfigError("shared-storage module id not found"); + await deps.syncSettings(flatten(storage, sharedModuleId)); + if (next === NEXT_STEP_DEPLOY_MODULE || next === NEXT_STEP_UPGRADE_MODULE) await deps.deploy(sharedModuleId, next === NEXT_STEP_UPGRADE_MODULE); +} + +/** Register the `shared-storage` command group. */ +export function registerSharedStorageCommands(program: Command, deps: SharedStorageDepsSource): Command { + const resolveDeps = async (options: SharedStorageOptions & { + awsProfile?: string; + }): Promise => typeof deps === "function" ? deps(options) : deps; + const group = program.command("shared-storage").description("shared storage commands"); + for (const [name, existing] of [["add-file-system", false], ["attach-file-system", true]] as const) { + group.command(name).option("--cluster-name ").requiredOption("--aws-region ").option("--aws-profile ").option("--kms-key-id ") + .action(async (options: SharedStorageOptions & { awsProfile?: string }) => { + await manageSharedStorage(await resolveDeps(options), options, existing); + }); + } + return group; +} diff --git a/source/idea/ideactl/src/cli/commands/sso.ts b/source/idea/ideactl/src/cli/commands/sso.ts new file mode 100644 index 00000000..63db07c9 --- /dev/null +++ b/source/idea/ideactl/src/cli/commands/sso.ts @@ -0,0 +1,354 @@ +/** + * Single sign-on operator commands. + * + * The client and configuration writer are injected so callers can provide the + * live implementations while tests replay each operation without a network. + */ + +import { readFileSync } from "node:fs"; + +import { Command } from "commander"; + +import { ClusterConfig, ClusterConfigError, isEmpty } from "../../config/cluster-config.ts"; + +export const SSO_PROVIDER_OIDC = "OIDC"; +export const SSO_PROVIDER_SAML = "SAML"; + +/** Invalid identity-provider input is reported without changing the process exit code. */ +export class InvalidSsoParams extends ClusterConfigError {} + +export interface CognitoUser { + Username?: string; + UserStatus?: string; + Attributes?: Array<{ Name?: string; Value?: string }>; +} + +export interface CognitoApi { + getIdentityProviderByIdentifier(input: Record): Promise<{ IdentityProvider?: Record }>; + createIdentityProvider(input: Record): Promise; + updateIdentityProvider(input: Record): Promise; + createUserPoolClient(input: Record): Promise<{ UserPoolClient?: { ClientId?: string; ClientSecret?: string } }>; + updateUserPoolClient(input: Record): Promise<{ UserPoolClient?: { ClientId?: string; ClientSecret?: string } }>; + listUsers(input: { UserPoolId: string; PaginationToken?: string }): Promise<{ Users?: CognitoUser[]; PaginationToken?: string }>; + adminLinkProviderForUser(input: Record): Promise; +} + +export interface SecretsApi { + describeSecret(input: { SecretId: string }): Promise<{ ARN?: string }>; + createSecret(input: Record): Promise<{ ARN?: string }>; + updateSecret(input: Record): Promise<{ ARN?: string }>; +} + +export interface SsoDeps { + cognito: CognitoApi; + secrets: SecretsApi; + config: ClusterConfig; + setConfigEntry(key: string, value: unknown): Promise; + sleep(ms: number): Promise; + out(line: string): void; +} + +/** Builds command-scoped dependencies after Commander has parsed the selected AWS profile. */ +export type SsoDepsFactory = (options: { + clusterName: string; + awsRegion: string; + awsProfile?: string; +}) => Promise; + +type SsoDepsSource = SsoDeps | SsoDepsFactory; + +export interface SsoConfigureOptions { + clusterName: string; + providerName: string; + providerType: string; + providerEmailAttribute: string; + refreshTokenValidityHours?: number; + oidcClientId?: string; + oidcClientSecret?: string; + oidcIssuer?: string; + oidcAttributesRequestMethod?: string; + oidcAuthorizeScopes?: string; + oidcAuthorizeUrl?: string; + oidcTokenUrl?: string; + oidcAttributesUrl?: string; + oidcJwksUri?: string; + samlMetadataUrl?: string; + samlMetadataFile?: string; +} + +function required(value: string | undefined, name: string): string { + if (isEmpty(value)) throw new InvalidSsoParams(`${name} is required`); + return value as string; +} + +function identityProviderModuleId(config: ClusterConfig): string { + return config.moduleId("identity-provider"); +} + +async function save(deps: SsoDeps, key: string, value: unknown): Promise { + await deps.setConfigEntry(`${identityProviderModuleId(deps.config)}.${key}`, value); +} + +function isNotFound(error: unknown): boolean { + return error instanceof Error && ((error as { name?: string }).name === "ResourceNotFoundException" || error.message.includes("ResourceNotFoundException")); +} + +export function getSamlProviderDetails(options: SsoConfigureOptions): Record { + if (isEmpty(options.samlMetadataUrl) && isEmpty(options.samlMetadataFile)) { + throw new InvalidSsoParams("Either one of [saml_metadata_url, saml_metadata_file] is required, when provider_type = SAML"); + } + if (!isEmpty(options.samlMetadataFile)) { + try { + return { MetadataFile: readFileSync(options.samlMetadataFile as string, "utf8") }; + } catch { + throw new InvalidSsoParams(`file not found: ${options.samlMetadataFile}`); + } + } + return { MetadataURL: options.samlMetadataUrl as string }; +} + +export function getOidcProviderDetails(options: SsoConfigureOptions): Record { + const clientId = required(options.oidcClientId, "oidc_client_id"); + const clientSecret = required(options.oidcClientSecret, "oidc_client_secret"); + const issuer = required(options.oidcIssuer, "oidc_issuer"); + const details: Record = { + client_id: clientId, + client_secret: clientSecret, + attributes_request_method: options.oidcAttributesRequestMethod ?? "GET", + authorize_scopes: (options.oidcAuthorizeScopes ?? "openid").replaceAll(",", " "), + oidc_issuer: issuer, + }; + const optional: ReadonlyArray<[string, string | undefined]> = [ + ["authorize_url", options.oidcAuthorizeUrl], + ["token_url", options.oidcTokenUrl], + ["attributes_url", options.oidcAttributesUrl], + ["jwks_uri", options.oidcJwksUri], + ]; + for (const [key, value] of optional) if (!isEmpty(value)) details[key] = value as string; + return details; +} + +function callbackUrls(config: ClusterConfig): string[] { + const loadBalancerDns = config.getString("cluster.load_balancers.external_alb.load_balancer_dns_name", undefined, { required: true }) as string; + const customDns = config.getString("cluster.load_balancers.external_alb.certificates.custom_dns_name") ?? + config.getString("cluster.load_balancers.external_alb.custom_dns_name"); + const contextPath = config.getString("cluster-manager.server.web_resources_context_path", undefined, { required: true }) as string; + const path = contextPath === "/" ? "/sso/oauth2/callback" : `${contextPath}/oauth2/callback`; + const callbackPath = path.startsWith("/") ? path : `/${path}`; + return [ + `https://${loadBalancerDns}${callbackPath}`, + ...(isEmpty(customDns) ? [] : [`https://${customDns}${callbackPath}`]), + ]; +} + +async function existingIdentityProvider(deps: SsoDeps, userPoolId: string, identifier: string): Promise { + try { + const result = await deps.cognito.getIdentityProviderByIdentifier({ UserPoolId: userPoolId, IdpIdentifier: identifier }); + return result.IdentityProvider !== undefined; + } catch (error) { + if (isNotFound(error)) return false; + throw error; + } +} + +async function configureIdentityProvider(deps: SsoDeps, options: SsoConfigureOptions): Promise { + required(options.providerName, "provider_name"); + required(options.providerType, "provider_type"); + required(options.providerEmailAttribute, "provider_email_attribute"); + const userPoolId = deps.config.getString("identity-provider.cognito.user_pool_id", undefined, { required: true }) as string; + const identifier = deps.config.getString("identity-provider.cognito.sso_idp_identifier", "single-sign-on-identity-provider") as string; + const providerDetails = options.providerType === SSO_PROVIDER_SAML ? getSamlProviderDetails(options) : + options.providerType === SSO_PROVIDER_OIDC ? getOidcProviderDetails(options) : + (() => { throw new InvalidSsoParams("provider type must be one of: SAML or OIDC"); })(); + const request = { + UserPoolId: userPoolId, + ProviderName: options.providerName, + ProviderDetails: providerDetails, + AttributeMapping: { email: options.providerEmailAttribute }, + IdpIdentifiers: [identifier], + }; + if (await existingIdentityProvider(deps, userPoolId, identifier)) { + await deps.cognito.updateIdentityProvider(request); + } else { + await deps.cognito.createIdentityProvider({ ...request, ProviderType: options.providerType }); + } + await save(deps, "cognito.sso_idp_provider_name", options.providerName); + await save(deps, "cognito.sso_idp_provider_type", options.providerType); + await save(deps, "cognito.sso_idp_identifier", identifier); + await save(deps, "cognito.sso_idp_provider_email_attribute", options.providerEmailAttribute); +} + +async function configureUserPoolClient(deps: SsoDeps, options: SsoConfigureOptions): Promise { + const userPoolId = deps.config.getString("identity-provider.cognito.user_pool_id", undefined, { required: true }) as string; + const configuredClientId = deps.config.getString("identity-provider.cognito.sso_client_id"); + const request: Record = { + UserPoolId: userPoolId, + ClientName: "single-sign-on-client", + AccessTokenValidity: 1, + IdTokenValidity: 1, + RefreshTokenValidity: options.refreshTokenValidityHours === undefined || options.refreshTokenValidityHours <= 0 ? 12 : options.refreshTokenValidityHours, + TokenValidityUnits: { AccessToken: "hours", IdToken: "hours", RefreshToken: "hours" }, + ReadAttributes: ["address", "birthdate", "custom:aws_region", "custom:cluster_name", "custom:password_last_set", "custom:password_max_age", "email", "email_verified", "family_name", "gender", "given_name", "locale", "middle_name", "name", "nickname", "phone_number", "phone_number_verified", "picture", "preferred_username", "profile", "updated_at", "website", "zoneinfo"], + AllowedOAuthFlows: ["code"], + AllowedOAuthScopes: ["email", "openid", "aws.cognito.signin.user.admin"], + CallbackURLs: callbackUrls(deps.config), + SupportedIdentityProviders: [options.providerName], + AllowedOAuthFlowsUserPoolClient: true, + }; + let client: { ClientId?: string; ClientSecret?: string } | undefined; + if (!isEmpty(configuredClientId)) { + client = (await deps.cognito.updateUserPoolClient({ ...request, ClientId: configuredClientId })).UserPoolClient; + } else { + client = (await deps.cognito.createUserPoolClient({ ...request, GenerateSecret: true })).UserPoolClient; + const secretName = `${options.clusterName}-sso-client-secret`; + const kmsKeyId = deps.config.getString("cluster.secretsmanager.kms_key_id"); + let existing: { ARN?: string } | undefined; + try { + existing = await deps.secrets.describeSecret({ SecretId: secretName }); + } catch (error) { + if (!isNotFound(error)) throw error; + } + const secretRequest: Record = existing === undefined ? { + Name: secretName, + Description: `Single Sign-On OAuth2 Client Secret for Cluster: ${options.clusterName}`, + Tags: [{ Key: "idea:ClusterName", Value: options.clusterName }, { Key: "idea:ModuleName", Value: "cluster-manager" }], + SecretString: client?.ClientSecret, + } : { SecretId: existing.ARN, SecretString: client?.ClientSecret }; + if (!isEmpty(kmsKeyId)) secretRequest.KmsKeyId = kmsKeyId; + const secret = existing === undefined ? await deps.secrets.createSecret(secretRequest) : await deps.secrets.updateSecret(secretRequest); + await save(deps, "cognito.sso_client_secret", secret.ARN); + await save(deps, "cognito.sso_client_id", client?.ClientId); + } +} + +async function linkExistingUsers( + deps: SsoDeps, + configured?: { providerName: string; providerType: string; providerEmailAttribute: string }, +): Promise { + const userPoolId = deps.config.getString("identity-provider.cognito.user_pool_id", undefined, { required: true }) as string; + const providerName = configured?.providerName ?? deps.config.getString("identity-provider.cognito.sso_idp_provider_name", undefined, { required: true }) as string; + const providerType = configured?.providerType ?? deps.config.getString("identity-provider.cognito.sso_idp_provider_type", undefined, { required: true }) as string; + const adminUsername = deps.config.getString("cluster.administrator_username", undefined, { required: true }) as string; + const attribute = providerType === SSO_PROVIDER_OIDC ? "email" : + configured?.providerEmailAttribute ?? deps.config.getString("identity-provider.cognito.sso_idp_provider_email_attribute", undefined, { required: true }) as string; + let token: string | undefined; + do { + const page = await deps.cognito.listUsers({ UserPoolId: userPoolId, PaginationToken: token }); + for (const user of page.Users ?? []) { + try { + if (user.UserStatus === "EXTERNAL_PROVIDER") continue; + const username = user.Username ?? ""; + if (adminUsername.includes(username) || username.startsWith("clusteradmin")) { + deps.out(`system administration user found: ${username}. skip linking with IDP.`); + continue; + } + const email = user.Attributes?.find((entry) => entry.Name === "email")?.Value; + // The JSON string identity attribute is not treated as a list. + const alreadyLinked = false; + if (isEmpty(email)) continue; + if (alreadyLinked) { + deps.out(`user: ${username}, email: ${email} already linked. skip.`); + continue; + } + deps.out(`linking user: ${username}, email: ${email} ...`); + await deps.cognito.adminLinkProviderForUser({ + UserPoolId: userPoolId, + DestinationUser: { ProviderName: "Cognito", ProviderAttributeName: "cognito:username", ProviderAttributeValue: username }, + SourceUser: { ProviderName: providerName, ProviderAttributeName: attribute, ProviderAttributeValue: email }, + }); + await deps.sleep(200); + } catch (error) { + deps.out(`failed to link user: ${JSON.stringify(user)} with IDP: ${providerName} - ${String(error)}`); + } + } + token = page.PaginationToken; + } while (!isEmpty(token)); +} + +/** Execute the identity-provider, client, linking and final enablement sequence. */ +export async function configureSso(deps: SsoDeps, options: SsoConfigureOptions): Promise { + await configureIdentityProvider(deps, options); + await configureUserPoolClient(deps, options); + await linkExistingUsers(deps, { + providerName: options.providerName, + providerType: options.providerType, + providerEmailAttribute: options.providerEmailAttribute, + }); + await save(deps, "cognito.sso_enabled", true); +} + +/** Return the redirect information printed by `sso show-idp-info`. */ +export function showIdpInfo(config: ClusterConfig, providerType: string): { redirectUrl: string; entityId?: string } { + if (![SSO_PROVIDER_SAML, SSO_PROVIDER_OIDC].includes(providerType.trim().toUpperCase())) { + throw new ClusterConfigError("Invalid provider type. Must be one of: [SAML, OIDC]"); + } + const domain = config.getString("identity-provider.cognito.domain_url", undefined, { required: true }) as string; + if (providerType === SSO_PROVIDER_SAML) { + const pool = config.getString("identity-provider.cognito.user_pool_id", undefined, { required: true }) as string; + return { redirectUrl: `${domain}/saml2/idpresponse`, entityId: `urn:amazon:cognito:sp:${pool}` }; + } + return { redirectUrl: `${domain}/oauth2/idpresponse` }; +} + +/** Register the `sso` command group. Its injected dependencies are supplied by the command core. */ +export function registerSsoCommands(program: Command, deps: SsoDepsSource): Command { + const resolveDeps = async (options: { + clusterName: string; + awsRegion: string; + awsProfile?: string; + }): Promise => typeof deps === "function" ? deps(options) : deps; + const group = program.command("sso").description("single sign-on configuration"); + group.command("show-idp-info") + .requiredOption("--cluster-name ") + .requiredOption("--aws-region ") + .option("--aws-profile ") + .requiredOption("--provider-type ") + .action(async (options: { clusterName: string; awsRegion: string; awsProfile?: string; providerType: string }) => { + const actionDeps = await resolveDeps(options); + const result = showIdpInfo(actionDeps.config, options.providerType); + actionDeps.out("Redirect URL"); + actionDeps.out(result.redirectUrl); + if (result.entityId !== undefined) { + actionDeps.out("Entity ID"); + actionDeps.out(result.entityId); + } + }); + group.command("configure") + .requiredOption("--cluster-name ") + .requiredOption("--aws-region ") + .option("--aws-profile ") + .requiredOption("--provider-name ") + .requiredOption("--provider-type ") + .requiredOption("--provider-email-attribute ") + .option("--refresh-token-validity-hours ", "Refresh token validity in hours. Default: 12", Number) + .option("--oidc-client-id ").option("--oidc-client-secret ").option("--oidc-issuer ") + .option("--oidc-attributes-request-method ").option("--oidc-authorize-scopes ") + .option("--oidc-authorize-url ").option("--oidc-token-url ").option("--oidc-attributes-url ") + .option("--oidc-jwks-uri ").option("--saml-metadata-url ").option("--saml-metadata-file ") + .action(async (options: Record) => { + const actionDeps = await resolveDeps({ + clusterName: String(options.clusterName), + awsRegion: String(options.awsRegion), + awsProfile: typeof options.awsProfile === "string" ? options.awsProfile : undefined, + }); + try { + await configureSso(actionDeps, { + clusterName: String(options.clusterName), providerName: String(options.providerName), providerType: String(options.providerType), + providerEmailAttribute: String(options.providerEmailAttribute), refreshTokenValidityHours: options.refreshTokenValidityHours as number | undefined, + oidcClientId: options.oidcClientId as string | undefined, oidcClientSecret: options.oidcClientSecret as string | undefined, + oidcIssuer: options.oidcIssuer as string | undefined, oidcAttributesRequestMethod: options.oidcAttributesRequestMethod as string | undefined, + oidcAuthorizeScopes: options.oidcAuthorizeScopes as string | undefined, oidcAuthorizeUrl: options.oidcAuthorizeUrl as string | undefined, + oidcTokenUrl: options.oidcTokenUrl as string | undefined, oidcAttributesUrl: options.oidcAttributesUrl as string | undefined, + oidcJwksUri: options.oidcJwksUri as string | undefined, samlMetadataUrl: options.samlMetadataUrl as string | undefined, + samlMetadataFile: options.samlMetadataFile as string | undefined, + }); + } catch (error) { + if (error instanceof InvalidSsoParams) { + actionDeps.out(error.message); + return; + } + throw error; + } + }); + return group; +} diff --git a/source/idea/ideactl/src/cli/commands/status.ts b/source/idea/ideactl/src/cli/commands/status.ts new file mode 100644 index 00000000..e625e975 --- /dev/null +++ b/source/idea/ideactl/src/cli/commands/status.ts @@ -0,0 +1,248 @@ +/** + * Port of `check-cluster-status`, `list-modules` and `show-connection-info` + * (`app_main.py:1301-1601`). + */ + +import { request } from 'node:https'; + +import type { Command } from 'commander'; + +import { ClusterConfig, isEmpty, type ModuleInfo } from '../../config/cluster-config.ts'; +import { ExitWithCode, type Deps } from '../cdk-invoker.ts'; +import { ideaVersion } from '../../version.ts'; +import { renderTable } from './config.ts'; + +const MODULE_TYPE_APP = 'app'; + +/** + * `check_cluster_status`'s endpoint list: the analytics dashboard, then one `/healthcheck` per app + * module, in modules-table order. + */ +export function statusEndpoints(config: ClusterConfig): Array<{ name: string; endpoint: string }> { + const clusterEndpoint = config.getClusterExternalEndpoint(); + const endpoints: Array<{ name: string; endpoint: string }> = []; + for (const module of config.modules()) { + if (module.name === 'analytics') { + endpoints.push({ name: 'OpenSearch Service Dashboard', endpoint: `${clusterEndpoint}/_dashboards/` }); + } else if (module.type === MODULE_TYPE_APP) { + endpoints.push({ + name: module.title ?? module.name, + endpoint: `${clusterEndpoint}/${module.module_id}/healthcheck`, + }); + } + } + return endpoints; +} + +/** Requests without a User-Agent are answered 403 by the load balancer. */ +export const PROBE_USER_AGENT = `ideactl/${ideaVersion()}`; + +/** `requests.get` follows up to 30 redirects; the dashboard endpoint answers one. */ +const MAX_PROBE_REDIRECTS = 30; + +/** + * The cluster's own certificate is usually self-signed, so this does not verify it. That matches + * `requests.get(url, verify=False)`. + * + * Two of that call's defaults have to be supplied by hand here. It sends a User-Agent, and the + * external load balancer answers 403 when a request carries none, so a probe without one reports + * every healthy endpoint as failing. It also follows redirects, and the analytics dashboard answers + * `/_dashboards/` with a 302 to `/_dashboards/app/home`, so a probe that stops at the first response + * reports a working dashboard as failing. + */ +export const liveHttpStatus = (url: string, redirectsLeft = MAX_PROBE_REDIRECTS): Promise => + new Promise((resolve) => { + const req = request( + url, + { rejectUnauthorized: false, method: 'GET', headers: { 'user-agent': PROBE_USER_AGENT } }, + (response) => { + response.resume(); + const status = response.statusCode ?? 0; + const location = response.headers.location; + if (status >= 300 && status < 400 && location !== undefined && redirectsLeft > 0) { + resolve(liveHttpStatus(new URL(location, url).toString(), redirectsLeft - 1)); + return; + } + resolve(status); + }, + ); + req.on('error', () => resolve(0)); + req.end(); + }); + +export interface CheckStatusOptions { + clusterName: string; + awsRegion: string; + awsProfile?: string; + wait?: boolean; + waitTimeout?: number; + debug?: boolean; + moduleSet: string; +} + +/** Loops every 60 s while `--wait`; exits 1 when any endpoint is still failing. */ +export async function checkClusterStatus(deps: Deps, options: CheckStatusOptions): Promise { + const config = await ClusterConfig.fromDynamoDb(options.clusterName, options.awsRegion, { + moduleSet: options.moduleSet, + scan: deps.scan, + }); + const clusterEndpoint = config.getClusterExternalEndpoint(); + const endpoints = statusEndpoints(config); + + const endTime = deps.now() + (options.waitTimeout ?? 900) * 1000; + let failCount = 0; + let currentTime = deps.now(); + + while (currentTime < endTime) { + deps.out(`checking endpoint status for cluster: ${options.clusterName}, url: ${clusterEndpoint} ...`); + failCount = 0; + const rows: string[][] = []; + for (const endpoint of endpoints) { + const status = await deps.httpStatus(endpoint.endpoint); + if (options.debug === true) deps.out(`${endpoint.endpoint} - ${status}`); + const success = status === 200; + if (!success) failCount += 1; + rows.push([endpoint.name, endpoint.endpoint, success ? 'SUCCESS' : 'FAIL']); + } + deps.out(renderTable(['Module', 'Endpoint', 'Status'], rows)); + + if (options.wait !== true) break; + if (failCount === 0) break; + deps.out('failed to verify all cluster endpoints. wait ... (Press Ctrl + C to exit) '); + await deps.sleep(60_000); + currentTime = deps.now(); + } + + if (options.wait === true && currentTime >= endTime) { + deps.err( + "check endpoint status timed-out. please verify your cluster's External ALB Security Group " + + 'configuration and check correct ingress rules have been configured.', + ); + } + if (failCount > 0) throw new ExitWithCode(1); + return failCount; +} + +/** `list_modules`: Title / Name / Module ID / Type / Stack Name / Version / Status. */ +export function modulesTable(modules: ModuleInfo[]): string { + return renderTable( + ['Title', 'Name', 'Module ID', 'Type', 'Stack Name', 'Version', 'Status'], + modules.map((module) => [ + module.title ?? '', + module.name, + module.module_id, + module.type, + module.stack_name ?? '-', + module.version ?? '-', + module.status ?? '', + ]), + ); +} + +/** `get_session_manager_url`; the console host differs per partition. */ +export function sessionManagerUrl(awsPartition: string, awsRegion: string, instanceId: string): string { + let consolePrefix = `${awsRegion}.`; + let consoleSuffix = '.aws.amazon.com'; + if (awsPartition === 'aws-cn') { + consolePrefix = ''; + consoleSuffix = '.amazonaws.cn'; + } else if (awsPartition === 'aws-us-gov') { + consolePrefix = ''; + consoleSuffix = '.amazonaws-us-gov.com'; + } + return `https://${consolePrefix}console${consoleSuffix}/systems-manager/session-manager/${instanceId}?region=${awsRegion}`; +} + +/** + * `show_connection_info`: only deployed modules contribute, and the entries print in the hardcoded + * weight order (portal, bastion ssh, bastion session manager, analytics). + */ +export function connectionInfo( + config: ClusterConfig, + awsRegion: string, +): Array<{ key: string; value: string; weight: number }> { + const entries: Array<{ key: string; value: string; weight: number }> = []; + const clusterEndpoint = config.getClusterExternalEndpoint(); + if (isEmpty(clusterEndpoint)) return entries; + + for (const module of config.modules()) { + if (module.status !== 'deployed') continue; + if (module.name === 'cluster-manager') { + entries.push({ key: 'Web Portal', value: clusterEndpoint, weight: 0 }); + } else if (module.name === 'analytics') { + entries.push({ key: 'Analytics Dashboard', value: `${clusterEndpoint}/_dashboards`, weight: 3 }); + } else if (module.name === 'bastion-host') { + const keyPairName = config.getString('cluster.network.ssh_key_pair'); + const ipAddress = + config.getString(`${module.module_id}.public_ip`) ?? config.getString(`${module.module_id}.private_ip`); + if (!isEmpty(ipAddress)) { + // Read for its refusal: a bastion with no base_os is a broken module row, not a default. + config.getString(`${module.module_id}.base_os`, undefined, { required: true }); + entries.push({ + key: 'Bastion Host (SSH Access)', + // Every supported base OS uses the same login user. + value: `ssh -i ~/.ssh/${keyPairName}.pem ec2-user@${ipAddress as string}`, + weight: 1, + }); + } + const instanceId = config.getString(`${module.module_id}.instance_id`); + if (!isEmpty(instanceId)) { + const partition = config.getString('cluster.aws.partition', undefined, { required: true }) as string; + entries.push({ + key: 'Bastion Host (Session Manager URL)', + value: sessionManagerUrl(partition, awsRegion, instanceId as string), + weight: 2, + }); + } + } + } + entries.sort((a, b) => a.weight - b.weight); + return entries; +} + +export function registerStatusCommands(program: Command, deps: Deps): void { + program + .command('check-cluster-status') + .description('check status for all applicable cluster endpoints') + .requiredOption('--cluster-name ', 'Cluster Name') + .requiredOption('--aws-region ', 'AWS Region') + .option('--aws-profile ', 'AWS Profile Name') + .option('--wait', 'Wait until all cluster endpoints are healthy.') + .option('--wait-timeout ', 'Wait timeout in seconds. Default: 900 (15 mins)', (value) => Number.parseInt(value, 10), 900) + .option('--debug', 'Print debug messages') + .option('--module-set ', 'Name of the ModuleSet. Default: default', 'default') + .action(async (options: CheckStatusOptions) => { + await checkClusterStatus(deps, options); + }); + + program + .command('list-modules') + .description('list all modules for a cluster') + .requiredOption('--cluster-name ', 'Cluster Name') + .requiredOption('--aws-region ', 'AWS Region') + .option('--aws-profile ', 'AWS Profile Name') + .action(async (options: { clusterName: string; awsRegion: string }) => { + const config = await ClusterConfig.fromDynamoDb(options.clusterName, options.awsRegion, { scan: deps.scan }); + deps.out(modulesTable(config.modules())); + }); + + program + .command('show-connection-info') + .description('print cluster connection information') + .requiredOption('--cluster-name ', 'Cluster Name') + .requiredOption('--aws-region ', 'AWS Region') + .option('--aws-profile ', 'AWS Profile Name') + .option('--module-set ', 'Name of the ModuleSet. Default: default', 'default') + .action(async (options: { clusterName: string; awsRegion: string; moduleSet: string }) => { + const config = await ClusterConfig.fromDynamoDb(options.clusterName, options.awsRegion, { + moduleSet: options.moduleSet, + scan: deps.scan, + }); + const entries = connectionInfo(config, options.awsRegion); + if (entries.length === 0) { + deps.err(`No connection information found for cluster: ${options.clusterName}. Is the cluster deployed?`); + return; + } + for (const entry of entries) deps.out(`${entry.key}: ${entry.value}`); + }); +} diff --git a/source/idea/ideactl/src/cli/commands/support.ts b/source/idea/ideactl/src/cli/commands/support.ts new file mode 100644 index 00000000..713c6ce8 --- /dev/null +++ b/source/idea/ideactl/src/cli/commands/support.ts @@ -0,0 +1,112 @@ +/** Build the deployment support package from local artifacts and a configuration snapshot. */ + +import { cpSync, existsSync, mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +import yaml from "js-yaml"; +import { Command } from "commander"; + +import { clusterCdkDir, clusterDeploymentsDir, clusterRegionDir } from "../cdk-invoker.ts"; + +export const PACKAGE_DEPLOYMENT_LOGS = "deployment-logs"; +export const PACKAGE_DEPLOYMENTS_DIR = "deployments-dir"; +export const PACKAGE_CDK_CONFIG = "cdk-config"; +export const PACKAGE_VALUES_FILE = "config-values-file"; +export const PACKAGE_CLUSTER_CONFIG_DB = "cluster-config-db"; +export const PACKAGE_CLUSTER_CONFIG_LOCAL = "cluster-config-local"; + +export interface SupportDeps { + databaseConfig?: () => Promise<{ configYaml: string; modulesYaml: string }>; + now(): Date; + chooseContents?: () => Promise; + archive(directory: string): Promise; + out(line: string): void; +} + +/** Builds dependencies for the profile selected by one support command. */ +export type SupportDepsFactory = (options: SupportOptions) => Promise; + +type SupportDepsSource = SupportDeps | SupportDepsFactory; + +export interface SupportOptions { + clusterName: string; + awsRegion: string; + awsProfile?: string; + moduleSet?: string; +} + +function timestamp(now: Date): string { + const pad = (value: number): string => String(value).padStart(2, "0"); + return `${now.getUTCFullYear()}${pad(now.getUTCMonth() + 1)}${pad(now.getUTCDate())}_${pad(now.getUTCHours())}${pad(now.getUTCMinutes())}${pad(now.getUTCSeconds())}`; +} + +function copyIfPresent(source: string, destination: string): void { + if (existsSync(source)) cpSync(source, destination, { recursive: true }); +} + +/** + * Copy the requested diagnostics and create an archive through the injected archiver. + * The archiver stays injected because the runtime image determines its available utility. + */ +export async function buildDeploymentSupportPackage( + deps: SupportDeps, + options: SupportOptions, + contents: readonly string[], +): Promise { + const regionDir = clusterRegionDir(options.clusterName, options.awsRegion, false); + const packageDir = join(regionDir, "support", `idea-deployment-debug-pkg-${timestamp(deps.now())}`); + mkdirSync(packageDir, { recursive: true }); + writeFileSync(join(packageDir, "package.yml"), yaml.dump({ + type: "deployment-debug", + created_on: deps.now().toISOString(), + options: { package_contents: [...contents] }, + }, { noRefs: true, sortKeys: false })); + if (contents.includes(PACKAGE_DEPLOYMENT_LOGS)) { + const logsDir = join(regionDir, "logs"); + deps.out(`copying deployment logs: ${logsDir} ...`); + copyIfPresent(logsDir, join(packageDir, "logs")); + } + if (contents.includes(PACKAGE_CDK_CONFIG)) { + const cdkDir = clusterCdkDir(options.clusterName, options.awsRegion); + deps.out(`copying cdk config: ${cdkDir} ...`); + copyIfPresent(cdkDir, join(packageDir, "_cdk")); + } + if (contents.includes(PACKAGE_DEPLOYMENTS_DIR)) { + const deployments = clusterDeploymentsDir(options.clusterName, options.awsRegion); + deps.out(`copying deployments: ${deployments} ...`); + copyIfPresent(deployments, join(packageDir, "deployments")); + } + if (contents.includes(PACKAGE_VALUES_FILE)) { + copyIfPresent(join(regionDir, "values.yml"), join(packageDir, "values.yml")); + } + if (contents.includes(PACKAGE_CLUSTER_CONFIG_LOCAL)) { + copyIfPresent(join(regionDir, "config"), join(packageDir, "config_local")); + } + if (contents.includes(PACKAGE_CLUSTER_CONFIG_DB) && deps.databaseConfig !== undefined) { + const dbDir = join(packageDir, "config_db"); + mkdirSync(dbDir, { recursive: true }); + const dump = await deps.databaseConfig(); + writeFileSync(join(dbDir, "config.yml"), dump.configYaml); + writeFileSync(join(dbDir, "modules.yml"), dump.modulesYaml); + } + return deps.archive(packageDir); +} + +/** Register the `support deployment` command. */ +export function registerSupportCommands(program: Command, deps: SupportDepsSource): Command { + const resolveDeps = async (options: SupportOptions): Promise => typeof deps === "function" ? deps(options) : deps; + const group = program.command("support").description("support options"); + group.command("deployment") + .requiredOption("--cluster-name ") + .requiredOption("--aws-region ") + .option("--aws-profile ") + .option("--module-set ", "Name of the ModuleSet. Default: default", "default") + .action(async (options: SupportOptions) => { + const actionDeps = await resolveDeps(options); + const defaults = [PACKAGE_DEPLOYMENT_LOGS, PACKAGE_VALUES_FILE, PACKAGE_CLUSTER_CONFIG_DB, PACKAGE_CLUSTER_CONFIG_LOCAL]; + const contents = actionDeps.chooseContents === undefined ? defaults : await actionDeps.chooseContents(); + const file = await buildDeploymentSupportPackage(actionDeps, options, contents); + actionDeps.out(`Debug Package: ${file}`); + }); + return group; +} diff --git a/source/idea/ideactl/src/cli/commands/tests.ts b/source/idea/ideactl/src/cli/commands/tests.ts new file mode 100644 index 00000000..762200f1 --- /dev/null +++ b/source/idea/ideactl/src/cli/commands/tests.ts @@ -0,0 +1,134 @@ +/** Run the shipped module integration-test cases through an injected test runner. */ + +import { Command } from "commander"; + +import { ClusterConfig, GeneralException, isEmpty } from "../../config/cluster-config.ts"; + +export class IntegrationTestFailed extends Error {} + +export interface IntegrationTestContext { + clusterName: string; + awsRegion: string; + awsProfile?: string; + adminUsername: string; + adminPassword: string; + debug: boolean; + extraParams: Record; + testCaseIds: string[]; + moduleIds: string[]; +} + +export interface IntegrationTestCase { + id: string; + run(context: IntegrationTestContext): Promise; +} + +export interface IntegrationTestDeps { + config: ClusterConfig; + casesForModule(moduleName: string): readonly IntegrationTestCase[] | undefined; + out(line: string): void; + err(line: string): void; +} + +/** Builds dependencies for the profile selected by one integration-test command. */ +export type IntegrationTestDepsFactory = (options: RunIntegrationTestsOptions) => Promise; + +type IntegrationTestDepsSource = IntegrationTestDeps | IntegrationTestDepsFactory; + +export interface RunIntegrationTestsOptions { + clusterName: string; + awsRegion: string; + awsProfile?: string; + adminUsername: string; + adminPassword: string; + testCaseId?: string; + debug?: boolean; + param?: string[]; + moduleSet?: string; +} + +/** Python keeps the last value for a duplicate parameter and ignores tokens without `=`. */ +export function parseIntegrationParams(params: readonly string[] = []): Record { + const result: Record = {}; + for (const param of params) { + const separator = param.indexOf("="); + if (separator >= 0) result[param.slice(0, separator)] = param.slice(separator + 1); + } + return result; +} + +function dedupe(moduleIds: readonly string[]): string[] { + const result: string[] = []; + for (const id of moduleIds) if (!result.includes(id)) result.push(id); + return result; +} + +/** Execute each selected test case and aggregate failures separately for each module. */ +export async function runIntegrationTests( + deps: IntegrationTestDeps, + options: RunIntegrationTestsOptions, + moduleIds: readonly string[], +): Promise { + const ids = dedupe(moduleIds); + const context: IntegrationTestContext = { + clusterName: options.clusterName, + awsRegion: options.awsRegion, + awsProfile: options.awsProfile, + adminUsername: options.adminUsername, + adminPassword: options.adminPassword, + debug: options.debug === true, + extraParams: parseIntegrationParams(options.param), + testCaseIds: isEmpty(options.testCaseId) ? [] : (options.testCaseId as string).split(","), + moduleIds: ids, + }; + for (const moduleId of ids) { + const module = deps.config.moduleInfoById(moduleId); + if (module === undefined) throw new GeneralException(`module not found for module id: ${moduleId}`); + if (module.status !== "deployed") throw new GeneralException(`module id: ${moduleId} is not deployed yet.`); + const testCases = deps.casesForModule(module.name); + if (testCases === undefined) { + deps.out(`no test cases found for module: ${module.name}`); + continue; + } + let total = 0; + let failures = 0; + for (const testCase of testCases) { + if (context.testCaseIds.length > 0 && !context.testCaseIds.includes(testCase.id)) continue; + total += 1; + deps.out(`${testCase.id} [STARTED]`); + try { + await testCase.run(context); + deps.out(`${testCase.id} [PASS]`); + } catch (error) { + deps.err(String(error)); + deps.err(`${testCase.id} [FAIL]`); + failures += 1; + } + } + if (failures > 0) { + const passed = total - failures; + const rate = Math.round((passed / total) * 10000) / 100; + throw new IntegrationTestFailed(`${failures} of ${total} test cases failed. success rate: ${rate}%`); + } + } +} + +/** Register `run-integration-tests`. */ +export function registerIntegrationTestCommands(program: Command, deps: IntegrationTestDepsSource): Command { + const resolveDeps = async (options: RunIntegrationTestsOptions): Promise => + typeof deps === "function" ? deps(options) : deps; + return program.command("run-integration-tests") + .requiredOption("--cluster-name ") + .requiredOption("--aws-region ") + .option("--aws-profile ") + .requiredOption("--admin-username ") + .requiredOption("--admin-password ") + .option("--test-case-id ") + .option("--debug") + .option("-p, --param ", "Additional test case parameter", (value: string, previous: string[] = []) => [...previous, value]) + .option("--module-set ", "Name of the ModuleSet. Default: default") + .argument("") + .action(async (modules: string[], options: RunIntegrationTestsOptions) => { + await runIntegrationTests(await resolveDeps(options), options, modules); + }); +} diff --git a/source/idea/ideactl/src/cli/commands/upgrade.ts b/source/idea/ideactl/src/cli/commands/upgrade.ts new file mode 100644 index 00000000..572e3017 --- /dev/null +++ b/source/idea/ideactl/src/cli/commands/upgrade.ts @@ -0,0 +1,1297 @@ +/** + * Upgrade an existing cluster by applying the administrator's ordered upgrade + * sequence. Each external operation is injected so callers can replay it. + */ + +import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import type { Command } from "commander"; + +import { ClusterConfigError, GeneralException, type ModuleInfo } from "../../config/cluster-config.ts"; +import { + convertConfigToKeyValuePairs, + generateConfigFromTemplates, + readModulesFromFiles, + type ConfigEntry, +} from "../../config/generator.ts"; +import { loadRegionAmiConfig, resolveRegionAmi, type RegionsConfig } from "../../config/region-ami.ts"; +import { + compareUpgradeDrift, + renderUpgradeDrift, + type CurrentConfigRow, + type StackSettingsPlan, + type UpgradeDriftInput, + type UpgradeDriftReport, +} from "../../config/upgrade-drift.ts"; +import { loadValuesFile } from "../../config/values.ts"; +import { + buildTree, + toYaml, + type ConfigDriftPreviewDeps, + type ConfigUpgradePreviewOptions, +} from "./config.ts"; +import { asBoolFlag } from "./deploy.ts"; +import { awsClientOptions, type AwsClientOptions } from "../aws-client-options.ts"; +import { DeploymentHelper } from "../deployment-helper.ts"; +import { ExitWithCode, VALUES_FILE_S3_KEY, valuesFilePath, type Deps } from "../cdk-invoker.ts"; + +export const EOL_BASE_OS: Readonly> = { + amazonlinux2: "amazonlinux2023", +}; + +export const UPGRADE_BASE_OS: readonly string[] = [ + "amazonlinux2023", + "rhel8", + "rhel9", + "rhel10", + "rocky8", + "rocky9", + "rocky10", +]; + +const ECS_MODULE = "ecs"; + +/** One effective ECS account setting returned by the pre-flight reader. */ +export interface EcsAccountSetting { + name: string; + value: string; +} + +/** Read-only ECS account settings required before an ECS deployment. */ +export interface EcsAccountSettingsApi { + listAccountSettings(input: { + awsProfile?: string; + awsRegion: string; + effectiveSettings: true; + name: "awsvpcTrunking"; + }): Promise; +} + +/** The injected dependencies used by the ECS trunking pre-flight. */ +export interface EcsTrunkingPreflightDeps { + accountId(): Promise; + ecsAccountSettings?: EcsAccountSettingsApi; + err(line: string): void; +} + +/** The operator-selected values needed to print the remediation command. */ +export interface EcsTrunkingPreflightOptions { + awsProfile?: string; + awsRegion: string; +} + +/** + * Builds the one-time operator command for the account-wide ECS prerequisite. + */ +export function awsvpcTrunkingCommand(options: EcsTrunkingPreflightOptions): string { + return [ + "aws", + "ecs", + "put-account-setting-default", + "--name", + "awsvpcTrunking", + "--value", + "enabled", + "--region", + options.awsRegion, + ...(options.awsProfile === undefined ? [] : ["--profile", options.awsProfile]), + ].join(" "); +} + +/** Client options for the live upgrade readers, with the operator's profile bound. */ +export async function upgradeLiveClientOptions( + awsRegion: string, + awsProfile?: string, +): Promise { + return awsClientOptions(awsRegion, awsProfile); +} + +/** + * Refuses an ECS deployment until the account's effective task ENI trunking + * setting is enabled. This check reads account state only. + */ +export async function checkAwsvpcTrunking( + deps: EcsTrunkingPreflightDeps, + options: EcsTrunkingPreflightOptions, +): Promise { + if (deps.ecsAccountSettings === undefined) { + throw new GeneralException("ECS account-settings reader is required for the awsvpcTrunking pre-flight"); + } + const [account, settings] = await Promise.all([ + deps.accountId(), + deps.ecsAccountSettings.listAccountSettings({ + // Read the account the rest of the run will deploy into, which a named profile selects. + ...(options.awsProfile === undefined ? {} : { awsProfile: options.awsProfile }), + awsRegion: options.awsRegion, + effectiveSettings: true, + name: "awsvpcTrunking", + }), + ]); + const enabled = settings.some((setting) => setting.name === "awsvpcTrunking" && setting.value === "enabled"); + if (enabled) return; + + deps.err( + `ECS awsvpcTrunking is not enabled for account ${account} in ${options.awsRegion}. Without it, an m7g.large host of the planned size fits only two tasks, so placement silently starves.`, + ); + deps.err("Run this once for the account, then repeat the deploy:"); + deps.err(awsvpcTrunkingCommand(options)); + throw new ExitWithCode(1); +} + +const MODULE_HOST_INSTANCE_TYPE = "m7i.large"; +const MODULE_HOST_INSTANCE_TYPE_OLD = "m6i.large"; +const OPENSEARCH_DATA_NODE_INSTANCE_TYPE = "m7g.large.search"; +const OPENSEARCH_DATA_NODE_INSTANCE_TYPE_OLD = "m5.large.search"; +const COMPUTE_IMAGE_PREFIX = ["idea", "compute", "node", ""].join("-"); + +const AMI_UPDATE_KEYS: Readonly>> = { + "bastion-host": [["base_os", "instance_ami"]], + "cluster-manager": [["ec2.autoscaling.base_os", "ec2.autoscaling.instance_ami"]], + directoryservice: [["base_os", "instance_ami"]], + scheduler: [ + ["base_os", "instance_ami"], + ["compute_node_os", "compute_node_ami"], + ], + "virtual-desktop-controller": [ + ["controller.autoscaling.base_os", "controller.autoscaling.instance_ami"], + ["dcv_broker.autoscaling.base_os", "dcv_broker.autoscaling.instance_ami"], + ["dcv_connection_gateway.autoscaling.base_os", "dcv_connection_gateway.autoscaling.instance_ami"], + ], +}; + +const HOST_INSTANCE_TYPE_KEYS: Readonly> = { + "bastion-host": ["instance_type"], + "cluster-manager": ["ec2.autoscaling.instance_type"], + directoryservice: ["instance_type"], + scheduler: ["instance_type"], + "virtual-desktop-controller": [ + "controller.autoscaling.instance_type", + "dcv_broker.autoscaling.instance_type", + "dcv_connection_gateway.autoscaling.instance_type", + ], +}; + +export interface InstanceImage { + ImageId?: string; + Name?: string; + CreationDate?: string; +} + +export interface UpgradeEc2Api { + describeImages(input: { awsRegion: string; imageIds: string[] }): Promise; + describeInstanceTypeOfferings(input: { awsRegion: string; instanceType: string }): Promise; + describeInstanceAttribute(input: { awsRegion: string; instanceId: string }): Promise; + modifyInstanceAttribute(input: { awsRegion: string; instanceId: string; protected: boolean }): Promise; + describeLiveInstances(input: { awsRegion: string; instanceIds: string[] }): Promise; +} + +export interface UpgradeCloudFormationApi { + listStackResources(input: { + awsRegion: string; + stackName: string; + nextToken?: string; + }): Promise<{ instanceIds: string[]; nextToken?: string }>; +} + +export interface UpgradeOpenSearchApi { + describeDomain(input: { awsRegion: string; domainName?: string }): Promise<{ engineVersion?: string }>; + listInstanceTypeDetails(input: { awsRegion: string; engineVersion?: string }): Promise; +} + +export interface EolSoftwareStackApi { + setEnabled(input: { awsRegion: string; tableName: string; baseOs: string; stackId: string; enabled: boolean }): Promise; + delete(input: { awsRegion: string; tableName: string; baseOs: string; stackId: string }): Promise; +} + +export interface UpgradeDeploymentOptions { + clusterName: string; + awsRegion: string; + awsProfile?: string; + terminationProtection: boolean; + deploymentId?: string; + forceBuildBootstrap: boolean; + rollback: boolean; + optimizeDeployment: boolean; + moduleSet: string; + allModules: boolean; + moduleIds?: readonly string[]; +} + +export interface UpgradeDeps extends ConfigDriftPreviewDeps { + ec2: UpgradeEc2Api; + ecsAccountSettings?: EcsAccountSettingsApi; + cloudFormation: UpgradeCloudFormationApi; + openSearch: UpgradeOpenSearchApi; + eolSoftwareStacks: EolSoftwareStackApi; + deploy(options: UpgradeDeploymentOptions): Promise; + regionAmiConfig?: () => RegionsConfig; +} + +export interface UpgradeCommandOptions { + clusterName: string; + awsRegion: string; + awsProfile?: string; + terminationProtection?: string | boolean; + deploymentId?: string; + baseOs?: string; + forceBuildBootstrap?: boolean; + rollback?: boolean; + optimizeDeployment?: boolean; + moduleSet: string; + force?: boolean; + acceptConfigDrift?: boolean; + skipGlobalSettingsUpdate?: boolean; + disableEolStacksInUse?: boolean; + modules?: readonly string[]; +} + +interface EolStack { + stackId: string; + baseOs: string; + name: string; + architecture: string; +} + +interface EolSession { + stackId: string; + baseOs: string; + sessionId: string; + owner: string; + name: string; +} + +interface EolPlan { + tableName: string; + sessions: EolSession[]; + toDelete: EolStack[]; + toDisable: EolStack[]; +} + +interface ClearedInstance { + stackName: string; + instanceId: string; +} + +function valueAsString(value: unknown, defaultValue = ""): string { + return typeof value === "string" && value !== "" ? value : defaultValue; +} + +function asRecord(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; +} + +function toModuleInfo(row: Record): ModuleInfo | undefined { + const moduleId = valueAsString(row["module_id"]); + const name = valueAsString(row["name"]); + const type = valueAsString(row["type"]); + return moduleId === "" || name === "" || type === "" ? undefined : { ...row, module_id: moduleId, name, type }; +} + +async function scanAll(deps: Deps, tableName: string): Promise>> { + const rows: Array> = []; + let startKey: Record | undefined; + do { + const page = await deps.scan({ TableName: tableName, ExclusiveStartKey: startKey }); + rows.push(...(page.Items ?? [])); + startKey = page.LastEvaluatedKey; + } while (startKey !== undefined); + return rows; +} + +async function scanModuleTable(deps: Deps, tableName: string): Promise>> { + try { + return await scanAll(deps, tableName); + } catch (error) { + if ((error as { name?: string }).name === "ResourceNotFoundException") return []; + throw error; + } +} + +async function clusterModules(deps: Deps, clusterName: string): Promise { + const result: ModuleInfo[] = []; + for (const row of await scanAll(deps, `${clusterName}.modules`)) { + const module = toModuleInfo(row); + if (module !== undefined) result.push(module); + } + return result; +} + +function describeStack(stack: EolStack): string { + return `${stack.stackId} (${stack.name}, ${stack.architecture})`; +} + +async function findEolReferences(deps: UpgradeDeps, clusterName: string): Promise { + const findings: string[] = []; + for (const entry of await scanAll(deps, `${clusterName}.cluster-settings`)) { + const key = valueAsString(entry["key"]); + const value = valueAsString(entry["value"]); + if ((key.endsWith("base_os") || key.endsWith("compute_node_os")) && EOL_BASE_OS[value] !== undefined) { + findings.push(`cluster setting ${key} = ${value}`); + } + } + for (const module of await clusterModules(deps, clusterName)) { + if (module.name !== "scheduler") continue; + for (const row of await scanModuleTable(deps, `${clusterName}.${module.module_id}.queue-profiles`)) { + const baseOs = valueAsString(row["param_base_os"]); + if (EOL_BASE_OS[baseOs] !== undefined) { + findings.push(`HPC queue profile ${valueAsString(row["queue_profile_name"], "")} = ${baseOs}`); + } + } + } + return findings; +} + +async function planEolSoftwareStacks(deps: UpgradeDeps, clusterName: string): Promise { + const plans: EolPlan[] = []; + for (const module of await clusterModules(deps, clusterName)) { + if (module.name !== "virtual-desktop-controller") continue; + const tableName = `${clusterName}.${module.module_id}.controller.software-stacks`; + const eolStacks = (await scanModuleTable(deps, tableName)) + .filter((row) => EOL_BASE_OS[valueAsString(row["base_os"])] !== undefined) + .map((row) => ({ + stackId: valueAsString(row["stack_id"]), + baseOs: valueAsString(row["base_os"]), + name: valueAsString(row["name"], ""), + architecture: valueAsString(row["architecture"], ""), + })); + if (eolStacks.length === 0) continue; + + const ids = new Set(eolStacks.map((stack) => stack.stackId)); + const sessions: EolSession[] = []; + for (const row of await scanModuleTable(deps, `${clusterName}.${module.module_id}.controller.user-sessions`)) { + if (valueAsString(row["state"]).toUpperCase() === "DELETED") continue; + const softwareStack = asRecord(row["software_stack"]); + const stackId = valueAsString(softwareStack["stack_id"]); + const baseOs = valueAsString(softwareStack["base_os"]) || valueAsString(row["base_os"]); + if (ids.has(stackId) || EOL_BASE_OS[baseOs] !== undefined) { + sessions.push({ + stackId, + baseOs, + sessionId: valueAsString(row["idea_session_id"], ""), + owner: valueAsString(row["owner"], ""), + name: valueAsString(row["name"], ""), + }); + } + } + + const idsInUse = new Set(sessions.filter((session) => session.stackId !== "").map((session) => session.stackId)); + const baseOsInUse = new Set(sessions.filter((session) => session.stackId === "").map((session) => session.baseOs)); + plans.push({ + tableName, + sessions, + toDelete: eolStacks.filter((stack) => !idsInUse.has(stack.stackId) && !baseOsInUse.has(stack.baseOs)), + toDisable: eolStacks.filter((stack) => idsInUse.has(stack.stackId) || baseOsInUse.has(stack.baseOs)), + }); + } + return plans; +} + +async function checkEolBaseOs( + deps: UpgradeDeps, + options: UpgradeCommandOptions, +): Promise { + const findings = await findEolReferences(deps, options.clusterName); + if (findings.length > 0) { + deps.err("This cluster still references a Base OS that has reached end-of-life and is no longer supported by IDEA."); + for (const finding of findings) deps.err(` - ${finding}`); + throw new ExitWithCode(1); + } + + const plans = await planEolSoftwareStacks(deps, options.clusterName); + const sessions = plans.flatMap((plan) => plan.sessions); + if (sessions.length > 0 && options.disableEolStacksInUse !== true) { + deps.err(`${sessions.length} virtual desktop session(s) still use a Base OS that has reached end-of-life. Nothing has been changed.`); + for (const session of sessions) { + deps.err(` - session ${session.sessionId} owned by ${session.owner} on software stack ${session.stackId || session.baseOs}`); + } + deps.err("Delete these virtual desktops, then re-run upgrade-cluster."); + throw new ExitWithCode(1); + } + + for (const plan of plans) { + for (const stack of plan.toDisable) deps.out(`will disable end-of-life eVDI software stack ${describeStack(stack)}`); + for (const stack of plan.toDelete) deps.out(`will delete end-of-life eVDI software stack ${describeStack(stack)}`); + } + return plans; +} + +async function applyEolSoftwareStacks(deps: UpgradeDeps, awsRegion: string, plans: EolPlan[]): Promise { + let disabled = 0; + for (const plan of plans) { + for (const stack of plan.toDisable) { + await deps.eolSoftwareStacks.setEnabled({ + awsRegion, + tableName: plan.tableName, + baseOs: stack.baseOs, + stackId: stack.stackId, + enabled: false, + }); + disabled += 1; + const inUse = plan.sessions + .filter((session) => session.stackId === stack.stackId || (session.stackId === "" && session.baseOs === stack.baseOs)) + .map((session) => `${session.owner} (${session.name})`) + .join(", "); + deps.out(`disabled end-of-life eVDI software stack ${describeStack(stack)}, still in use by ${inUse}`); + } + for (const stack of plan.toDelete) { + await deps.eolSoftwareStacks.delete({ + awsRegion, + tableName: plan.tableName, + baseOs: stack.baseOs, + stackId: stack.stackId, + }); + deps.out(`deleted end-of-life eVDI software stack ${describeStack(stack)}`); + } + } + if (disabled > 0) { + deps.out(`${disabled} software stack(s) are disabled in DynamoDB but still read as enabled in the eVDI search index until it is reindexed.`); + } +} + +async function resolveUpgradeBaseOs( + deps: UpgradeDeps, + options: Pick, +): Promise { + let current: string[] = []; + try { + current = [...new Set( + (await scanAll(deps, `${options.clusterName}.cluster-settings`)) + .filter((entry) => valueAsString(entry["key"]).endsWith(".base_os")) + .map((entry) => valueAsString(entry["value"])) + .filter((value) => value !== ""), + )].sort(); + } catch (error) { + if (options.baseOs === undefined || options.baseOs === "") { + deps.err(`Could not read the cluster settings to determine the current Base OS: ${(error as Error).message}. Re-run with an explicit --base-os.`); + throw new ExitWithCode(1); + } + } + + if (options.baseOs === undefined || options.baseOs === "") { + if (current.length !== 1) { + const found = current.length === 0 ? "no base_os setting found" : current.join(", "); + deps.err(`Could not determine the Base OS this cluster runs from its settings (${found}). Re-run with an explicit --base-os to say which Base OS every module should use.`); + throw new ExitWithCode(1); + } + const [baseOs] = current; + deps.out(`No --base-os given: keeping the Base OS this cluster runs: ${baseOs}`); + return baseOs; + } + + if (current.length > 0 && !current.includes(options.baseOs)) { + deps.out(`--base-os ${options.baseOs} changes this cluster from ${current.join(", ")}: every module is redeployed onto ${options.baseOs}.`); + } + return options.baseOs; +} + +async function validateBaseOs(deps: UpgradeDeps, options: UpgradeCommandOptions, baseOs: string): Promise { + const replacement = EOL_BASE_OS[baseOs]; + if (replacement !== undefined) { + deps.err(`Base OS ${baseOs} has reached end-of-life and is no longer supported by IDEA. Upgrade to ${replacement} instead.`); + throw new ExitWithCode(1); + } + if (!UPGRADE_BASE_OS.includes(baseOs)) { + deps.err(`Invalid base_os: ${baseOs}. Must be one of: ${UPGRADE_BASE_OS.join(", ")}`); + throw new ExitWithCode(1); + } + if (baseOs !== "rhel10" && baseOs !== "rocky10") return; + let modules: ModuleInfo[]; + try { + modules = await clusterModules(deps, options.clusterName); + } catch (error) { + deps.err(`Could not read the cluster modules table to validate ${baseOs} eVDI compatibility: ${(error as Error).message}`); + throw new ExitWithCode(1); + } + if (modules.some((module) => module.name === "virtual-desktop-controller" && module.status === "deployed")) { + deps.err(`base_os ${baseOs} is not supported on clusters with the virtual-desktop-controller module deployed: Amazon DCV publishes no EL10 packages.`); + throw new ExitWithCode(1); + } +} + +function backupDir(configDir: string, now: number): string { + const date = new Date(now); + const pad = (value: number): string => String(value).padStart(2, "0"); + return `${configDir}.golden.${pad(date.getMonth() + 1)}${pad(date.getDate())}${date.getFullYear()}_${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`; +} + +async function updateValuesBaseOs(deps: UpgradeDeps, options: UpgradeCommandOptions, baseOs: string): Promise { + const path = valuesFilePath(options.clusterName, options.awsRegion); + if (existsSync(path)) { + try { + const bucket = (await scanAll(deps, `${options.clusterName}.cluster-settings`)) + .find((entry) => entry["key"] === "cluster.cluster_s3_bucket")?.["value"]; + if (typeof bucket === "string") { + const remote = await deps.s3.getObject({ Bucket: bucket, Key: VALUES_FILE_S3_KEY }); + if (remote !== readFileSync(path, "utf8")) deps.out("warning: local values.yml differs from the copy in the cluster bucket."); + } + } catch (error) { + deps.out(`warning: could not compare local values.yml with the cluster bucket: ${(error as Error).message}`); + } + } else { + const bucket = (await scanAll(deps, `${options.clusterName}.cluster-settings`)) + .find((entry) => entry["key"] === "cluster.cluster_s3_bucket")?.["value"]; + if (typeof bucket !== "string" || bucket === "") throw new ClusterConfigError("cluster.cluster_s3_bucket is required to restore values.yml"); + deps.out(`values.yml not found at ${path}, restoring it from the cluster bucket ...`); + mkdirSync(join(path, ".."), { recursive: true }); + writeFileSync(path, await deps.s3.getObject({ Bucket: bucket, Key: VALUES_FILE_S3_KEY })); + } + + const values = readFileSync(path, "utf8"); + const updated = values.replace(/^base_os:.*$/m, `base_os: ${baseOs}`); + if (updated === values && !/^base_os:.*$/m.test(values)) { + deps.err(`${path} has no base_os key, so the upgrade cannot set it to ${baseOs}.`); + throw new ExitWithCode(1); + } + writeFileSync(path, updated); + deps.out(`Successfully updated base_os to ${baseOs} in values.yml`); + return path; +} + +async function exportConfiguration(deps: UpgradeDeps, options: UpgradeCommandOptions, configDir: string): Promise { + const entries = await scanAll(deps, `${options.clusterName}.cluster-settings`); + const modules = await clusterModules(deps, options.clusterName); + const tree = buildTree(entries.map((entry) => ({ key: valueAsString(entry["key"]), value: entry["value"] }))); + mkdirSync(configDir, { recursive: true }); + const idea: { modules: Array<{ name: string; id: string; type: string; config_files: string[] }> } = { modules: [] }; + for (const module of modules) { + const moduleDir = join(configDir, module.module_id); + mkdirSync(moduleDir, { recursive: true }); + writeFileSync(join(moduleDir, "settings.yml"), toYaml(tree[module.module_id] ?? {})); + idea.modules.push({ name: module.name, id: module.module_id, type: module.type, config_files: ["settings.yml"] }); + } + writeFileSync(join(configDir, "idea.yml"), toYaml(idea)); +} + +async function backupAndUpdateGlobalSettings(deps: UpgradeDeps, options: UpgradeCommandOptions): Promise { + const regionDir = join(valuesFilePath(options.clusterName, options.awsRegion), ".."); + const configDir = join(regionDir, "config"); + await exportConfiguration(deps, options, configDir); + const golden = backupDir(configDir, deps.now()); + if (existsSync(golden)) rmSync(golden, { recursive: true }); + cpSync(configDir, golden, { recursive: true }); + deps.out(`Backup created successfully at ${golden}`); + + generateConfigFromTemplates(loadValuesFile(join(regionDir, "values.yml")), configDir); + const writer = await deps.configWriter({ + clusterName: options.clusterName, + awsRegion: options.awsRegion, + awsProfile: options.awsProfile, + }); + await writer.deleteConfigEntries("global-settings."); + await writer.syncClusterSettingsInDb(convertConfigToKeyValuePairs(configDir, "global-settings"), true); + return configDir; +} + +async function syncFullConfiguration(deps: UpgradeDeps, options: UpgradeCommandOptions, configDir: string): Promise { + const writer = await deps.configWriter({ + clusterName: options.clusterName, + awsRegion: options.awsRegion, + awsProfile: options.awsProfile, + }); + // A regenerated configuration can describe a module the table has no row for. The deployment + // order reads the table, so an unregistered module is skipped while every other stack deploys. + // Add-only: an existing row keeps its type, status, stack name and version. + await writer.syncModulesInDb( + readModulesFromFiles(configDir).map((module) => ({ id: module.id, name: module.name, type: module.type })), + ); + await writer.syncClusterSettingsInDb(convertConfigToKeyValuePairs(configDir), false); +} + +export function buildAmiUpdateEntries( + amiId: string, + baseOs: string, + modules: ModuleInfo[], + keepKeys: ReadonlySet = new Set(), +): Array<{ key: string; value: string }> { + const entries: Array<{ key: string; value: string }> = []; + for (const module of modules) { + for (const [baseOsKey, amiKey] of AMI_UPDATE_KEYS[module.name] ?? []) { + const keys = [`${module.module_id}.${baseOsKey}`, `${module.module_id}.${amiKey}`]; + if (keys.some((key) => keepKeys.has(key))) continue; + entries.push({ key: keys[0] as string, value: baseOs }, { key: keys[1] as string, value: amiId }); + } + } + return entries; +} + +export function keepBuiltComputeImage(current: InstanceImage | undefined, stock: InstanceImage | undefined): boolean { + return ( + current?.Name?.startsWith(COMPUTE_IMAGE_PREFIX) === true && + typeof current.CreationDate === "string" && + current.CreationDate !== "" && + typeof stock?.CreationDate === "string" && + stock.CreationDate !== "" && + current.CreationDate > stock.CreationDate + ); +} + +async function computeAmiKeepKeys( + deps: UpgradeDeps, + options: Pick, + modules: ModuleInfo[], + amiId: string, + settings: readonly CurrentConfigRow[], +): Promise> { + const scheduler = modules.find((module) => module.name === "scheduler"); + if (scheduler === undefined) return new Set(); + const current = settings.find((entry) => entry.key === `${scheduler.module_id}.compute_node_ami`)?.value; + if (typeof current !== "string" || current === "" || current === amiId) return new Set(); + try { + const images = await deps.ec2.describeImages({ awsRegion: options.awsRegion, imageIds: [current, amiId] }); + const currentImage = images.find((image) => image.ImageId === current); + const stockImage = images.find((image) => image.ImageId === amiId); + if (!keepBuiltComputeImage(currentImage, stockImage)) return new Set(); + deps.out(`keeping built compute image ${current}, newer than the release image ${amiId}`); + return new Set([`${scheduler.module_id}.compute_node_os`, `${scheduler.module_id}.compute_node_ami`]); + } catch (error) { + deps.out(`warning: could not describe compute image ${current} or release image ${amiId}: ${(error as Error).message}. Compute moves to ${amiId}.`); + return new Set(); + } +} + +async function planModuleHostInstanceTypes( + deps: UpgradeDeps, + options: Pick, + modules: ModuleInfo[], + settings: readonly CurrentConfigRow[], +): Promise { + const values = new Map(settings.map((entry) => [entry.key, valueAsString(entry.value)])); + const keys = modules.flatMap((module) => (HOST_INSTANCE_TYPE_KEYS[module.name] ?? []).map((key) => `${module.module_id}.${key}`)) + .filter((key) => values.get(key) !== ""); + const oldKeys = keys.filter((key) => values.get(key) === MODULE_HOST_INSTANCE_TYPE_OLD); + if (oldKeys.length === 0) return []; + let offered: string[]; + try { + offered = await deps.ec2.describeInstanceTypeOfferings({ awsRegion: options.awsRegion, instanceType: MODULE_HOST_INSTANCE_TYPE }); + } catch (error) { + deps.out(`warning: could not read whether this region offers ${MODULE_HOST_INSTANCE_TYPE}: ${(error as Error).message}. Keeping ${MODULE_HOST_INSTANCE_TYPE_OLD}.`); + return []; + } + if (!offered.includes(MODULE_HOST_INSTANCE_TYPE)) { + deps.out(`${MODULE_HOST_INSTANCE_TYPE} is not offered in this region. Keeping ${MODULE_HOST_INSTANCE_TYPE_OLD}.`); + return []; + } + return oldKeys.sort().map((key) => ({ key, value: MODULE_HOST_INSTANCE_TYPE })); +} + +async function planOpenSearchDataNodeInstanceType( + deps: UpgradeDeps, + options: Pick, + modules: ModuleInfo[], + settings: readonly CurrentConfigRow[], +): Promise { + const analytics = modules.find((module) => module.name === "analytics"); + if (analytics === undefined) return []; + const current = settings.find((entry) => entry.key === `${analytics.module_id}.opensearch.data_node_instance_type`)?.value; + if (typeof current !== "string" || current === "" || current !== OPENSEARCH_DATA_NODE_INSTANCE_TYPE_OLD) return []; + const domainName = settings.find((entry) => entry.key === `${analytics.module_id}.opensearch.domain_name`)?.value; + try { + const domain = await deps.openSearch.describeDomain({ + awsRegion: options.awsRegion, + domainName: typeof domainName === "string" ? domainName : undefined, + }); + const offered = await deps.openSearch.listInstanceTypeDetails({ awsRegion: options.awsRegion, engineVersion: domain.engineVersion }); + if (!offered.includes(OPENSEARCH_DATA_NODE_INSTANCE_TYPE)) { + deps.out(`${OPENSEARCH_DATA_NODE_INSTANCE_TYPE} is not offered for ${domain.engineVersion ?? ""} in this region. Keeping analytics data node instance type ${current}.`); + return []; + } + return [{ + key: `${analytics.module_id}.opensearch.data_node_instance_type`, + value: OPENSEARCH_DATA_NODE_INSTANCE_TYPE, + }]; + } catch (error) { + deps.out(`warning: could not read the instance types offered for the analytics domain: ${(error as Error).message}. Keeping analytics data node instance type ${current}.`); + return []; + } +} + +/** Resolve every Phase 3 write before the upgrade asks for approval. */ +export async function planUpgradePhase3Entries( + deps: UpgradeDeps, + options: Pick, + modules: ModuleInfo[], + settings: readonly CurrentConfigRow[], + amiId: string, + baseOs: string, +): Promise { + const keepKeys = await computeAmiKeepKeys(deps, options, modules, amiId, settings); + return [ + ...buildAmiUpdateEntries(amiId, baseOs, modules, keepKeys), + ...await planModuleHostInstanceTypes(deps, options, modules, settings), + ...await planOpenSearchDataNodeInstanceType(deps, options, modules, settings), + ]; +} + +/** Apply the already previewed Phase 3 plan without recalculating it after approval. */ +async function applyPhase3Entries( + writer: Awaited>, + entries: readonly ConfigEntry[], + current: readonly CurrentConfigRow[], + out: (line: string) => void, +): Promise { + const previous = new Map(current.map((entry) => [entry.key, entry.value])); + for (const entry of entries) { + await writer.setConfigEntry(entry.key, entry.value); + if (entry.value === MODULE_HOST_INSTANCE_TYPE && previous.get(entry.key) === MODULE_HOST_INSTANCE_TYPE_OLD) { + out(`${entry.key} moves from ${MODULE_HOST_INSTANCE_TYPE_OLD} to ${MODULE_HOST_INSTANCE_TYPE}; the host runs it when the instance is next replaced`); + } else if ( + entry.value === OPENSEARCH_DATA_NODE_INSTANCE_TYPE && + previous.get(entry.key) === OPENSEARCH_DATA_NODE_INSTANCE_TYPE_OLD + ) { + out(`analytics data nodes move from ${OPENSEARCH_DATA_NODE_INSTANCE_TYPE_OLD} to ${OPENSEARCH_DATA_NODE_INSTANCE_TYPE}. OpenSearch Service applies this as a blue/green deployment.`); + } + } +} + +/** Keep only table fields used by the value-free comparison report. */ +function currentConfigRows(rows: readonly Record[]): CurrentConfigRow[] { + return rows.flatMap((row) => { + const key = row["key"]; + if (typeof key !== "string" || key === "") return []; + return [{ + key, + value: row["value"], + source: typeof row["source"] === "string" ? row["source"] : undefined, + version: typeof row["version"] === "number" ? row["version"] : undefined, + }]; + }); +} + +/** + * Generate the upgrade target in a temporary directory. The cluster's local + * values and generated configuration remain unchanged until approval. + */ +async function generatedPreviewEntries( + deps: UpgradeDeps, + options: ConfigUpgradePreviewOptions, + baseOs: string, + current: readonly CurrentConfigRow[], +): Promise { + const root = mkdtempSync(join(tmpdir(), "ideactl-drift-preview-")); + try { + const configuredValuesPath = options.valuesFile ?? valuesFilePath(options.clusterName, options.awsRegion); + let sourceValuesPath = configuredValuesPath; + if (!existsSync(configuredValuesPath)) { + if (options.valuesFile !== undefined) { + throw new ClusterConfigError(`file not found: ${configuredValuesPath}`); + } + const bucket = current.find((entry) => entry.key === "cluster.cluster_s3_bucket")?.value; + if (typeof bucket !== "string" || bucket === "") { + throw new ClusterConfigError("cluster.cluster_s3_bucket is required to preview a missing values.yml"); + } + sourceValuesPath = join(root, "values.yml"); + writeFileSync(sourceValuesPath, await deps.s3.getObject({ Bucket: bucket, Key: VALUES_FILE_S3_KEY })); + } + + const values = { ...loadValuesFile(sourceValuesPath), base_os: baseOs }; + const configDir = join(root, "config"); + generateConfigFromTemplates(values, configDir); + return convertConfigToKeyValuePairs(configDir); + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +/** + * Build a conservative stack ownership plan from current source markers. + * + * The marker identifies rows a stack owns, not who last edited them. Without a + * resolved target settings map the preview reports the unconditional rewrite + * but does not guess future values or deletions. + */ +function inferredStackPlans( + current: readonly CurrentConfigRow[], + modules: readonly ModuleInfo[], + selectedModuleIds: readonly string[] | undefined, +): StackSettingsPlan[] { + const settings = new Map>(); + for (const row of current) { + if (row.source !== "stack") continue; + const module = modules.find((candidate) => row.key.startsWith(`${candidate.module_id}.`)); + if (module === undefined) continue; + const relativeKey = row.key.slice(module.module_id.length + 1); + const rows = settings.get(module.module_id) ?? []; + rows.push([relativeKey, row.value]); + settings.set(module.module_id, rows); + } + + const selected = new Set(selectedModuleIds ?? []); + const allModules = selected.size === 0; + return modules.flatMap((module) => { + const rows = settings.get(module.module_id); + if (rows === undefined) return []; + return [{ + moduleId: module.module_id, + selected: allModules || selected.has(module.module_id), + previous: Object.fromEntries(rows), + }]; + }); +} + +/** + * Read and generate every input needed by both preview entry points. + * + * Replay callers may inject a complete input. The normal path reads the table, + * resolves Phase 3 conditions, and generates configuration only in a temporary + * directory. + */ +export async function prepareUpgradeDriftInput( + deps: UpgradeDeps, + options: ConfigUpgradePreviewOptions, +): Promise { + if (deps.loadUpgradeDriftInput !== undefined) return deps.loadUpgradeDriftInput(options); + + const baseOs = options.baseOs ?? await resolveUpgradeBaseOs(deps, options); + const current = currentConfigRows(await scanAll(deps, `${options.clusterName}.cluster-settings`)); + const modules = await clusterModules(deps, options.clusterName); + const amiId = resolveRegionAmi( + (deps.regionAmiConfig ?? loadRegionAmiConfig)(), + options.awsRegion, + baseOs, + ); + const generated = await generatedPreviewEntries(deps, options, baseOs, current); + const phase3 = await planUpgradePhase3Entries(deps, options, modules, current, amiId, baseOs); + + return { + current, + generated, + phase3, + stacks: inferredStackPlans(current, modules, options.modules), + replaceGlobalSettings: options.skipGlobalSettingsUpdate !== true, + syncFullConfiguration: true, + }; +} + +async function moduleInstances(deps: UpgradeDeps, options: UpgradeCommandOptions): Promise { + const instances: ClearedInstance[] = []; + for (const module of await clusterModules(deps, options.clusterName)) { + const stackName = valueAsString(module.stack_name) || `${options.clusterName}-${module.module_id}`; + let nextToken: string | undefined; + try { + do { + const page = await deps.cloudFormation.listStackResources({ awsRegion: options.awsRegion, stackName, nextToken }); + instances.push(...page.instanceIds.map((instanceId) => ({ stackName, instanceId }))); + nextToken = page.nextToken; + } while (nextToken !== undefined); + } catch (error) { + const errorShape = error as { name?: string; message?: string }; + if (errorShape.name === "ValidationError" && errorShape.message?.includes("does not exist") === true) continue; + throw error; + } + } + return instances; +} + +async function clearTerminationProtection( + deps: UpgradeDeps, + awsRegion: string, + instances: ClearedInstance[], +): Promise { + const cleared: ClearedInstance[] = []; + for (const instance of instances) { + try { + if (!await deps.ec2.describeInstanceAttribute({ awsRegion, instanceId: instance.instanceId })) continue; + await deps.ec2.modifyInstanceAttribute({ awsRegion, instanceId: instance.instanceId, protected: false }); + cleared.push(instance); + deps.out(`cleared instance termination protection on ${instance.instanceId} (${instance.stackName})`); + } catch (error) { + deps.out(`warning: could not clear termination protection on ${instance.instanceId} (${instance.stackName}): ${(error as Error).message}.`); + } + } + return cleared; +} + +async function restoreTerminationProtection(deps: UpgradeDeps, awsRegion: string, cleared: ClearedInstance[]): Promise { + if (cleared.length === 0) return; + const alive = new Set(await deps.ec2.describeLiveInstances({ awsRegion, instanceIds: cleared.map((instance) => instance.instanceId) })); + for (const instance of cleared) { + if (!alive.has(instance.instanceId)) { + deps.out(`${instance.instanceId} (${instance.stackName}) was replaced by the upgrade, so it has no termination protection to restore`); + continue; + } + try { + await deps.ec2.modifyInstanceAttribute({ awsRegion, instanceId: instance.instanceId, protected: true }); + deps.out(`restored instance termination protection on ${instance.instanceId} (${instance.stackName})`); + } catch (error) { + deps.out(`warning: could not restore termination protection on ${instance.instanceId} (${instance.stackName}): ${(error as Error).message}. Re-enable it by hand.`); + } + } +} + +function warnClearedProtection(deps: UpgradeDeps, cleared: ClearedInstance[]): void { + if (cleared.length > 0) { + deps.out(`warning: termination protection is still cleared on ${cleared.map((instance) => instance.instanceId).join(", ")}. Re-enable it by hand once the cluster is stable.`); + } +} + +async function saveValuesFile(deps: UpgradeDeps, options: UpgradeCommandOptions): Promise { + let bucket: string | undefined; + try { + const found = (await scanAll(deps, `${options.clusterName}.cluster-settings`)) + .find((entry) => entry["key"] === "cluster.cluster_s3_bucket")?.["value"]; + bucket = typeof found === "string" ? found : undefined; + if (typeof bucket !== "string" || bucket === "") throw new ClusterConfigError("cluster.cluster_s3_bucket is required"); + await deps.s3.putObject({ + Bucket: bucket, + Key: VALUES_FILE_S3_KEY, + Body: readFileSync(valuesFilePath(options.clusterName, options.awsRegion)), + }); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + const location = + typeof bucket === "string" && bucket !== "" + ? `s3://${bucket}/${VALUES_FILE_S3_KEY}` + : "the cluster bucket"; + throw new ClusterConfigError( + `Upgrade of ${options.clusterName} finished its stack steps, but values.yml was not saved to ${location} (${detail}). The cluster bucket still has the previous file. Retry ideactl config save-values --cluster-name ${options.clusterName} --aws-region ${options.awsRegion}. Until that works, a run on another machine can restore the old values.yml.`, + ); + } +} + +async function defaultDeployment(deps: Deps, options: UpgradeDeploymentOptions): Promise { + const helper = await DeploymentHelper.open({ + clusterName: options.clusterName, + awsRegion: options.awsRegion, + awsProfile: options.awsProfile, + terminationProtection: options.terminationProtection, + deploymentId: options.deploymentId, + upgrade: true, + moduleSet: options.moduleSet, + allModules: options.allModules, + forceBuildBootstrap: options.forceBuildBootstrap, + optimizeDeployment: options.optimizeDeployment, + moduleIds: options.moduleIds, + rollback: options.rollback, + deps, + }); + await helper.invoke(); +} + +/** + * Stop where a value the upgrade overwrites differs from what the generator would produce, and + * nowhere else. An upgrade whose rows all match proceeds without asking: a question asked on every + * run is a question nobody reads. `--force` skips confirmations; it does not accept losing an edit, + * so accepting these rows in an unattended run needs the flag that says only that. + */ +async function confirmConfigDrift( + deps: UpgradeDeps, + options: UpgradeCommandOptions, + report: UpgradeDriftReport, +): Promise { + const atRisk = report.changedRowsDifferingFromGenerated; + if (atRisk.length === 0) return; + + deps.err( + `${atRisk.length} configuration row(s) hold a value this upgrade overwrites, and the value differs from generated configuration: ${atRisk.join(", ")}`, + ); + if (options.acceptConfigDrift === true) { + deps.out("--accept-config-drift: overwriting the rows above"); + return; + } + if (options.force === true) { + deps.err( + "Reconcile those rows, or re-run with --accept-config-drift. --force skips confirmations and does not cover them.", + ); + throw new ExitWithCode(1); + } + const confirm = await deps.prompt({ + message: `Overwrite the ${atRisk.length} row(s) above and continue with the cluster upgrade?`, + default: false, + }); + if (confirm !== true && confirm !== "Yes") throw new ExitWithCode(0); +} + +/** + * Whether the run will deploy the container module. An upgrade with no module list deploys every + * module the cluster has, so the operator's arguments are not the question: what the deployment + * reaches is, the same way `runDeploy` asks `helper.getDeploymentModuleNames()`. + */ +async function upgradeReachesEcs(deps: UpgradeDeps, options: UpgradeCommandOptions): Promise { + const requested = options.modules ?? []; + if (requested.length > 0) return requested.includes(ECS_MODULE); + const modules = await clusterModules(deps, options.clusterName); + return modules.some((module) => module.module_id === ECS_MODULE || module.name === ECS_MODULE); +} + +/** Execute Phases 1 through 4 after every pre-flight refusal has passed. */ +export async function upgradeCluster(deps: UpgradeDeps, options: UpgradeCommandOptions): Promise { + if (await upgradeReachesEcs(deps, options)) { + await checkAwsvpcTrunking(deps, options); + } + const baseOs = await resolveUpgradeBaseOs(deps, options); + await validateBaseOs(deps, options, baseOs); + const eolPlans = await checkEolBaseOs(deps, options); + const allModules = options.modules === undefined || options.modules.length === 0; + deps.out(allModules ? "No modules specified, upgrading all modules" : `Upgrade scope: Specific modules - ${options.modules?.join(", ")}`); + const driftInput = await prepareUpgradeDriftInput(deps, { ...options, baseOs }); + const driftReport = compareUpgradeDrift(driftInput); + deps.out(renderUpgradeDrift(driftReport)); + await confirmConfigDrift(deps, options, driftReport); + await applyEolSoftwareStacks(deps, options.awsRegion, eolPlans); + + let cleared: ClearedInstance[] = []; + try { + deps.out("Phase 1: Update Base OS in values.yml"); + resolveRegionAmi((deps.regionAmiConfig ?? loadRegionAmiConfig)(), options.awsRegion, baseOs); + await updateValuesBaseOs(deps, options, baseOs); + + let configDir = join(valuesFilePath(options.clusterName, options.awsRegion), "..", "config"); + if (options.skipGlobalSettingsUpdate !== true) { + if (options.force !== true) { + const confirm = await deps.prompt({ message: "Continue with global settings backup and update?", default: true }); + if (confirm !== true && confirm !== "Yes") throw new ExitWithCode(0); + } + deps.out("Phase 2: Global Settings Backup and Update"); + configDir = await backupAndUpdateGlobalSettings(deps, options); + } + + let syncFullConfig = options.force === true; + if (options.force !== true) { + const confirm = await deps.prompt({ message: "Sync full configuration to add new values?", default: true }); + syncFullConfig = confirm === true || confirm === "Yes"; + } + if (syncFullConfig) { + if (options.skipGlobalSettingsUpdate === true) { + generateConfigFromTemplates(loadValuesFile(valuesFilePath(options.clusterName, options.awsRegion)), configDir); + } + deps.out("Phase 2b: Sync full configuration without overwrite"); + await syncFullConfiguration(deps, options, configDir); + } + + deps.out("Phase 3: Update AMI IDs and Settings"); + let updateAmis = options.force === true; + if (options.force !== true) { + const confirm = await deps.prompt({ message: "Continue with AMI and settings updates?", default: true }); + updateAmis = confirm === true || confirm === "Yes"; + } + if (updateAmis) { + const writer = await deps.configWriter({ clusterName: options.clusterName, awsRegion: options.awsRegion, awsProfile: options.awsProfile }); + await applyPhase3Entries(writer, driftInput.phase3 ?? [], driftInput.current, deps.out); + } + + deps.out("Phase 4: Module Deployment"); + if (!options.force && allModules) { + const confirm = await deps.prompt({ message: "Proceed with deploying all modules?", default: true }); + if (confirm !== true && confirm !== "Yes") throw new ExitWithCode(0); + } + try { + cleared = await clearTerminationProtection(deps, options.awsRegion, await moduleInstances(deps, options)); + } catch (error) { + deps.out(`warning: pre-upgrade termination-protection sweep failed: ${(error as Error).message}. Verify replaced instances are terminated after the upgrade.`); + } + const deployment: UpgradeDeploymentOptions = { + clusterName: options.clusterName, + awsRegion: options.awsRegion, + awsProfile: options.awsProfile, + terminationProtection: asBoolFlag(options.terminationProtection, true), + deploymentId: options.deploymentId, + forceBuildBootstrap: options.forceBuildBootstrap === true, + rollback: options.rollback !== false, + optimizeDeployment: options.optimizeDeployment === true, + moduleSet: options.moduleSet, + allModules, + moduleIds: allModules ? undefined : options.modules, + }; + await deps.deploy(deployment); + await restoreTerminationProtection(deps, options.awsRegion, cleared); + await saveValuesFile(deps, options); + deps.out("All upgrade phases completed successfully"); + } catch (error) { + warnClearedProtection(deps, cleared); + throw error; + } +} + +/** Register the command group. The caller supplies all replayable external effects. */ +export function registerUpgradeCommands(program: Command, deps: UpgradeDeps): void { + program + .command("upgrade-cluster") + .description("upgrade an existing cluster") + .requiredOption("--cluster-name ", "Cluster Name") + .requiredOption("--aws-region ", "AWS Region") + .option("--aws-profile ", "AWS Profile Name") + .option("--termination-protection ", "Set termination protection to true or false. Default: true", "true") + .option("--deployment-id ", "A UUID to identify the deployment.") + .option("--base-os ", "Base OS to upgrade to.") + .option("--force-build-bootstrap", "Render bootstrap packages again.") + .option("--rollback", "Rollback stack to stable state on failure. Default.", true) + .option("--no-rollback", "Do not roll back on failure.") + .option("--optimize-deployment", "Deploy applicable stacks in parallel.") + .option("--module-set ", "Name of the ModuleSet. Default: default", "default") + .option("--force", "Skip all confirmation prompts.") + .option( + "--accept-config-drift", + "Overwrite configuration rows whose value differs from generated configuration. Not covered by --force.", + ) + .option("--skip-global-settings-update", "Skip updating global settings.") + .option("--disable-eol-stacks-in-use", "Disable end-of-life eVDI software stacks that are in use.") + .argument("[modules...]", "module ids") + .action(async (modules: string[], commandOptions: UpgradeCommandOptions) => { + await upgradeCluster(deps, { ...commandOptions, modules }); + }); +} + +/** + * Attach live SDK implementations to the command-core dependencies. Imports + * occur only when an upgrade operation reaches the corresponding phase. + */ +/** + * The live account-settings reader behind the container pre-flight. Deploy, quick-setup and + * upgrade all run that pre-flight, so they all need it: without it the check cannot read the + * account and refuses every container deployment. + */ +export function liveEcsAccountSettings(): EcsAccountSettingsApi { + return { + async listAccountSettings(input) { + const { ECSClient, ListAccountSettingsCommand } = await import("@aws-sdk/client-ecs"); + const client = new ECSClient(await upgradeLiveClientOptions(input.awsRegion, input.awsProfile)); + const result = await client.send( + new ListAccountSettingsCommand({ effectiveSettings: input.effectiveSettings, name: input.name }), + ); + return (result.settings ?? []).flatMap((setting) => + setting.name === undefined || setting.value === undefined + ? [] + : [{ name: setting.name.toString(), value: setting.value }], + ); + }, + }; +} + +export function createLiveUpgradeDeps(deps: Deps): UpgradeDeps { + const ecsAccountSettings = liveEcsAccountSettings(); + const ec2: UpgradeEc2Api = { + async describeImages(input) { + const { DescribeImagesCommand, EC2Client } = await import("@aws-sdk/client-ec2"); + const result = await new EC2Client(await awsClientOptions(input.awsRegion)).send( + new DescribeImagesCommand({ ImageIds: input.imageIds }), + ); + return (result.Images ?? []).map((image) => ({ + ImageId: image.ImageId, + Name: image.Name, + CreationDate: image.CreationDate, + })); + }, + async describeInstanceTypeOfferings(input) { + const { DescribeInstanceTypeOfferingsCommand, EC2Client } = await import("@aws-sdk/client-ec2"); + const result = await new EC2Client(await awsClientOptions(input.awsRegion)).send( + new DescribeInstanceTypeOfferingsCommand({ + LocationType: "region", + Filters: [{ Name: "instance-type", Values: [input.instanceType] }], + }), + ); + return (result.InstanceTypeOfferings ?? []).flatMap((offering) => + offering.InstanceType === undefined ? [] : [offering.InstanceType.toString()], + ); + }, + async describeInstanceAttribute(input) { + const { DescribeInstanceAttributeCommand, EC2Client } = await import("@aws-sdk/client-ec2"); + const result = await new EC2Client(await awsClientOptions(input.awsRegion)).send( + new DescribeInstanceAttributeCommand({ InstanceId: input.instanceId, Attribute: "disableApiTermination" }), + ); + return result.DisableApiTermination?.Value === true; + }, + async modifyInstanceAttribute(input) { + const { EC2Client, ModifyInstanceAttributeCommand } = await import("@aws-sdk/client-ec2"); + await new EC2Client(await awsClientOptions(input.awsRegion)).send( + new ModifyInstanceAttributeCommand({ + InstanceId: input.instanceId, + DisableApiTermination: { Value: input.protected }, + }), + ); + }, + async describeLiveInstances(input) { + const { DescribeInstancesCommand, EC2Client } = await import("@aws-sdk/client-ec2"); + const result = await new EC2Client(await awsClientOptions(input.awsRegion)).send( + new DescribeInstancesCommand({ + Filters: [ + { Name: "instance-id", Values: input.instanceIds }, + { Name: "instance-state-name", Values: ["pending", "running", "stopping", "stopped"] }, + ], + }), + ); + return (result.Reservations ?? []).flatMap((reservation) => reservation.Instances ?? []) + .flatMap((instance) => instance.InstanceId === undefined ? [] : [instance.InstanceId]); + }, + }; + + return { + ...deps, + ec2, + ecsAccountSettings, + cloudFormation: { + async listStackResources(input) { + const { CloudFormationClient, ListStackResourcesCommand } = await import("@aws-sdk/client-cloudformation"); + const result = await new CloudFormationClient(await awsClientOptions(input.awsRegion)).send( + new ListStackResourcesCommand({ StackName: input.stackName, NextToken: input.nextToken }), + ); + return { + instanceIds: (result.StackResourceSummaries ?? []) + .filter((resource) => resource.ResourceType === "AWS::EC2::Instance") + .map((resource) => resource.PhysicalResourceId) + .filter((instanceId): instanceId is string => instanceId !== undefined), + nextToken: result.NextToken, + }; + }, + }, + openSearch: { + async describeDomain(input) { + const { DescribeDomainCommand, OpenSearchClient } = await import("@aws-sdk/client-opensearch"); + const result = await new OpenSearchClient(await awsClientOptions(input.awsRegion)).send( + new DescribeDomainCommand({ DomainName: input.domainName }), + ); + return { engineVersion: result.DomainStatus?.EngineVersion }; + }, + async listInstanceTypeDetails(input) { + const { ListInstanceTypeDetailsCommand, OpenSearchClient } = await import("@aws-sdk/client-opensearch"); + const result = await new OpenSearchClient(await awsClientOptions(input.awsRegion)).send( + new ListInstanceTypeDetailsCommand({ EngineVersion: input.engineVersion }), + ); + return (result.InstanceTypeDetails ?? []).flatMap((detail) => + detail.InstanceType === undefined ? [] : [detail.InstanceType.toString()], + ); + }, + }, + eolSoftwareStacks: { + async setEnabled(input) { + const { DynamoDBClient } = await import("@aws-sdk/client-dynamodb"); + const { DynamoDBDocumentClient, UpdateCommand } = await import("@aws-sdk/lib-dynamodb"); + const client = DynamoDBDocumentClient.from(new DynamoDBClient(await awsClientOptions(input.awsRegion))); + await client.send( + new UpdateCommand({ + TableName: input.tableName, + Key: { base_os: input.baseOs, stack_id: input.stackId }, + UpdateExpression: "SET #enabled = :enabled", + ExpressionAttributeNames: { "#enabled": "enabled" }, + ExpressionAttributeValues: { ":enabled": input.enabled }, + }), + ); + }, + async delete(input) { + const { DynamoDBClient } = await import("@aws-sdk/client-dynamodb"); + const { DeleteCommand, DynamoDBDocumentClient } = await import("@aws-sdk/lib-dynamodb"); + const client = DynamoDBDocumentClient.from(new DynamoDBClient(await awsClientOptions(input.awsRegion))); + await client.send( + new DeleteCommand({ + TableName: input.tableName, + Key: { base_os: input.baseOs, stack_id: input.stackId }, + }), + ); + }, + }, + deploy: (options) => defaultDeployment(deps, options), + }; +} + +/** Build the standard deployment adapter for callers that have no special deployment hook. */ +export function withDefaultUpgradeDeployment(deps: Omit): UpgradeDeps { + // Layered rather than copied: a copy drops the prototype methods of a class based deps object + // and hides anything the caller replaces after this returns. + return new Proxy(deps, { + get: (target, property, receiver) => + property === "deploy" + ? (options: UpgradeDeploymentOptions) => defaultDeployment(deps, options) + : Reflect.get(target, property, receiver), + }) as UpgradeDeps; +} diff --git a/source/idea/ideactl/src/cli/commands/utils.ts b/source/idea/ideactl/src/cli/commands/utils.ts new file mode 100644 index 00000000..e409bf78 --- /dev/null +++ b/source/idea/ideactl/src/cli/commands/utils.ts @@ -0,0 +1,261 @@ +/** Utility operator commands for service discovery and managed prefix lists. */ + +import { cpSync, existsSync, rmSync } from "node:fs"; + +import { Command } from "commander"; + +import { ClusterConfig, ClusterConfigError, isEmpty } from "../../config/cluster-config.ts"; +import { convertConfigToKeyValuePairs, generateConfigFromTemplates } from "../../config/generator.ts"; +import { loadValuesFile } from "../../config/values.ts"; +import { clusterConfigDir, valuesFilePath } from "../cdk-invoker.ts"; +import { renderTable } from "./config.ts"; +import { registerDirectoryServiceCommands, type DirectoryServiceDeps, type DirectoryServiceDepsFactory } from "./directoryservice.ts"; +import { registerSharedStorageCommands, type SharedStorageDeps, type SharedStorageDepsFactory } from "./shared-storage.ts"; +import { registerSsoCommands, type SsoDeps, type SsoDepsFactory } from "./sso.ts"; +import { registerSupportCommands, type SupportDeps, type SupportDepsFactory } from "./support.ts"; +import { registerIntegrationTestCommands, type IntegrationTestDeps, type IntegrationTestDepsFactory } from "./tests.ts"; + +export interface ServiceInfo { + title: string; + required: boolean; +} + +export const IDEA_SERVICES: Readonly> = { + acm: { title: "AWS Certificate Manager (ACM)", required: true }, "acm-pca": { title: "ACM Private CA", required: false }, + aps: { title: "Amazon Managed Service for Prometheus", required: false }, backup: { title: "AWS Backup", required: true }, + budgets: { title: "AWS Budgets", required: false }, cloudformation: { title: "AWS CloudFormation", required: true }, + cloudwatch: { title: "Amazon CloudWatch", required: true }, "cognito-idp": { title: "Amazon Cognito - User Pools", required: true }, + ds: { title: "AWS Directory Service for Microsoft Active Directory", required: false }, dynamodb: { title: "Amazon DynamoDB", required: true }, + dynamodbstreams: { title: "Amazon DynamoDB Streams", required: true }, ebs: { title: "Amazon Elastic Block Store (EBS)", required: true }, + ec2: { title: "Amazon Elastic Compute Cloud (EC2)", required: true }, efs: { title: "Amazon Elastic File System (EFS)", required: true }, + elb: { title: "Amazon Elastic Load Balancing (ELB)", required: true }, es: { title: "Amazon OpenSearch Service", required: true }, + eventbridge: { title: "Amazon EventBridge", required: true }, events: { title: "Amazon Events", required: true }, + filecache: { title: "Amazon File Cache", required: false }, fsx: { title: "Amazon FSx", required: false }, + "fsx-lustre": { title: "Amazon FSx for Lustre", required: false }, "fsx-ontap": { title: "Amazon FSx for NetApp ONTAP", required: false }, + "fsx-openzfs": { title: "Amazon FSx for OpenZFS", required: false }, "fsx-windows": { title: "Amazon FSx for Windows File Server", required: false }, + grafana: { title: "Amazon Managed Grafana", required: false }, iam: { title: "AWS Identity and Access Management (IAM)", required: true }, + kinesis: { title: "Amazon Kinesis", required: true }, kms: { title: "AWS Key Management Service (KMS)", required: true }, + lambda: { title: "AWS Lambda", required: true }, logs: { title: "Amazon CloudWatch Logs", required: true }, + pricing: { title: "AWS Pricing API", required: false }, route53: { title: "Amazon Route 53", required: true }, + route53resolver: { title: "Amazon Route 53 Resolver", required: false }, s3: { title: "Amazon Simple Storage Service (S3)", required: true }, + secretsmanager: { title: "AWS Secrets Manager", required: true }, "service-quotas": { title: "AWS Service Quotas", required: true }, + ses: { title: "Amazon Simple Email Service (SES)", required: false }, sns: { title: "Amazon Simple Notification Service (SNS)", required: true }, + sqs: { title: "Amazon Simple Queue Service (SQS)", required: true }, ssm: { title: "AWS Systems Manager (SSM)", required: true }, + sts: { title: "AWS Security Token Service (STS)", required: true }, vpc: { title: "Amazon Virtual Private Cloud (VPC)", required: true }, +}; + +const GATEWAY_ENDPOINTS = ["s3", "dynamodb"]; +const INTERFACE_ENDPOINTS = ["application-autoscaling", "autoscaling", "cloudformation", "ec2", "ec2messages", "ebs", "elasticfilesystem", "elasticfilesystem-fips", "elasticloadbalancing", "logs", "monitoring", "secretsmanager", "sns", "sqs", "events", "ssm", "ssmmessages", "fsx", "fsx-fips", "backup", "grafana", "acm-pca", "kinesis-streams"]; + +export interface UtilsApi { + getParametersByPath(input: { Path: string; NextToken?: string }): Promise<{ Parameters?: Array<{ Value?: string }>; NextToken?: string }>; + describeVpcEndpointServices(input?: { Filters?: Array<{ Name: string; Values: string[] }> }): Promise<{ + ServiceDetails?: Array<{ ServiceName?: string; ServiceType?: Array<{ ServiceType?: string }>; AvailabilityZones?: string[] }>; + }>; + getManagedPrefixListEntries(input: { PrefixListId: string; NextToken?: string }): Promise<{ Entries?: Array<{ Cidr?: string; Description?: string }>; NextToken?: string }>; + describeManagedPrefixLists(input: { PrefixListIds: string[] }): Promise<{ PrefixLists?: Array<{ Version?: number }> }>; + modifyManagedPrefixList(input: { PrefixListId: string; CurrentVersion: number; AddEntries?: Array<{ Cidr: string; Description?: string }>; RemoveEntries?: Array<{ Cidr: string }> }): Promise; +} + +export interface UtilsDeps { + api: UtilsApi; + config: ClusterConfig; + dnsSuffix(): Promise; + syncGlobalSettings?: (input: { deletePrefix: string; entries: Array<{ key: string; value: unknown }> }) => Promise; + exportConfig?: (input: { clusterName: string; awsRegion: string; moduleSet?: string; configDir: string }) => Promise; + now?: () => Date; + prompt?: (message: string) => Promise; + out(line: string): void; +} + +export type UtilsDepsFactory = (options: { + clusterName?: string; + awsRegion?: string; + awsProfile?: string; + moduleSet?: string; +}) => Promise; + +type UtilsDepsSource = UtilsDeps | UtilsDepsFactory; + +/** Dependencies for every command group owned by the remaining operator commands. */ +export interface RemainingOperatorCommandDeps { + sso: SsoDeps | SsoDepsFactory; + directoryService: DirectoryServiceDeps | DirectoryServiceDepsFactory; + sharedStorage: SharedStorageDeps | SharedStorageDepsFactory; + utils: UtilsDeps | UtilsDepsFactory; + support: SupportDeps | SupportDepsFactory; + integrationTests: IntegrationTestDeps | IntegrationTestDepsFactory; +} + +/** Build the stable service matrix printed by `utils aws-services`. */ +export function awsServicesTable(): string { + return renderTable(["AWS Service", "Name", "Required"], Object.keys(IDEA_SERVICES).sort().map((name) => { + const service = IDEA_SERVICES[name] as ServiceInfo; + return [service.title, name, service.required ? "Yes" : "No"]; + })); +} + +/** Read all SSM pages and print each service availability for every requested region. */ +export async function awsServiceAvailability(deps: UtilsDeps, regions: readonly string[]): Promise { + const servicesByRegion = new Map>(); + for (const region of regions) { + const available = new Set(); + let token: string | undefined; + do { + const page = await deps.api.getParametersByPath({ Path: `/aws/service/global-infrastructure/regions/${region}/services`, NextToken: token }); + for (const parameter of page.Parameters ?? []) if (parameter.Value !== undefined) available.add(parameter.Value); + token = page.NextToken; + } while (token !== undefined); + servicesByRegion.set(region, available); + } + const rows = Object.keys(IDEA_SERVICES).sort().map((name) => { + const service = IDEA_SERVICES[name] as ServiceInfo; + return [`${service.title} [${name}]`, service.required ? "Yes" : "No", ...regions.map((region) => servicesByRegion.get(region)?.has(name) === true ? "Yes" : "No")]; + }); + return renderTable(["Service", "Required", ...regions], rows); +} + +/** Return supported endpoint services from the API response. */ +export async function vpcEndpointServiceInfo(deps: UtilsDeps, region: string): Promise { + const suffixTokens = (await deps.dnsSuffix()).split(".").reverse(); + const domain = suffixTokens.join("."); + const requested = [...GATEWAY_ENDPOINTS, ...INTERFACE_ENDPOINTS].map((shortName) => `${domain}.${region}.${shortName}`); + const details = (await deps.api.describeVpcEndpointServices()).ServiceDetails ?? []; + const rows: string[][] = []; + for (const serviceName of requested) { + const matching = details.filter((detail) => detail.ServiceName === serviceName); + if (matching.length === 0) rows.push([serviceName, "No", "-", "-"]); + for (const detail of matching) for (const type of detail.ServiceType ?? []) { + rows.push([serviceName, "Yes", type.ServiceType ?? "", (detail.AvailabilityZones ?? []).join(", ")]); + } + } + return renderTable(["Service Name", `Is Available in ${region}`, "Service Type", "Availability Zones"], rows); +} + +async function prefixListId(config: ClusterConfig): Promise { + const value = config.getString("cluster.network.cluster_prefix_list_id", undefined, { required: true }); + if (isEmpty(value)) throw new ClusterConfigError("cluster.network.cluster_prefix_list_id is required"); + return value as string; +} + +/** Scan all prefix-list entry pages. */ +export async function prefixListEntries(deps: UtilsDeps): Promise> { + const id = await prefixListId(deps.config); + const entries: Array<{ cidr: string; description?: string }> = []; + let token: string | undefined; + do { + const page = await deps.api.getManagedPrefixListEntries({ PrefixListId: id, NextToken: token }); + for (const entry of page.Entries ?? []) if (entry.Cidr !== undefined) entries.push({ cidr: entry.Cidr, description: entry.Description }); + token = page.NextToken; + } while (token !== undefined); + return entries; +} + +async function currentVersion(deps: UtilsDeps, id: string): Promise { + const version = (await deps.api.describeManagedPrefixLists({ PrefixListIds: [id] })).PrefixLists?.[0]?.Version; + if (version === undefined) throw new ClusterConfigError(`cluster prefix list not found: ${id}`); + return Math.trunc(version); +} + +export async function addPrefixListEntry(deps: UtilsDeps, cidr: string, description: string): Promise { + if (isEmpty(cidr)) throw new ClusterConfigError("cidr is required"); + const id = await prefixListId(deps.config); + if ((await prefixListEntries(deps)).some((entry) => entry.cidr === cidr)) throw new ClusterConfigError(`CIDR: ${cidr} already exists in cluster prefix list: ${id}`); + await deps.api.modifyManagedPrefixList({ PrefixListId: id, CurrentVersion: await currentVersion(deps, id), AddEntries: [{ Cidr: cidr, ...(isEmpty(description) ? {} : { Description: description }) }] }); + deps.out(`CIDR: ${cidr} added to cluster prefix list: ${id}.`); +} + +export async function removePrefixListEntry(deps: UtilsDeps, cidr: string): Promise { + if (isEmpty(cidr)) throw new ClusterConfigError("cidr is required"); + const id = await prefixListId(deps.config); + if (!(await prefixListEntries(deps)).some((entry) => entry.cidr === cidr)) throw new ClusterConfigError(`CIDR: ${cidr} not found in cluster prefix list: ${id}`); + await deps.api.modifyManagedPrefixList({ PrefixListId: id, CurrentVersion: await currentVersion(deps, id), RemoveEntries: [{ Cidr: cidr }] }); + deps.out(`CIDR: ${cidr} was removed from cluster prefix list: ${id}.`); +} + +function backupSuffix(now: Date): string { + const part = (value: number): string => String(value).padStart(2, "0"); + return `${part(now.getUTCMonth() + 1)}${part(now.getUTCDate())}${now.getUTCFullYear()}_${part(now.getUTCHours())}${part(now.getUTCMinutes())}${part(now.getUTCSeconds())}`; +} + +/** + * Export the local configuration as a timestamped golden directory, regenerate + * it from values, then replace only global settings in the backing store. + */ +export async function backupUpdateGlobalSettings( + deps: UtilsDeps, + options: { clusterName: string; awsRegion: string; force?: boolean; moduleSet?: string }, +): Promise { + if (deps.syncGlobalSettings === undefined) throw new ClusterConfigError("global settings writer is not configured"); + if (deps.exportConfig === undefined) throw new ClusterConfigError("configuration export is not configured"); + if (options.force !== true && deps.prompt !== undefined && !(await deps.prompt("Continue with global settings backup and update?"))) { + deps.out("Operation aborted by user"); + return ""; + } + const configDir = clusterConfigDir(options.clusterName, options.awsRegion); + const backupDir = `${configDir}.golden.${backupSuffix((deps.now ?? (() => new Date()))())}`; + await deps.exportConfig({ clusterName: options.clusterName, awsRegion: options.awsRegion, configDir }); + if (existsSync(backupDir)) rmSync(backupDir, { recursive: true, force: true }); + if (!existsSync(configDir)) throw new ClusterConfigError(`config directory not found: ${configDir}`); + cpSync(configDir, backupDir, { recursive: true }); + const valuesPath = valuesFilePath(options.clusterName, options.awsRegion); + const values = loadValuesFile(valuesPath); + generateConfigFromTemplates(values, configDir); + const entries = convertConfigToKeyValuePairs(configDir, "global-settings"); + await deps.syncGlobalSettings({ deletePrefix: "global-settings.", entries }); + deps.out("Global settings backup and update completed successfully"); + return backupDir; +} + +/** Register `utils`, `utils vpc-endpoints`, and `utils cluster-prefix-list`. */ +export function registerUtilsCommands(program: Command, deps: UtilsDepsSource): Command { + const resolveDeps = async (options: { + clusterName?: string; + awsRegion?: string; + awsProfile?: string; + moduleSet?: string; + }): Promise => typeof deps === "function" ? deps(options) : deps; + const utils = program.command("utils").description("utility commands"); + utils.command("aws-services").action(async () => (await resolveDeps({})).out(awsServicesTable())); + utils.command("check-aws-services").option("--aws-profile ").argument("").action(async (regions: string[], options: { awsProfile?: string }) => { + const actionDeps = await resolveDeps({ awsRegion: regions[0], awsProfile: options.awsProfile }); + actionDeps.out(await awsServiceAvailability(actionDeps, regions)); + }); + const endpoints = utils.command("vpc-endpoints").description("vpc endpoint commands"); + endpoints.command("service-info").requiredOption("--aws-region ").option("--aws-profile ").action(async (options: { awsRegion: string; awsProfile?: string }) => { + const actionDeps = await resolveDeps(options); + actionDeps.out(await vpcEndpointServiceInfo(actionDeps, options.awsRegion)); + }); + const prefixes = utils.command("cluster-prefix-list").description("cluster prefix list commands"); + const shared = (command: Command): Command => command.requiredOption("--cluster-name ").requiredOption("--aws-region ").option("--aws-profile "); + shared(prefixes.command("show")).action(async (options: { clusterName: string; awsRegion: string; awsProfile?: string }) => { + const actionDeps = await resolveDeps(options); + actionDeps.out(renderTable(["CIDR", "Description"], (await prefixListEntries(actionDeps)).map((entry) => [entry.cidr, entry.description ?? "-"]))); + }); + shared(prefixes.command("add-entry")).requiredOption("--cidr ").requiredOption("--description ").action(async (options: { clusterName: string; awsRegion: string; awsProfile?: string; cidr: string; description: string }) => { + await addPrefixListEntry(await resolveDeps(options), options.cidr, options.description); + }); + shared(prefixes.command("remove-entry")).requiredOption("--cidr ").action(async (options: { clusterName: string; awsRegion: string; awsProfile?: string; cidr: string }) => { + await removePrefixListEntry(await resolveDeps(options), options.cidr); + }); + program.command("backup-update-global-settings") + .requiredOption("--cluster-name ") + .requiredOption("--aws-region ") + .option("--aws-profile ") + .option("--force") + .option("--module-set ", "Name of the ModuleSet. Default: default") + .action(async (options: { clusterName: string; awsRegion: string; awsProfile?: string; force?: boolean; moduleSet?: string }) => { + await backupUpdateGlobalSettings(await resolveDeps(options), options); + }); + return utils; +} + +/** Register all remaining operator command groups onto the command program. */ +export function registerRemainingOperatorCommands(program: Command, deps: RemainingOperatorCommandDeps): void { + registerSsoCommands(program, deps.sso); + registerDirectoryServiceCommands(program, deps.directoryService); + registerSharedStorageCommands(program, deps.sharedStorage); + registerUtilsCommands(program, deps.utils); + registerSupportCommands(program, deps.support); + registerIntegrationTestCommands(program, deps.integrationTests); +} diff --git a/source/idea/ideactl/src/cli/deployment-helper.ts b/source/idea/ideactl/src/cli/deployment-helper.ts new file mode 100644 index 00000000..16079d1f --- /dev/null +++ b/source/idea/ideactl/src/cli/deployment-helper.ts @@ -0,0 +1,277 @@ +/** + * Port of `app/deployment_helper.py`: which modules deploy, in what order, and how + * `--optimize-deployment` groups them. + * + * The ordering is the whole point. A module's stack reads settings that an earlier module's stack + * wrote, so `analytics` before `cluster-manager` is not a preference. Priorities come from the + * module metadata table in `config/cluster-config.ts`, so there is one copy of them. + */ + +import { ClusterConfig, GeneralException, MODULE_METADATA, type ModuleInfo } from '../config/cluster-config.ts'; +import { buildBootstrapContext } from './bootstrap-context.ts'; +import { CdkInvoker, type Deps } from './cdk-invoker.ts'; + +/** `deployment_helper.py:194`: the stagger between two modules of the same priority group. */ +export const OPTIMIZED_DEPLOYMENT_STAGGER_MS = 10_000; + +const MODULE_TYPE_CONFIG = 'config'; + +const PRIORITY_BY_MODULE_NAME = new Map(MODULE_METADATA.map((entry) => [entry.name, entry.deployment_priority])); + +/** + * `ModuleMetadataHelper.get_module_deployment_priority(module_name=...)`: the priority comes from + * the module NAME, never from the table row, so an unjoined `ModuleInfo` still orders correctly. + */ +export function deploymentPriority(moduleName: string): number { + const priority = PRIORITY_BY_MODULE_NAME.get(moduleName); + if (priority === undefined) throw new GeneralException(`module not found for name: ${moduleName}`); + return priority; +} + +export interface DeploymentHelperOptions { + clusterName: string; + awsRegion: string; + moduleSet: string; + awsProfile?: string; + terminationProtection?: boolean; + deploymentId?: string; + upgrade?: boolean; + allModules?: boolean; + forceBuildBootstrap?: boolean; + optimizeDeployment?: boolean; + rollback?: boolean; + moduleIds?: readonly string[]; + allowReplacement?: readonly string[]; + allowReplacementOfType?: ReadonlyMap; + staggerMs?: number; + deps: Deps; +} + +/** + * `get_deployment_order` and `get_optimized_deployment_order` as pure functions over the modules + * table, so both the CLI and its tests share one implementation. + */ +export function deploymentOrder( + modules: ModuleInfo[], + moduleIds: readonly string[], + upgrade: boolean, +): string[] { + const byId = new Map(modules.map((module) => [module.module_id, module])); + const selected: Array<{ moduleId: string; priority: number }> = []; + for (const moduleId of moduleIds) { + const module = byId.get(moduleId); + if (module === undefined) continue; + if (module.type === MODULE_TYPE_CONFIG) continue; + if (module.status === 'deployed' && !upgrade) continue; + selected.push({ moduleId, priority: deploymentPriority(module.name) }); + } + // Python's list.sort is stable, so equal priorities keep the modules-table order. + selected.sort((a, b) => a.priority - b.priority); + return selected.map((entry) => entry.moduleId); +} + +/** The same selection, grouped by priority, in first-seen priority order. */ +export function optimizedDeploymentOrder( + modules: ModuleInfo[], + moduleIds: readonly string[], + upgrade: boolean, +): string[][] { + const byId = new Map(modules.map((module) => [module.module_id, module])); + const groups = new Map(); + for (const moduleId of deploymentOrder(modules, moduleIds, upgrade)) { + const moduleName = byId.get(moduleId)?.name; + if (moduleName === undefined) continue; + const priority = deploymentPriority(moduleName); + const group = groups.get(priority); + if (group === undefined) groups.set(priority, [moduleId]); + else group.push(moduleId); + } + return [...groups.values()]; +} + +export class DeploymentHelper { + readonly clusterName: string; + readonly awsRegion: string; + readonly moduleSet: string; + readonly deploymentId: string; + readonly upgrade: boolean; + readonly allModules: boolean; + private readonly options: DeploymentHelperOptions; + private readonly deps: Deps; + private readonly staggerMs: number; + private config: ClusterConfig; + + private constructor(options: DeploymentHelperOptions, config: ClusterConfig, deploymentId: string) { + this.options = options; + // A caller can replace the provider for isolated tests. Normal deploys use the ported context. + // The default is layered over the caller's object, never copied onto a new one: a copy drops the + // prototype methods of a class-based Deps, and it hides a hook the caller replaces after this + // constructor runs. + this.deps = + options.deps.bootstrapContext === undefined + ? new Proxy(options.deps, { + get: (target, property, receiver) => + property === 'bootstrapContext' + ? buildBootstrapContext + : Reflect.get(target, property, receiver), + }) + : options.deps; + this.clusterName = options.clusterName; + this.awsRegion = options.awsRegion; + this.moduleSet = options.moduleSet; + this.deploymentId = deploymentId; + this.upgrade = options.upgrade === true; + this.allModules = options.allModules === true; + this.staggerMs = options.staggerMs ?? OPTIMIZED_DEPLOYMENT_STAGGER_MS; + this.config = config; + } + + static async open(options: DeploymentHelperOptions): Promise { + const config = await ClusterConfig.fromDynamoDb(options.clusterName, options.awsRegion, { + moduleSet: options.moduleSet, + scan: options.deps.scan, + }); + const deploymentId = + options.deploymentId !== undefined && options.deploymentId !== '' + ? options.deploymentId + : options.deps.uuid(); + return new DeploymentHelper(options, config, deploymentId); + } + + private moduleIds(): string[] { + if (this.allModules) return this.config.modules().map((module) => module.module_id); + return [...(this.options.moduleIds ?? [])]; + } + + getDeploymentOrder(): string[] { + return deploymentOrder(this.config.modules(), this.moduleIds(), this.upgrade); + } + + /** Module names resolved from the same selected deployment order used for invocation. */ + getDeploymentModuleNames(): string[] { + return this.getDeploymentOrder().flatMap((moduleId) => { + const name = this.config.moduleInfoById(moduleId)?.name; + return name === undefined ? [] : [name]; + }); + } + + getOptimizedDeploymentOrder(): string[][] { + return optimizedDeploymentOrder(this.config.modules(), this.moduleIds(), this.upgrade); + } + + private printNoOpMessage(): void { + if (this.upgrade) { + this.deps.out('could not find any modules to upgrade.'); + return; + } + const moduleIds = this.moduleIds(); + if (moduleIds.length === 1) { + this.deps.out( + `${moduleIds[0]} is already deployed. use the --upgrade flag to upgrade or re-deploy the module.`, + ); + } else { + this.deps.out( + `[${moduleIds.join(', ')}] are already deployed. use the --upgrade flag to re-deploy these modules.`, + ); + } + } + + async deployModule(moduleId: string): Promise { + const moduleInfo = this.config.moduleInfoById(moduleId); + if (moduleInfo === undefined) throw new GeneralException(`module not found for module_id: ${moduleId}`); + this.deps.out(`deploying module: ${moduleInfo.name}, module id: ${moduleId}`); + const invoker = await CdkInvoker.open({ + clusterName: this.clusterName, + awsRegion: this.awsRegion, + moduleId, + moduleSet: this.moduleSet, + awsProfile: this.options.awsProfile, + deploymentId: this.deploymentId, + terminationProtection: this.options.terminationProtection, + rollback: this.options.rollback, + allowReplacement: this.options.allowReplacement, + allowReplacementOfType: this.options.allowReplacementOfType, + deps: this.deps, + }); + await invoker.invoke({ forceBuildBootstrap: this.options.forceBuildBootstrap }); + } + + /** + * Re-read the modules table. A deployment can take more than an hour, so hourly STS credentials + * can expire mid-run: Python rebuilt the `ClusterConfigDB` and the boto session on + * `ExpiredTokenException`. The SDK's credential provider refreshes on the next call, so retrying + * the scan once is the whole fix; anything else is a real error. + */ + private async refreshModules(): Promise { + const read = (): Promise => + ClusterConfig.fromDynamoDb(this.clusterName, this.awsRegion, { + moduleSet: this.moduleSet, + scan: this.deps.scan, + }); + try { + this.config = await read(); + } catch (error) { + if ((error as { name?: string }).name !== 'ExpiredTokenException') throw error; + this.config = await read(); + } + } + + async invoke(): Promise { + if (this.options.optimizeDeployment === true && this.moduleIds().length > 1) { + const groups = this.getOptimizedDeploymentOrder(); + if (groups.length === 0) { + this.printNoOpMessage(); + return; + } + this.deps.out(`optimized deployment order: ${JSON.stringify(groups)}`); + + for (const group of groups) { + const failures = new Map(); + const running: Array> = []; + for (const moduleId of group) { + running.push( + this.deployModule(moduleId).catch((error: unknown) => { + failures.set(moduleId, error); + }), + ); + // process the next entry after 10 seconds. + await this.deps.sleep(this.staggerMs); + } + await Promise.all(running); + + // The status check below cannot see this: a module already deployed by the previous + // release still reads 'deployed' after a failed re-deploy. + if (failures.size > 0) { + const detail = [...failures.entries()] + .map(([moduleId, error]) => `${moduleId} (${errorMessage(error)})`) + .join(', '); + throw new GeneralException(`deployment failed. could not deploy module(s): ${detail}`); + } + + await this.refreshModules(); + for (const moduleId of group) { + const moduleInfo = this.config.moduleInfoById(moduleId); + if (moduleInfo?.status !== 'deployed') { + throw new GeneralException( + `Module ${moduleId} on ${this.clusterName} is not deployed after its stack run. See CloudFormation events for stack ${this.clusterName}-${moduleId}, then re-run ideactl deploy ${moduleId} --cluster-name ${this.clusterName} --aws-region ${this.awsRegion}.`, + ); + } + } + } + return; + } + + const order = this.getDeploymentOrder(); + if (order.length === 0) { + this.printNoOpMessage(); + return; + } + for (const moduleId of order) { + await this.deployModule(moduleId); + } + } +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/source/idea/ideactl/src/cli/installer-params.ts b/source/idea/ideactl/src/cli/installer-params.ts new file mode 100644 index 00000000..6b4f0831 --- /dev/null +++ b/source/idea/ideactl/src/cli/installer-params.ts @@ -0,0 +1,435 @@ +/** + * The installer parameter flow declared in `resources/input_params/install_params.yml`. + * + * This reads the source declaration at runtime, preserves its section order and conditions, and + * collects values through an injected prompt driver. Resource-dependent choices are supplied by + * an injected provider so the flow remains deterministic in tests and replayed environments. + */ + +import { readFileSync } from "node:fs"; + +import yaml from "js-yaml"; + +import { resourcePath } from "../config/values.ts"; +import type { InstallerChoice, InstallerPromptType, InstallerQuestion, PromptDriver } from "./prompts.ts"; + +/** A resolved account identity inserted after the AWS account section. */ +export interface InstallerIdentity { + accountId: string; + partition: string; + dnsSuffix: string; +} + +/** Details used for choice display and existing-resource validation. */ +export interface InstallerSubnet { + id: string; + availabilityZone: string; + isOutpost?: boolean; +} + +/** Supplies choices that Python obtains from the selected AWS account. */ +export interface InstallerChoiceProvider { + choices?(name: string, values: Readonly>): InstallerChoice[]; + subnets?(vpcId: string): InstallerSubnet[]; + vpcCidrs?(): string[]; + clusterNames?(): string[]; +} + +/** Inputs for one run of either installer module. */ +export interface InstallerRunOptions { + driver: PromptDriver; + identity(values: Readonly>): Promise; + existingResources?: boolean; + regenerate?: boolean; + answers?: Readonly>; + choices?: InstallerChoiceProvider; + inputParamsFile?: string; +} + +/** A validation failure that the interactive loop displays before asking again. */ +export class InstallerValidationError extends Error {} + +interface RawModule { + name: string; + sections: RawSection[]; +} + +interface RawSection { + name: string; + params: Array<{ name: string }>; +} + +interface RawChoice { + title?: string; + value?: string; + disabled?: boolean; +} + +interface RawCondition { + param?: string; + eq?: unknown; + contains?: unknown; + and?: RawCondition[]; +} + +interface RawParam { + name: string; + title?: string; + description?: string; + param_type?: string; + data_type?: string; + multiple?: boolean; + default?: unknown; + choices?: RawChoice[]; + help_text?: string; + validate?: { required?: boolean; regex?: string; min?: number; max?: number }; + when?: RawCondition; + custom?: { defaults?: Record }; +} + +interface RawSpec { + SocaInputParamSpec?: { + modules?: RawModule[]; + params?: RawParam[]; + }; +} + +const REQUIRED_MODULES: readonly InstallerChoice[] = [ + { title: "Global Settings (required)", value: "global-settings", disabled: true }, + { title: "Cluster (required)", value: "cluster", disabled: true }, + { title: "Analytics (required)", value: "analytics", disabled: true }, + { title: "Identity Provider (required)", value: "identity-provider", disabled: true }, + { title: "Directory Service (required)", value: "directoryservice", disabled: true }, + { title: "Shared Storage (required)", value: "shared-storage", disabled: true }, + { title: "Cluster Manager (required)", value: "cluster-manager", disabled: true }, +]; + +/** + * Every new cluster runs its control plane as container tasks, so the installer asks no question + * about it and writes the key the generator splices the container module in from. There is no + * second shape to choose: the per-module release archives a control-plane host downloads are no + * longer produced. + * + * The key belongs in the values file rather than in the generator's default because regenerating + * an existing cluster's configuration has to keep producing what that cluster already has. + */ +const CONTAINER_MODULE_VALUES_KEY = "enable_ecs"; + +/** Modules the container stack runs as tasks, which it names unconditionally. */ +const CONTAINER_REQUIRED_MODULES = ["scheduler", "virtual-desktop-controller"]; + +/** How many times one question may be re-asked before the validation failure is raised instead. */ +const MAX_PROMPT_ATTEMPTS = 50; + +const OPTIONAL_MODULES: readonly InstallerChoice[] = [ + { title: "Metrics and Monitoring", value: "metrics" }, + { title: "Scale-out Computing on AWS (SOCA) for HPC", value: "scheduler" }, + { title: "Enterprise Virtual Desktop Infrastructure (eVDI)", value: "virtual-desktop-controller" }, + { title: "Bastion Host", value: "bastion-host" }, +]; + +const METRICS_PROVIDERS: readonly InstallerChoice[] = [ + { title: "AWS CloudWatch", value: "cloudwatch" }, + { title: "Amazon Managed Service for Prometheus", value: "amazon_managed_prometheus" }, + { title: "Datadog agent (DogStatsD)", value: "dogstatsd" }, + { title: "Custom Prometheus Server", value: "prometheus" }, +]; + +/** Parses the declaration with runtime shape checks instead of trusting YAML data. */ +function loadSpec(file: string): { modules: RawModule[]; params: Map } { + const parsed = yaml.load(readFileSync(file, "utf-8")); + if (!isRecord(parsed) || !isRecord(parsed["SocaInputParamSpec"])) { + throw new InstallerValidationError(`invalid installer parameter declaration: ${file}`); + } + const root = parsed["SocaInputParamSpec"] as RawSpec["SocaInputParamSpec"]; + if (!Array.isArray(root?.modules) || !Array.isArray(root.params)) { + throw new InstallerValidationError(`invalid installer parameter declaration: ${file}`); + } + const modules = root.modules.filter(isModule); + const params = new Map(root.params.filter(isParam).map((param) => [param.name, param])); + if (modules.length !== root.modules.length || params.size !== root.params.length) { + throw new InstallerValidationError(`invalid installer parameter declaration: ${file}`); + } + return { modules, params }; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isModule(value: unknown): value is RawModule { + return isRecord(value) && typeof value.name === "string" && Array.isArray(value.sections) && + value.sections.every(isSection); +} + +function isSection(value: unknown): value is RawSection { + return isRecord(value) && typeof value.name === "string" && Array.isArray(value.params) && + value.params.every((param) => isRecord(param) && typeof param.name === "string"); +} + +function isParam(value: unknown): value is RawParam { + return isRecord(value) && typeof value.name === "string"; +} + +function isEmpty(value: unknown): boolean { + return value === undefined || value === null || + (typeof value === "string" && value.trim() === "") || + (Array.isArray(value) && value.length === 0); +} + +function yamlBoolean(value: unknown): boolean | undefined { + if (typeof value === "boolean") return value; + if (typeof value !== "string") return undefined; + if (["yes", "y", "true", "1", "on"].includes(value.toLowerCase())) return true; + if (["no", "n", "false", "0", "off"].includes(value.toLowerCase())) return false; + return undefined; +} + +function conditionMatches(condition: RawCondition | undefined, values: Readonly>): boolean { + if (condition === undefined) return true; + if (Array.isArray(condition.and)) return condition.and.every((item) => conditionMatches(item, values)); + if (typeof condition.param !== "string") return false; + const value = values[condition.param]; + if (condition.contains !== undefined) { + return Array.isArray(value) && value.some((item) => item === condition.contains); + } + if (condition.eq !== undefined) return value === condition.eq; + return false; +} + +function promptType(param: RawParam): InstallerPromptType { + if (param.param_type === "text" || param.param_type === "select" || + param.param_type === "checkbox" || param.param_type === "confirm") { + return param.param_type; + } + throw new InstallerValidationError(`unsupported prompt type for ${param.name}: ${String(param.param_type)}`); +} + +function declaredChoices(param: RawParam, options: InstallerRunOptions, values: Readonly>): InstallerChoice[] { + const dynamic = options.choices?.choices?.(param.name, values); + if (dynamic !== undefined) return dynamic; + if (param.name === "enabled_modules") return [...REQUIRED_MODULES, ...OPTIONAL_MODULES]; + if (param.name === "metrics_provider") return [...METRICS_PROVIDERS]; + return (param.choices ?? []).flatMap((choice) => + typeof choice.value === "string" + ? [{ title: choice.title ?? choice.value, value: choice.value, disabled: choice.disabled }] + : [], + ); +} + +function defaultValue(param: RawParam, choices: readonly InstallerChoice[], values: Readonly>): unknown { + if (param.name === "aws_region" && isRecord(param.custom?.defaults)) { + const partition = values["aws_partition"]; + if (typeof partition === "string") return param.custom.defaults[partition]; + } + if (param.default === "$first") return choices.find((choice) => choice.disabled !== true)?.value; + if (param.param_type === "confirm") return yamlBoolean(param.default) ?? param.default; + return param.default; +} + +function toBoolean(value: unknown, name: string): boolean { + if (typeof value === "boolean") return value; + if (typeof value === "string") { + const normalised = value.trim().toLowerCase(); + if (["yes", "y", "true", "1", "on"].includes(normalised)) return true; + if (["no", "n", "false", "0", "off"].includes(normalised)) return false; + } + throw new InstallerValidationError(`${name} must be a boolean`); +} + +function normaliseAnswer(value: unknown, param: RawParam): unknown { + const type = promptType(param); + if (type === "confirm") return toBoolean(value, param.name); + if (type === "checkbox") { + if (Array.isArray(value)) return value.map((item) => String(item).trim()).filter((item) => item !== ""); + if (typeof value === "string") return value.split(",").map((item) => item.trim()).filter((item) => item !== ""); + throw new InstallerValidationError(`${param.name} must be a list`); + } + if (param.data_type === "int") { + if (typeof value === "number" && Number.isInteger(value)) return value; + if (typeof value === "string" && /^[+-]?\d+$/.test(value.trim())) return Number.parseInt(value.trim(), 10); + throw new InstallerValidationError(`${param.name} must be an integer`); + } + if (typeof value === "string") return value.trim(); + if (value === undefined || value === null) return value; + return String(value); +} + +function validateDeclaredRules(value: unknown, param: RawParam, choices: readonly InstallerChoice[]): void { + const rules = param.validate; + if (yamlBoolean(rules?.required) === true && isEmpty(value)) { + throw new InstallerValidationError(`${param.title ?? param.name} is required`); + } + if (typeof rules?.regex === "string" && typeof value === "string" && !new RegExp(rules.regex).test(value)) { + throw new InstallerValidationError(`${param.title ?? param.name} is invalid`); + } + if (typeof value === "number") { + if (rules?.min !== undefined && value < rules.min) throw new InstallerValidationError(`${param.title ?? param.name} must be at least ${rules.min}`); + if (rules?.max !== undefined && value > rules.max) throw new InstallerValidationError(`${param.title ?? param.name} must be at most ${rules.max}`); + } + if ((param.param_type === "select" || param.param_type === "checkbox") && choices.length > 0 && !isEmpty(value)) { + const selected = Array.isArray(value) ? value : [value]; + for (const item of selected) { + const choice = choices.find((entry) => entry.value === item); + if (choice === undefined || choice.disabled === true) throw new InstallerValidationError(`${param.title ?? param.name} has an invalid selection`); + } + } +} + +function validateCidr(value: string): void { + for (const token of value.split(",")) { + const cidr = token.trim(); + const match = /^(\d{1,3}(?:\.\d{1,3}){3})(?:\/(\d{1,2}))?$/.exec(cidr); + if (match === null) throw new InstallerValidationError(`CIDR Value: ${cidr} is invalid. Please enter a valid CIDR block`); + const octets = match[1].split(".").map((part) => Number.parseInt(part, 10)); + const prefix = match[2] === undefined ? 32 : Number.parseInt(match[2], 10); + if (octets.some((octet) => octet > 255) || prefix > 32) { + throw new InstallerValidationError(`CIDR Value: ${cidr} is invalid. Please enter a valid CIDR block`); + } + const address = (((octets[0] ?? 0) << 24) >>> 0) + ((octets[1] ?? 0) << 16) + ((octets[2] ?? 0) << 8) + (octets[3] ?? 0); + const mask = prefix === 0 ? 0 : (0xffffffff << (32 - prefix)) >>> 0; + if (((address & mask) >>> 0) !== address) throw new InstallerValidationError(`CIDR Value: ${cidr} is invalid. Please enter a valid CIDR block`); + } +} + +function filterValue(value: unknown, param: RawParam, values: Record): unknown { + if (param.name === "cluster_name") { + const token = String(value ?? "").trim().toLowerCase().replace(/^idea-/, ""); + return token === "" ? "" : `idea-${token}`; + } + if (param.name === "client_ip") { + return String(value).split(",").map((entry) => entry.trim()).filter((entry) => entry !== "") + .map((entry) => entry.includes("/") ? entry : `${entry}/32`); + } + if (param.name === "prefix_list_ids") { + return String(value).split(",").map((entry) => entry.trim()).filter((entry) => entry !== ""); + } + if (param.name === "vpc_id") values["use_existing_vpc"] = true; + if (param.name === "existing_apps_fs_id") values["use_existing_apps_fs"] = true; + if (param.name === "existing_data_fs_id") values["use_existing_data_fs"] = true; + if (param.name === "opensearch_domain_endpoint") values["use_existing_opensearch_cluster"] = true; + if (param.name === "directory_id") values["use_existing_directory_service"] = true; + return value; +} + +function validateCustomRules( + rawValue: unknown, + value: unknown, + param: RawParam, + values: Readonly>, + options: InstallerRunOptions, +): void { + if (param.name === "cluster_name" && typeof value === "string") { + if (value === "idea-other") throw new InstallerValidationError(`Invalid ClusterName: ${value}. "other" is a reserved keyword and cannot be used in ClusterName.`); + if (value.replace(/^idea-/, "").includes("idea")) throw new InstallerValidationError(`Invalid ClusterName: ${value}. "${value}" contains value "idea" and is not allowed.`); + if (value.length < 8 || value.length > 11) throw new InstallerValidationError(`ClusterName: (${value}) length must be between 8 and 11 characters. Current: ${value.length}`); + if (options.regenerate !== true && options.choices?.clusterNames?.().includes(value)) { + throw new InstallerValidationError(`Cluster: (${value}) already exists and is in use.`); + } + } + if (param.name === "client_ip" && typeof rawValue === "string") validateCidr(rawValue); + if (param.name === "vpc_cidr_block" && typeof value === "string" && options.regenerate !== true && + options.choices?.vpcCidrs?.().includes(value)) { + throw new InstallerValidationError(`VPC CIDR Block: ${value} is already used by an existing VPC. Please enter a different CIDR block to avoid IP Address conflicts between VPCs.`); + } + if (param.name === "enabled_modules" && Array.isArray(value)) { + // The container stack builds one service per control-plane role and reads each role's module + // id, so a selection without these fails at synthesis with a module lookup rather than with + // anything an operator can act on. + const missing = CONTAINER_REQUIRED_MODULES.filter((module) => !value.includes(module)); + if (missing.length > 0) { + throw new InstallerValidationError( + `The container control plane runs these modules as tasks, so they are part of every new cluster. Missing: ${missing.join(", ")}`, + ); + } + } + if (param.name === "existing_resources" && Array.isArray(value) && + !value.includes("subnets:public") && !value.includes("subnets:private")) { + throw new InstallerValidationError("Either one of [Subnets: Public, Subnets: Private] is required"); + } + if ((param.name === "private_subnet_ids" || param.name === "public_subnet_ids") && Array.isArray(value)) { + const vpcId = values["vpc_id"]; + const subnets = typeof vpcId === "string" ? options.choices?.subnets?.(vpcId) : undefined; + if (subnets !== undefined) { + const selected = subnets.filter((subnet) => value.includes(subnet.id)); + const isOutpost = selected.some((subnet) => subnet.isOutpost === true); + const zones = new Set(); + for (const subnet of selected) { + if (!isOutpost && zones.has(subnet.availabilityZone)) throw new InstallerValidationError("Multiple subnet selection from the same Availability Zone is not supported unless using AWS Outposts."); + zones.add(subnet.availabilityZone); + } + const otherName = param.name === "private_subnet_ids" ? "public_subnet_ids" : "private_subnet_ids"; + const other = values[otherName]; + if (Array.isArray(other) && selected.some((subnet) => other.includes(subnet.id))) { + throw new InstallerValidationError(`SubnetId is already selected as part of ${otherName.replace(/_/g, " ")} selection.`); + } + if (param.name === "private_subnet_ids" && selected.length < 2 && !isOutpost) { + throw new InstallerValidationError("Minimum 2 subnet selections are required to ensure high availability."); + } + } + } +} + +/** Runs the selected YAML module and returns the values map that Python writes as `values.yml`. */ +export async function collectInstallerValues(options: InstallerRunOptions): Promise> { + const declaration = loadSpec(options.inputParamsFile ?? resourcePath("input_params/install_params.yml")); + const moduleName = options.existingResources === true ? "install-idea-using-existing-resources" : "install-idea"; + const module = declaration.modules.find((entry) => entry.name === moduleName); + if (module === undefined) throw new InstallerValidationError(`installer module not found: ${moduleName}`); + + const providedAnswers = options.answers ?? {}; + const values: Record = { + _regenerate: options.regenerate === true, + [CONTAINER_MODULE_VALUES_KEY]: true, + }; + for (const section of module.sections) { + for (const reference of section.params) { + const name = reference.name; + const param = declaration.params.get(name); + if (param === undefined) throw new InstallerValidationError(`installer parameter not found: ${name}`); + if (!conditionMatches(param.when, values)) continue; + + const choices = declaredChoices(param, options, values); + const fallback = defaultValue(param, choices, values); + const question: InstallerQuestion = { + name, + title: param.title ?? name, + description: param.description ?? "", + promptType: promptType(param), + multiple: param.multiple === true, + defaultValue: fallback, + choices, + helpText: param.help_text === null ? undefined : param.help_text, + }; + + for (let attempt = 1; ; attempt += 1) { + try { + const answered = Object.hasOwn(providedAnswers, name) ? providedAnswers[name] : await options.driver.ask(question); + const normalised = normaliseAnswer(answered === undefined ? fallback : answered, param); + validateDeclaredRules(normalised, param, choices); + const validationValue = param.name === "cluster_name" + ? filterValue(normalised, param, {}) + : normalised; + validateCustomRules(answered === undefined ? fallback : answered, validationValue, param, values, options); + values[name] = filterValue(normalised, param, values); + break; + } catch (error) { + if (Object.hasOwn(providedAnswers, name)) throw error; + // Re-asking is for a person who can correct the answer. A driver that cannot be + // corrected returns the same answer forever, and an unbounded retry turns that into a + // hang with no output rather than the validation message. The cap is far above what + // anyone types at one question. + if (attempt >= MAX_PROMPT_ATTEMPTS) throw error; + options.driver.report(error instanceof Error ? error.message : String(error)); + } + } + } + if (section.name === "aws-account") { + const identity = await options.identity(values); + values["aws_partition"] = identity.partition; + values["aws_account_id"] = identity.accountId; + values["aws_dns_suffix"] = identity.dnsSuffix; + } + } + return values; +} diff --git a/source/idea/ideactl/src/cli/live-migrate-adapters.ts b/source/idea/ideactl/src/cli/live-migrate-adapters.ts new file mode 100644 index 00000000..796a0c53 --- /dev/null +++ b/source/idea/ideactl/src/cli/live-migrate-adapters.ts @@ -0,0 +1,1504 @@ +/** + * Live dependencies for the one-phase migration command. + * + * Cluster reads and writes reach the account through the shared command + * dependencies and the upgrade command's live adapters, so the migration uses + * one credential path with every other command. This file adds only what the + * migration needs and no other command has: the durable operation record store, + * the deployed-template and routing reads its before-state capture requires, and + * the table that says which migration steps this release can execute. + */ + +import { createHash } from "node:crypto"; + +import { ClusterConfig, GeneralException, type ModuleInfo } from "../config/cluster-config.ts"; +import { compareUpgradeDrift, type UpgradeDriftInput } from "../config/upgrade-drift.ts"; +import type { + UpgradeStateObjectApi, + VersionedUpgradeStateObject, +} from "../config/upgrade-state.ts"; +import { ideaVersion } from "../version.ts"; +import { awsClientOptions } from "./aws-client-options.ts"; +import type { Deps } from "./cdk-invoker.ts"; +import { + MIGRATION_STEPS, + MigrationRefusedError, + type ExecutableMigrationStepId, + type MigrateDeps, + type MigrationContext, + type MigrationObservation, + type MigrationReconciliation, + type MigrationStepExecutor, + type MigrationStepId, + type SchedulerClosureObservation, +} from "./commands/migrate.ts"; +import { createLiveUpgradeDeps, prepareUpgradeDriftInput } from "./commands/upgrade.ts"; +import { + SchedulerStateUnreadableError, + liveSsmReadChannel, + readBatchServerState, + renderBatchServerState, + type BatchServerState, +} from "./scheduler-state-read.ts"; +import { + PreflightRegistry, + createAwsvpcTrunkingCheck, + createConfigurationDriftCheck, + createTemplateComparisonCheck, + renderPreflightReport, + runPreflight, + type PreflightContext, +} from "./preflight.ts"; + +// --------------------------------------------------------------------------------------------- +// what this release can execute +// --------------------------------------------------------------------------------------------- + +/** An adapter or artifact a migration step needs and this release does not carry. */ +export interface MigrationCapability { + id: string; + /** What the missing piece would do, in operator terms. */ + description: string; + /** Where the change is written down for its owner. */ + request: string; +} + +export const MIGRATION_CAPABILITIES: readonly MigrationCapability[] = [ + { + id: "scheduler-state-read-container", + description: + "a read-only report of the batch server's own state once it runs as a task: job inventory, the scheduling flag and per-queue enablement. Before the cutover the same facts are read from the batch server host over the systems manager channel; after it, nothing reaches the server", + request: "docs/port/requests/migrate-live.md section 1", + }, + { + id: "module-deployment", + description: + "a deployment hook on the executor, so a boundary can deploy one module in a fixed order and read its stack result back", + request: "docs/port/requests/migrate-live.md section 6", + }, + { + id: "job-canary", + description: + "one isolated operator job, submitted and observed while user queues stay disabled, to prove a scheduler task replacement", + request: "docs/port/requests/migrate-live.md section 7", + }, + { + id: "scheduler-state", + description: + "stop the legacy scheduler, capture its spool directory, verify the archive by reading it back, and restore it into the shared scheduler directory", + request: "docs/port/requests/migrate-live.md section 2", + }, + { + id: "container-configuration", + description: + "generated container module configuration on the production path: the module row, the settings template and the module-set entry", + request: "docs/port/requests/migrate-live.md section 4", + }, + { + id: "software-stack-plan", + description: + "the end-of-life desktop software-stack plan, which the upgrade command builds for its own run and does not export", + request: "docs/port/requests/migrate-live.md section 5", + }, +]; + +const CAPABILITY_BY_ID = new Map(MIGRATION_CAPABILITIES.map((capability) => [capability.id, capability])); + +/** + * Capabilities each step needs beyond the shared adapters. + * + * A step is listed against every capability its stated precondition or action + * requires, including the ones that only verify a result. A step with an empty + * list is executed and verified by this release. + */ +export const MIGRATION_STEP_CAPABILITIES: Readonly> = { + PREFLIGHT_PASSED: [], + OPERATION_STARTED: [], + ADMISSION_CLOSED: [], + WORKLOAD_DRAINED: [], + LEGACY_SCHEDULER_CAPTURED: ["scheduler-state"], + SOFTWARE_STACKS_RECONCILED: ["software-stack-plan"], + CONFIGURATION_STAGED: ["container-configuration"], + PROVIDERS_COMMITTED: ["module-deployment"], + SCHEDULER_DNS_RETAINED: ["module-deployment"], + ECS_CONFIGURATION_ACTIVE: ["container-configuration"], + ECS_STAGED: ["container-configuration", "module-deployment"], + PBS_STATE_SEEDED: ["scheduler-state"], + ECS_SCHEDULER_READY: ["scheduler-state", "scheduler-state-read-container"], + CLUSTER_MANAGER_ROUTED: ["module-deployment"], + CLUSTER_MANAGER_LEGACY_REMOVED: ["module-deployment"], + VDC_ROUTED: ["module-deployment"], + VDC_LEGACY_REMOVED: ["module-deployment"], + SCHEDULER_ROUTED: ["module-deployment"], + SCHEDULER_LEGACY_REMOVED: ["module-deployment"], + TARGET_PROVED: ["module-deployment", "job-canary"], + ADMISSION_REOPENED: ["scheduler-state-read-container"], + OPERATION_COMPLETED: ["scheduler-state-read-container"], +}; + +/** + * Steps that do the part they own and then refuse on what they cannot finish. + * + * They are blocked, so they appear in the table above, but calling them is not a + * no-op: each writes the configuration row its boundary is defined by, or, for + * the admission boundary, announces maintenance and verifies what is observable. + */ +export const ACTING_MIGRATION_STEPS: ReadonlySet = new Set([ + "CONFIGURATION_STAGED", + "SCHEDULER_DNS_RETAINED", + "ECS_CONFIGURATION_ACTIVE", + "CLUSTER_MANAGER_ROUTED", + "CLUSTER_MANAGER_LEGACY_REMOVED", + "VDC_ROUTED", + "VDC_LEGACY_REMOVED", + "SCHEDULER_ROUTED", + "SCHEDULER_LEGACY_REMOVED", +]); + +/** Capabilities one step needs, resolved to their descriptions. */ +export function stepCapabilityGaps(step: MigrationStepId): MigrationCapability[] { + return (MIGRATION_STEP_CAPABILITIES[step] ?? []).map((id) => { + const capability = CAPABILITY_BY_ID.get(id); + if (capability === undefined) throw new TypeError(`Unknown migration capability: ${id}`); + return capability; + }); +} + +/** Every step in the fixed order, with the capabilities it still needs. */ +function stepsInOrder(): Array<{ step: MigrationStepId; capabilities: readonly string[] }> { + return MIGRATION_STEPS.map((step) => ({ + step: step.id, + capabilities: MIGRATION_STEP_CAPABILITIES[step.id] ?? [], + })); +} + +/** The first step of the fixed order this release cannot execute. */ +export function firstUnsupportedStep(): MigrationStepId | undefined { + return stepsInOrder().find((entry) => entry.capabilities.length > 0)?.step; +} + +/** Where the run will stop, and every missing capability with the steps waiting on it. */ +export function capabilityReport(): string | undefined { + const all = stepsInOrder(); + const blocked = all.filter((entry) => entry.capabilities.length > 0); + if (blocked.length === 0) return undefined; + const acting = blocked.filter((entry) => ACTING_MIGRATION_STEPS.has(entry.step)).length; + const lines = [ + `STOPS AT ${firstUnsupportedStep() ?? "an unknown step"}: ${all.length - blocked.length} of ${all.length} boundaries complete on their own and ${acting} more do the part they own before refusing. Each remaining step names what it waits on, so closing one does not wait on the others.`, + ]; + for (const capability of MIGRATION_CAPABILITIES) { + const steps = blocked + .filter((entry) => entry.capabilities.includes(capability.id)) + .map((entry) => entry.step); + if (steps.length === 0) continue; + lines.push( + ` ${capability.id}: ${capability.description}. Needed by ${steps.length} step(s): ${steps.join(", ")}. Change recorded in ${capability.request}.`, + ); + } + return lines.join("\n"); +} + +// --------------------------------------------------------------------------------------------- +// durable record store +// --------------------------------------------------------------------------------------------- + +/** The two response shapes the record store reads. */ +interface StateGetResult { + Body?: { transformToString(): Promise }; + ETag?: string; +} + +interface StatePutResult { + ETag?: string; +} + +/** + * The S3 client surface the record store uses, as a structural interface so + * tests drive the conditional-write branches without a network client. + */ +export interface MigrationStateStoreClient { + send(command: unknown): Promise; +} + +/** True for the status codes S3 returns when a write condition does not hold. */ +export function isWriteConditionFailure(error: unknown): boolean { + const name = (error as { name?: string })?.name ?? ""; + const status = (error as { $metadata?: { httpStatusCode?: number } })?.$metadata?.httpStatusCode; + return ( + name === "PreconditionFailed" || + name === "ConditionalRequestConflict" || + status === 409 || + status === 412 + ); +} + +/** True when the record has never been written. */ +export function isMissingObject(error: unknown): boolean { + const name = (error as { name?: string })?.name ?? ""; + const status = (error as { $metadata?: { httpStatusCode?: number } })?.$metadata?.httpStatusCode; + return name === "NoSuchKey" || name === "NotFound" || status === 404; +} + +/** + * Conditional object store for the operation record. + * + * The revision is the object ETag: `IfNoneMatch` claims a record that does not + * exist and `IfMatch` advances the one the caller read, so two runs cannot + * advance the same record. + */ +export function createMigrationStateObjects(input: { + client(): Promise; +}): UpgradeStateObjectApi { + return { + async getObject(request): Promise { + const { GetObjectCommand } = await import("@aws-sdk/client-s3"); + const client = await input.client(); + try { + const result = (await client.send( + new GetObjectCommand({ Bucket: request.bucket, Key: request.key }), + )) as StateGetResult; + const body = await result.Body?.transformToString(); + if (body === undefined || result.ETag === undefined) { + throw new GeneralException( + `The operation record at s3://${request.bucket}/${request.key} returned no body or no revision. Retry, and check the bucket's versioning and replication settings.`, + ); + } + return { body, revision: result.ETag }; + } catch (error) { + if (isMissingObject(error)) return undefined; + throw error; + } + }, + async putObject(request): Promise<{ revision: string } | undefined> { + const { PutObjectCommand } = await import("@aws-sdk/client-s3"); + const client = await input.client(); + const condition = request.condition.kind === "absent" + ? { IfNoneMatch: "*" } + : { IfMatch: request.condition.revision }; + try { + const result = (await client.send( + new PutObjectCommand({ + Bucket: request.bucket, + Key: request.key, + Body: request.body, + ContentType: "application/json", + ...condition, + }), + )) as StatePutResult; + if (result.ETag === undefined) { + throw new GeneralException( + `The write to s3://${request.bucket}/${request.key} returned no revision, so the next conditional write cannot be built. Retry the command.`, + ); + } + return { revision: result.ETag }; + } catch (error) { + if (isWriteConditionFailure(error)) return undefined; + throw error; + } + }, + }; +} + +/** The live record store, bound to one region and profile. */ +export function liveMigrationStateObjects( + awsRegion: () => string, + awsProfile: () => string | undefined, +): UpgradeStateObjectApi { + return createMigrationStateObjects({ + async client() { + const { S3Client } = await import("@aws-sdk/client-s3"); + return new S3Client(await awsClientOptions(awsRegion(), awsProfile())); + }, + }); +} + +// --------------------------------------------------------------------------------------------- +// account reads the migration adds +// --------------------------------------------------------------------------------------------- + +/** Where a read happens. Every migration read names its own region and profile. */ +export interface MigrationReadTarget { + awsRegion: string; + awsProfile?: string; +} + +export interface MigrationStackSummary { + status: string; + lastUpdated?: string; + parameters: Record; +} + +export interface MigrationInstanceRecord { + instanceId: string; + state: string; + moduleId?: string; + nodeType?: string; +} + +export interface MigrationListenerRecord { + loadBalancerArn: string; + listenerArn: string; + port?: number; + /** Target group ARNs the default action forwards to. */ + defaultTargetGroupArns: string[]; + rules: Array<{ ruleArn: string; priority?: string; targetGroupArns: string[] }>; +} + +export interface MigrationTargetGroupRecord { + targetGroupArn: string; + targetGroupName?: string; + healthy: number; + unhealthy: number; +} + +export interface MigrationRecordSetRecord { + name: string; + type: string; + values: string[]; +} + +export interface MigrationObjectRecord { + key: string; + etag?: string; + size?: number; +} + +/** Account reads the shared dependencies do not already carry. */ +export interface MigrationAccountReads { + stackTemplate(input: MigrationReadTarget & { stackName: string }): Promise; + stackSummary(input: MigrationReadTarget & { stackName: string }): Promise; + clusterInstances(input: MigrationReadTarget & { clusterName: string }): Promise; + clusterListeners(input: MigrationReadTarget & { clusterName: string }): Promise; + targetGroupHealth(input: MigrationReadTarget & { targetGroupArns: readonly string[] }): Promise; + recordSets(input: MigrationReadTarget & { hostedZoneId: string }): Promise; + bucketObjects(input: MigrationReadTarget & { bucket: string; prefix: string }): Promise; +} + +function tagValue(tags: Array<{ Key?: string; Value?: string }> | undefined, key: string): string | undefined { + return tags?.find((tag) => tag.Key === key)?.Value; +} + +/** The live implementation: one lazily imported client per service, bound to the caller's profile. */ +export function liveMigrationAccountReads(): MigrationAccountReads { + const cloudFormation = async (input: MigrationReadTarget) => { + const { CloudFormationClient } = await import("@aws-sdk/client-cloudformation"); + return new CloudFormationClient(await awsClientOptions(input.awsRegion, input.awsProfile)); + }; + const elbv2 = async (input: MigrationReadTarget) => { + const { ElasticLoadBalancingV2Client } = await import("@aws-sdk/client-elastic-load-balancing-v2"); + return new ElasticLoadBalancingV2Client(await awsClientOptions(input.awsRegion, input.awsProfile)); + }; + + return { + async stackTemplate(input) { + const { GetTemplateCommand } = await import("@aws-sdk/client-cloudformation"); + const client = await cloudFormation(input); + const result = await client.send( + new GetTemplateCommand({ StackName: input.stackName, TemplateStage: "Original" }), + ); + if (result.TemplateBody === undefined) { + throw new GeneralException(`stack ${input.stackName} returned no template body`); + } + return result.TemplateBody; + }, + async stackSummary(input) { + const { DescribeStacksCommand } = await import("@aws-sdk/client-cloudformation"); + const client = await cloudFormation(input); + const result = await client.send(new DescribeStacksCommand({ StackName: input.stackName })); + const stack = result.Stacks?.[0]; + if (stack === undefined) throw new GeneralException(`stack not found: ${input.stackName}`); + const parameters: Record = {}; + for (const parameter of stack.Parameters ?? []) { + if (parameter.ParameterKey !== undefined) { + parameters[parameter.ParameterKey] = parameter.ParameterValue ?? ""; + } + } + return { + status: stack.StackStatus ?? "UNKNOWN", + lastUpdated: (stack.LastUpdatedTime ?? stack.CreationTime)?.toISOString(), + parameters, + }; + }, + async clusterInstances(input) { + const { DescribeInstancesCommand, EC2Client } = await import("@aws-sdk/client-ec2"); + const client = new EC2Client(await awsClientOptions(input.awsRegion, input.awsProfile)); + const records: MigrationInstanceRecord[] = []; + let token: string | undefined; + do { + const result = await client.send( + new DescribeInstancesCommand({ + Filters: [ + { Name: "tag:idea:ClusterName", Values: [input.clusterName] }, + { Name: "instance-state-name", Values: ["pending", "running", "stopping", "stopped"] }, + ], + NextToken: token, + }), + ); + for (const reservation of result.Reservations ?? []) { + for (const instance of reservation.Instances ?? []) { + if (instance.InstanceId === undefined) continue; + records.push({ + instanceId: instance.InstanceId, + state: instance.State?.Name ?? "unknown", + moduleId: tagValue(instance.Tags, "idea:ModuleId"), + nodeType: tagValue(instance.Tags, "idea:NodeType"), + }); + } + } + token = result.NextToken; + } while (token !== undefined); + return records.sort((left, right) => left.instanceId.localeCompare(right.instanceId)); + }, + async clusterListeners(input) { + const { DescribeListenersCommand, DescribeLoadBalancersCommand, DescribeRulesCommand } = + await import("@aws-sdk/client-elastic-load-balancing-v2"); + const client = await elbv2(input); + const balancers = await client.send(new DescribeLoadBalancersCommand({})); + const owned = (balancers.LoadBalancers ?? []).filter((balancer) => + (balancer.LoadBalancerName ?? "").startsWith(`${input.clusterName}-`), + ); + const records: MigrationListenerRecord[] = []; + for (const balancer of owned) { + if (balancer.LoadBalancerArn === undefined) continue; + const listeners = await client.send( + new DescribeListenersCommand({ LoadBalancerArn: balancer.LoadBalancerArn }), + ); + for (const listener of listeners.Listeners ?? []) { + if (listener.ListenerArn === undefined) continue; + const rules = balancer.Type === "network" + ? { Rules: [] } + : await client.send(new DescribeRulesCommand({ ListenerArn: listener.ListenerArn })); + records.push({ + loadBalancerArn: balancer.LoadBalancerArn, + listenerArn: listener.ListenerArn, + port: listener.Port, + defaultTargetGroupArns: (listener.DefaultActions ?? []).flatMap((action) => [ + ...(action.TargetGroupArn === undefined ? [] : [action.TargetGroupArn]), + ...(action.ForwardConfig?.TargetGroups ?? []).flatMap((target) => + target.TargetGroupArn === undefined ? [] : [target.TargetGroupArn], + ), + ]), + rules: (rules.Rules ?? []).flatMap((rule) => + rule.RuleArn === undefined ? [] : [{ + ruleArn: rule.RuleArn, + priority: rule.Priority, + targetGroupArns: (rule.Actions ?? []).flatMap((action) => [ + ...(action.TargetGroupArn === undefined ? [] : [action.TargetGroupArn]), + ...(action.ForwardConfig?.TargetGroups ?? []).flatMap((target) => + target.TargetGroupArn === undefined ? [] : [target.TargetGroupArn], + ), + ]), + }], + ), + }); + } + } + return records; + }, + async targetGroupHealth(input) { + const { DescribeTargetGroupsCommand, DescribeTargetHealthCommand } = + await import("@aws-sdk/client-elastic-load-balancing-v2"); + const client = await elbv2(input); + const records: MigrationTargetGroupRecord[] = []; + for (const targetGroupArn of input.targetGroupArns) { + const [group, health] = await Promise.all([ + client.send(new DescribeTargetGroupsCommand({ TargetGroupArns: [targetGroupArn] })), + client.send(new DescribeTargetHealthCommand({ TargetGroupArn: targetGroupArn })), + ]); + const states = (health.TargetHealthDescriptions ?? []).map( + (description) => description.TargetHealth?.State ?? "unknown", + ); + records.push({ + targetGroupArn, + targetGroupName: group.TargetGroups?.[0]?.TargetGroupName, + healthy: states.filter((state) => state === "healthy").length, + unhealthy: states.filter((state) => state !== "healthy").length, + }); + } + return records; + }, + async recordSets(input) { + const { ListResourceRecordSetsCommand, Route53Client } = await import("@aws-sdk/client-route-53"); + const client = new Route53Client(await awsClientOptions(input.awsRegion, input.awsProfile)); + const records: MigrationRecordSetRecord[] = []; + let startName: string | undefined; + let startType: string | undefined; + for (;;) { + const result = await client.send( + new ListResourceRecordSetsCommand({ + HostedZoneId: input.hostedZoneId, + StartRecordName: startName, + StartRecordType: startType as never, + }), + ); + for (const record of result.ResourceRecordSets ?? []) { + records.push({ + name: record.Name ?? "", + type: record.Type?.toString() ?? "", + values: [ + ...(record.ResourceRecords ?? []).flatMap((value) => + value.Value === undefined ? [] : [value.Value], + ), + ...(record.AliasTarget?.DNSName === undefined ? [] : [record.AliasTarget.DNSName]), + ], + }); + } + if (result.IsTruncated !== true) return records; + startName = result.NextRecordName; + startType = result.NextRecordType?.toString(); + } + }, + async bucketObjects(input) { + const { ListObjectsV2Command, S3Client } = await import("@aws-sdk/client-s3"); + const client = new S3Client(await awsClientOptions(input.awsRegion, input.awsProfile)); + const records: MigrationObjectRecord[] = []; + let token: string | undefined; + do { + const result = await client.send( + new ListObjectsV2Command({ Bucket: input.bucket, Prefix: input.prefix, ContinuationToken: token }), + ); + for (const object of result.Contents ?? []) { + if (object.Key === undefined) continue; + records.push({ key: object.Key, etag: object.ETag, size: object.Size }); + } + token = result.IsTruncated === true ? result.NextContinuationToken : undefined; + } while (token !== undefined); + return records; + }, + }; +} + +// --------------------------------------------------------------------------------------------- +// fingerprints +// --------------------------------------------------------------------------------------------- + +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +/** Order-independent digest of one template, so formatting cannot mask a change. */ +export function templateDigest(templateBody: string): string { + let parsed: unknown; + try { + parsed = JSON.parse(templateBody); + } catch { + return sha256(templateBody); + } + return sha256(canonicalJson(parsed)); +} + +/** JSON with object keys in sorted order. */ +export function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object" && value !== null) { + const entries = Object.entries(value as Record) + .filter(([, member]) => member !== undefined) + .sort(([left], [right]) => left.localeCompare(right)); + return `{${entries.map(([key, member]) => `${JSON.stringify(key)}:${canonicalJson(member)}`).join(",")}}`; + } + return JSON.stringify(value) ?? "null"; +} + +/** + * The value an operator accepts with `--accept-template-comparison`. + * + * It binds the acceptance to this release and to the exact templates deployed + * now, so a comparison that was green against an older template cannot be + * replayed against a cluster that has changed since. + */ +export function templateComparisonFingerprint( + release: string, + stacks: ReadonlyArray<{ stackName: string; digest: string }>, +): string { + const lines = [...stacks] + .map((stack) => `${stack.stackName}=${stack.digest}`) + .sort((left, right) => left.localeCompare(right)); + return sha256([`release=${release}`, ...lines].join("\n")); +} + +/** The value an operator accepts with `--accept-drift`. */ +export function driftReportFingerprint(release: string, lossKeys: readonly string[]): string { + return sha256([`release=${release}`, ...[...lossKeys].sort((left, right) => left.localeCompare(right))].join("\n")); +} + +/** Module rows that own a deployed stack, in deployment order. */ +export function deployableStacks(modules: readonly ModuleInfo[], clusterName: string): Array<{ moduleId: string; stackName: string }> { + return modules + .filter((module) => module.type === "stack" || module.type === "app") + .flatMap((module) => { + const stackName = typeof module.stack_name === "string" && module.stack_name.startsWith(`${clusterName}-`) + ? module.stack_name + : undefined; + return stackName === undefined ? [] : [{ moduleId: module.module_id, stackName }]; + }) + .sort((left, right) => left.stackName.localeCompare(right.stackName)); +} + +// --------------------------------------------------------------------------------------------- +// the live step executor +// --------------------------------------------------------------------------------------------- + +/** Everything the executor reads or writes, injected so tests drive every branch. */ +export interface LiveMigrationStepsInput { + deps: Deps; + reads: MigrationAccountReads; + /** Live drift preview input, as the upgrade command prepares it. */ + driftInput(context: Readonly): Promise; + /** Effective account setting for task network interface trunking. */ + trunkingEnabled(context: Readonly): Promise; + clusterConfig(context: Readonly): Promise; + /** + * The batch server's own state, read-only, or a throw. + * + * It never answers with a default, because the values it would default to are + * the ones that let the admission boundary pass. + */ + schedulerState(context: Readonly): Promise; + release?(): string; +} + +/** The banner text the portal shows and the scheduler returns while admission is closed. */ +const MAINTENANCE_MESSAGE = "This cluster is undergoing maintenance. Job submission is closed."; + +function blockedObservation(step: MigrationStepId): MigrationObservation { + const gaps = stepCapabilityGaps(step); + return { + ok: false, + detail: [ + `${step} cannot be executed or verified by this release.`, + ...gaps.map((gap) => ` ${gap.id}: ${gap.description}. Change recorded in ${gap.request}.`), + ].join("\n"), + }; +} + +/** + * The production step executor. + * + * Steps this release can execute do their work through the shared adapters and + * are verified from a read after the write. Every other step refuses at its + * precondition, which is checked before the driver writes a started marker, so + * a run that cannot finish changes nothing. + */ +export class LiveMigrationSteps implements MigrationStepExecutor { + readonly #input: LiveMigrationStepsInput; + + constructor(input: LiveMigrationStepsInput) { + this.#input = input; + } + + async checkPrecondition( + step: MigrationStepId, + context: Readonly, + ): Promise { + if (step === "PREFLIGHT_PASSED") return this.#preflight(context); + if (step === "ADMISSION_CLOSED") return this.#admissionPathsReachable(context); + if (step === "WORKLOAD_DRAINED") return this.#drainPrecondition(context); + if (step === "CONFIGURATION_STAGED") return this.#stagingPrecondition(context); + if (step === "SCHEDULER_DNS_RETAINED") return this.#schedulerStackStable(context); + if (step === "ECS_CONFIGURATION_ACTIVE") return this.#containerModuleRegistered(context); + if (ROUTING_STEPS[step] !== undefined) return this.#routingPrecondition(step, context); + if (stepCapabilityGaps(step).length > 0) return blockedObservation(step); + if (step === "OPERATION_STARTED") return this.#operationStartedPrecondition(context); + return { + ok: false, + detail: `${step} has no precondition implementation in this release, and a step is never executed on an unchecked precondition`, + }; + } + + async execute( + step: ExecutableMigrationStepId, + context: Readonly, + ): Promise { + if (step === "OPERATION_STARTED") return this.#captureBeforeState(context); + if (step === "WORKLOAD_DRAINED") return this.#confirmDrained(context); + if (step === "CONFIGURATION_STAGED") return this.#stageConfiguration(context); + if (step === "SCHEDULER_DNS_RETAINED") return this.#retainSchedulerDns(context); + if (step === "ECS_CONFIGURATION_ACTIVE") return this.#activateContainerConfiguration(context); + if (ROUTING_STEPS[step] !== undefined) return this.#routeOrRemove(step, context); + const blocked = blockedObservation(step); + throw new MigrationRefusedError(blocked.detail); + } + + /** + * Announce maintenance, then verify. The tool never closes the batch server. + * + * It writes the maintenance rows, which is settings state it already owns, reads + * them back, reads every queue profile, reads the batch server's own state, and + * returns what it observed. A read that fails throws rather than returning a + * zero: an unobservable scheduler is not a closed scheduler. + * + * The driver decides on the returned values, and it refuses while anything still + * admits work, so this method prints the exact commands a person runs before it + * hands them back. + */ + async closeScheduler(context: Readonly): Promise { + const { deps } = this.#input; + const config = await this.#input.clusterConfig(context); + const maintenanceKey = `${config.moduleId("cluster-manager")}.maintenance`; + + const writer = await deps.configWriter({ + clusterName: context.clusterName, + awsRegion: context.awsRegion, + awsProfile: context.awsProfile, + }); + await writer.setConfigEntry(`${maintenanceKey}.message`, MAINTENANCE_MESSAGE); + await writer.setConfigEntry(`${maintenanceKey}.enabled`, true); + + const after = await this.#input.clusterConfig(context); + const maintenanceEnabled = after.getBool(`${maintenanceKey}.enabled`, false); + if (!maintenanceEnabled) { + throw new MigrationRefusedError( + `ADMISSION_CLOSED refused: ${maintenanceKey}.enabled read back as false after it was written. Check the settings table and the writer's permissions, then rerun.`, + ); + } + + const profiles = await this.#queueProfiles(context, config); + const enabledProfiles = profiles.filter((profile) => profile.enabled); + const server = await this.#input.schedulerState(context); + const serverEnabledQueues = server.queues.filter((queue) => queue.enabled).map((queue) => queue.name); + + // Both admission paths are named, because closing one leaves the other open: the profile + // rows gate the portal and the API, and the server's own queues gate a direct submission. + const enabledQueues = [ + ...serverEnabledQueues, + ...enabledProfiles.map((profile) => `profile:${profile.name}`), + ]; + if (server.scheduling || enabledQueues.length > 0) { + deps.out(`OBSERVED [ADMISSION_CLOSED] ${renderBatchServerState(server)}`); + deps.out("OBSERVED [ADMISSION_CLOSED] close the remaining admission controls by hand, then rerun:"); + // Only what is still open is printed, so a rerun does not ask for a command already run. + if (server.scheduling) { + deps.out(' on the batch server: /opt/pbs/bin/qmgr -c "set server scheduling = False"'); + } + for (const queue of serverEnabledQueues) { + deps.out(` on the batch server: /opt/pbs/bin/qmgr -c "set queue ${queue} enabled = False"`); + } + for (const profile of enabledProfiles) { + deps.out(` queue profile ${profile.name} still admits work through the portal and the API`); + } + } + + return { + detail: [ + `maintenance is announced (${maintenanceKey}.enabled=true)`, + `${profiles.length - enabledProfiles.length} of ${profiles.length} queue profiles are disabled`, + renderBatchServerState(server), + ].join("; "), + maintenanceEnabled, + schedulingEnabled: server.scheduling, + enabledQueues, + queuedJobs: server.queuedJobs, + provisioningJobs: server.provisioningJobs, + runningJobs: server.runningJobs, + }; + } + + /** Queue profiles, with the batch-server queues each one admits work to. */ + async #queueProfiles( + context: Readonly, + config: ClusterConfig, + ): Promise> { + const table = `${context.clusterName}.${config.moduleId("scheduler")}.queue-profiles`; + const rows = await scanTable(this.#input.deps, table); + return rows.map((row) => ({ + name: typeof row["name"] === "string" ? row["name"] : "unnamed", + enabled: row["enabled"] === true, + queues: Array.isArray(row["queues"]) ? row["queues"].filter((queue): queue is string => typeof queue === "string") : [], + })); + } + + /** + * Every admission path must be readable before anything is announced. + * + * A missing table, an unwritable settings store or a batch server that does not + * answer is a refusal here, where no maintenance row has been written yet. + */ + async #admissionPathsReachable(context: Readonly): Promise { + const config = await this.#input.clusterConfig(context); + let profiles: Array<{ name: string; enabled: boolean; queues: string[] }>; + try { + profiles = await this.#queueProfiles(context, config); + } catch (error) { + return { + ok: false, + detail: `the queue profile table for ${context.clusterName} could not be read (${error instanceof Error ? error.message : String(error)}), and an unobservable scheduler is not a closed scheduler`, + }; + } + let server: BatchServerState; + try { + server = await this.#input.schedulerState(context); + } catch (error) { + return { + ok: false, + detail: `the batch server's own state could not be read (${error instanceof Error ? error.message : String(error)}), and an unobservable scheduler is not a closed scheduler`, + }; + } + const enabled = profiles.filter((profile) => profile.enabled).length; + return { + ok: true, + detail: `the maintenance rows under ${config.moduleId("cluster-manager")}.maintenance are writable, ${profiles.length} queue profiles are readable with ${enabled} enabled, and the batch server answers: ${renderBatchServerState(server)}`, + }; + } + + /** + * Nothing may admit work before the drain is confirmed. + * + * Every value is read again here rather than carried from the closure, because a + * resumed run can reach this boundary long after admission was closed and a queue + * that was reopened in between must stop the run. + */ + async #drainPrecondition(context: Readonly): Promise { + const config = await this.#input.clusterConfig(context); + const maintenanceKey = `${config.moduleId("cluster-manager")}.maintenance.enabled`; + let server: BatchServerState; + let profiles: Array<{ name: string; enabled: boolean; queues: string[] }>; + try { + profiles = await this.#queueProfiles(context, config); + server = await this.#input.schedulerState(context); + } catch (error) { + return { + ok: false, + detail: `admission state could not be read (${error instanceof Error ? error.message : String(error)}), and an unobservable scheduler is not a closed scheduler`, + }; + } + const open = [ + ...(config.getBool(maintenanceKey, false) ? [] : [`${maintenanceKey} is not true`]), + ...(server.scheduling ? ["the batch server is still scheduling"] : []), + ...server.queues.filter((queue) => queue.enabled).map((queue) => `batch queue ${queue.name} is enabled`), + ...profiles.filter((profile) => profile.enabled).map((profile) => `queue profile ${profile.name} is enabled`), + ]; + if (open.length > 0) { + return { ok: false, detail: `these admission controls are still open: ${open.join(", ")}` }; + } + return { + ok: true, + detail: `maintenance is enabled, no queue profile admits work, and ${renderBatchServerState(server)}`, + }; + } + + /** + * Confirm the inventories are empty and name any compute node still carrying the + * legacy scheduler address. + * + * Retiring a node is a user-visible action, so this boundary refuses and names + * them rather than terminating them. With admission closed no new node is + * created, so the list can only shrink. + */ + async #confirmDrained(context: Readonly): Promise { + const server = await this.#input.schedulerState(context); + const active = server.queuedJobs + server.provisioningJobs + server.runningJobs; + if (active > 0) { + return { + ok: false, + detail: `the batch server still holds ${active} jobs (${renderBatchServerState(server)}). Wait for them to finish; do not cancel them here.`, + }; + } + const instances = await this.#input.reads.clusterInstances({ + awsRegion: context.awsRegion, + awsProfile: context.awsProfile, + clusterName: context.clusterName, + }); + const computeNodes = instances.filter( + (instance) => instance.nodeType === COMPUTE_NODE_TYPE && instance.state !== "terminated", + ); + if (computeNodes.length > 0) { + return { + ok: false, + detail: `${computeNodes.length} compute nodes still carry the legacy scheduler address and must be retired before the server moves: ${computeNodes.map((node) => `${node.instanceId}=${node.state}`).join(", ")}`, + }; + } + return { + ok: true, + detail: `the batch server holds no queued, provisioning or running jobs and no compute node carries the legacy scheduler address (${renderBatchServerState(server)})`, + }; + } + + async reconcile( + step: MigrationStepId, + context: Readonly, + ): Promise { + if (step !== "OPERATION_STARTED") { + return { + state: "retryable", + detail: `${step} never ran, because this release refuses it before its started marker is written`, + }; + } + const key = captureKey(context); + const bucket = await this.#captureBucket(context); + const objects = await this.#input.reads.bucketObjects({ + awsRegion: context.awsRegion, + awsProfile: context.awsProfile, + bucket, + prefix: key, + }); + const stored = objects.find((object) => object.key === key); + if (stored === undefined) { + return { state: "retryable", detail: `no before-state capture exists at s3://${bucket}/${key}` }; + } + return { + state: "committed", + detail: `the before-state capture at s3://${bucket}/${key} exists, ${stored.size ?? 0} bytes`, + }; + } + + // ------------------------------------------------------------------------------------------- + // step 0 + // ------------------------------------------------------------------------------------------- + + async #preflight(context: Readonly): Promise { + const identity = await this.#identity(context); + const preflightContext: PreflightContext = { + command: "migrate", + account: identity.account, + region: context.awsRegion, + cluster: context.clusterName, + ...(context.awsProfile === undefined ? {} : { profile: context.awsProfile }), + }; + + const registry = new PreflightRegistry() + .register(createAwsvpcTrunkingCheck(async () => this.#input.trunkingEnabled(context))) + .register(createTemplateComparisonCheck(async () => this.#templateComparisonEvidence(context))) + .register(createConfigurationDriftCheck(async () => this.#driftEvidence(context))); + + const report = await runPreflight(registry, preflightContext); + const notChecked = [ + "NOT CHECKED: that the control-plane image digest is present in this account's registry. No registry client is declared, so the account prerequisite is verified by hand.", + ]; + // The capability report says where the run stops; it is not a pre-flight failure. The run is + // meant to reach the boundary it cannot pass and refuse there, because that boundary is where + // a person is in the loop, and because every mutating step refuses before its started marker. + const detail = [ + renderPreflightReport(report), + ...notChecked, + ...(capabilityReport() === undefined ? [] : [capabilityReport() as string]), + ].join("\n"); + return { ok: report.passed, detail }; + } + + async #identity(context: Readonly): Promise<{ account: string; arn: string }> { + const { deps } = this.#input; + if (deps.callerIdentity !== undefined) { + return deps.callerIdentity({ awsRegion: context.awsRegion, awsProfile: context.awsProfile }); + } + return { account: await deps.accountId(), arn: "unknown" }; + } + + async #deployedStacks(context: Readonly): Promise> { + const config = await this.#input.clusterConfig(context); + const stacks = deployableStacks(config.modules(), context.clusterName); + if (stacks.length === 0) { + throw new GeneralException( + `No deployed module stacks were found for cluster ${context.clusterName}. Check --cluster-name, --aws-region and the module table.`, + ); + } + const digests: Array<{ moduleId: string; stackName: string; digest: string }> = []; + for (const stack of stacks) { + const template = await this.#input.reads.stackTemplate({ + awsRegion: context.awsRegion, + awsProfile: context.awsProfile, + stackName: stack.stackName, + }); + digests.push({ ...stack, digest: templateDigest(template) }); + } + return digests; + } + + async #templateComparisonEvidence( + context: Readonly, + ): Promise<{ matches: boolean; remedyCommand: string }> { + const stacks = await this.#deployedStacks(context); + const expected = templateComparisonFingerprint(this.#release(), stacks); + const remedyCommand = + `the target-template comparison for all ${stacks.length} module stacks and accept its result with --accept-template-comparison ${expected}`; + return { matches: context.acceptTemplateComparison === expected, remedyCommand }; + } + + async #driftEvidence( + context: Readonly, + ): Promise<{ reportHash: string; lossKeys: string[]; acceptedReportHash?: string }> { + const report = compareUpgradeDrift(await this.#input.driftInput(context)); + const lossKeys = [...report.changedRowsDifferingFromGenerated]; + return { + reportHash: driftReportFingerprint(this.#release(), lossKeys), + lossKeys, + ...(context.acceptDrift === undefined ? {} : { acceptedReportHash: context.acceptDrift }), + }; + } + + #release(): string { + return (this.#input.release ?? ideaVersion)(); + } + + // ------------------------------------------------------------------------------------------- + // step 1 + // ------------------------------------------------------------------------------------------- + + /** + * The fingerprints the comparison was accepted against must still hold, and + * no stack may be mid-operation. Both are read from the account. + */ + async #operationStartedPrecondition(context: Readonly): Promise { + const stacks = await this.#deployedStacks(context); + const fingerprint = templateComparisonFingerprint(this.#release(), stacks); + if (context.acceptTemplateComparison !== undefined && context.acceptTemplateComparison !== fingerprint) { + return { + ok: false, + detail: `a deployed template changed after the accepted comparison: the fingerprint is now ${fingerprint}. Repeat the comparison and accept the new value.`, + }; + } + const unstable: string[] = []; + for (const stack of stacks) { + const summary = await this.#input.reads.stackSummary({ + awsRegion: context.awsRegion, + awsProfile: context.awsProfile, + stackName: stack.stackName, + }); + if (!summary.status.endsWith("_COMPLETE") || summary.status.startsWith("DELETE")) { + unstable.push(`${stack.stackName}=${summary.status}`); + } + } + if (unstable.length > 0) { + return { + ok: false, + detail: `these stacks are not in a stable complete state: ${unstable.join(", ")}. Let the current operation finish, then rerun.`, + }; + } + return { + ok: true, + detail: `${stacks.length} module stacks are complete and the deployed-template fingerprint is ${fingerprint}`, + }; + } + + // ------------------------------------------------------------------------------------------- + // configuration boundaries + // ------------------------------------------------------------------------------------------- + + /** Write rows through the shared writer and read every one of them back. */ + async #writeRows( + context: Readonly, + rows: ReadonlyArray<{ key: string; value: unknown }>, + ): Promise { + const writer = await this.#input.deps.configWriter({ + clusterName: context.clusterName, + awsRegion: context.awsRegion, + awsProfile: context.awsProfile, + }); + for (const row of rows) await writer.setConfigEntry(row.key, row.value); + + const after = await this.#input.clusterConfig(context); + const wrong = rows.flatMap((row) => { + const actual = after.get(row.key, undefined); + // The DynamoDB type matters as much as the value: a boolean stored as a string reads as true. + return actual === row.value && typeof actual === typeof row.value + ? [] + : [`${row.key}=${JSON.stringify(actual)} (wanted ${JSON.stringify(row.value)})`]; + }); + if (wrong.length > 0) { + throw new MigrationRefusedError( + `these rows did not read back as written: ${wrong.join(", ")}. Check the settings table and the writer's permissions, then rerun.`, + ); + } + return rows.map((row) => `${row.key}=${JSON.stringify(row.value)}`); + } + + /** + * Container routing must still be off when configuration is staged. + * + * The flag turns on three steps later. With it on here, any module synthesis + * takes its hostless branch before the container target groups exist. + */ + async #stagingPrecondition(context: Readonly): Promise { + const config = await this.#input.clusterConfig(context); + if (config.getBool(CONTAINER_ENABLED_KEY, false)) { + return { + ok: false, + detail: `${CONTAINER_ENABLED_KEY} is already true, which is the ECS_CONFIGURATION_ACTIVE boundary, not this one. Stage the values file with the container key false, then rerun.`, + }; + } + return { ok: true, detail: `${CONTAINER_ENABLED_KEY} is off, so staging cannot select a hostless branch` }; + } + + /** Seed the container scheduler at zero. The later add-only sync keeps the seed. */ + async #stageConfiguration(context: Readonly): Promise { + const written = await this.#writeRows(context, [{ key: SCHEDULER_DESIRED_KEY, value: 0 }]); + throw new MigrationRefusedError( + [ + `CONFIGURATION_STAGED refused: seeded ${written.join(", ")}, which is the row this step owns, but the generated target configuration cannot be produced by this release.`, + ...stepCapabilityGaps("CONFIGURATION_STAGED").map( + (gap) => ` waiting on ${gap.id}: ${gap.description}. Change recorded in ${gap.request}.`, + ), + ].join("\n"), + ); + } + + /** The retain-only scheduler update needs a stable stack to update. */ + async #schedulerStackStable(context: Readonly): Promise { + const config = await this.#input.clusterConfig(context); + const stackName = deployableStacks(config.modules(), context.clusterName) + .find((stack) => stack.moduleId === config.moduleId("scheduler"))?.stackName; + if (stackName === undefined) { + return { ok: false, detail: `no deployed scheduler stack was found for ${context.clusterName}` }; + } + const summary = await this.#input.reads.stackSummary({ + awsRegion: context.awsRegion, + awsProfile: context.awsProfile, + stackName, + }); + if (!summary.status.endsWith("_COMPLETE") || summary.status.startsWith("DELETE")) { + return { ok: false, detail: `${stackName} is ${summary.status}, so a retain-only update cannot be the only change` }; + } + return { ok: true, detail: `${stackName} is ${summary.status}` }; + } + + /** + * Set the retain flag on the existing scheduler DNS record. + * + * The scheduler stack already expresses the retain shape behind this row. Without + * it the target template stops managing the record and the name execution hosts + * resolve is deleted with the resource. + */ + async #retainSchedulerDns(context: Readonly): Promise { + const written = await this.#writeRows(context, [{ key: SCHEDULER_RETAIN_DNS_KEY, value: true }]); + throw new MigrationRefusedError( + [ + `SCHEDULER_DNS_RETAINED refused: set ${written.join(", ")}, so the next scheduler synthesis keeps the record with a retain policy, but this release cannot deploy that one module and read the committed template back.`, + ...stepCapabilityGaps("SCHEDULER_DNS_RETAINED").map( + (gap) => ` waiting on ${gap.id}: ${gap.description}. Change recorded in ${gap.request}.`, + ), + ].join("\n"), + ); + } + + /** The container module must be registered before its flag turns on. */ + async #containerModuleRegistered(context: Readonly): Promise { + const config = await this.#input.clusterConfig(context); + const registered = config.modules().some((module) => module.name === "ecs" || module.module_id === "ecs"); + if (!registered) { + return { + ok: false, + detail: [ + `the container module has no row in ${context.clusterName}.modules, so turning ${CONTAINER_ENABLED_KEY} on would make every later module synthesis select a hostless branch with no module to deploy.`, + ...stepCapabilityGaps("ECS_CONFIGURATION_ACTIVE").map( + (gap) => ` waiting on ${gap.id}: ${gap.description}. Change recorded in ${gap.request}.`, + ), + ].join("\n"), + }; + } + return { ok: true, detail: "the container module is registered in the module table" }; + } + + /** A route or removal boundary only exists under container routing. */ + async #routingPrecondition( + step: MigrationStepId, + context: Readonly, + ): Promise { + const config = await this.#input.clusterConfig(context); + if (!config.getBool(CONTAINER_ENABLED_KEY, false)) { + return { + ok: false, + detail: `${CONTAINER_ENABLED_KEY} is off, so there is nothing to route to. This boundary runs after ECS_CONFIGURATION_ACTIVE.`, + }; + } + const plan = ROUTING_STEPS[step]; + return { + ok: true, + detail: `container routing is on and ${plan?.moduleName ?? "the module"} ${plan?.retainHosts === true ? "keeps" : "loses"} its legacy hosts at this boundary`, + }; + } + + /** + * Drive the route and removal split with the two inputs the stacks read. + * + * `ecs.retain_existing_hosts` is what makes a routed step a route-only change: + * the endpoints move to the container targets and every legacy host stays, so + * the new target can be proved before the old one is gone. The removal step + * clears the same row, and only the module named at that boundary is deployed. + */ + async #routeOrRemove( + step: MigrationStepId, + context: Readonly, + ): Promise { + const plan = ROUTING_STEPS[step]; + if (plan === undefined) throw new TypeError(`${step} is not a routing boundary`); + const written = await this.#writeRows(context, [{ key: RETAIN_HOSTS_KEY, value: plan.retainHosts }]); + throw new MigrationRefusedError( + [ + `${step} refused: set ${written.join(", ")}, so a deploy of ${plan.moduleName} now ${plan.retainHosts ? "routes its endpoints to the container targets and keeps every legacy host" : "removes only its legacy host resources"}, but this release cannot deploy that one module and prove its production path.`, + ...stepCapabilityGaps(step).map( + (gap) => ` waiting on ${gap.id}: ${gap.description}. Change recorded in ${gap.request}.`, + ), + ].join("\n"), + ); + } + + /** Turn container routing on, with the stable scheduler name and the task at zero. */ + async #activateContainerConfiguration(context: Readonly): Promise { + const written = await this.#writeRows(context, [ + { key: CONTAINER_IMAGE_KEY, value: context.imageDigest }, + { key: SCHEDULER_STABLE_NAME_KEY, value: true }, + { key: SCHEDULER_DESIRED_KEY, value: 0 }, + { key: CONTAINER_ENABLED_KEY, value: true }, + ]); + return { ok: true, detail: `container configuration active: ${written.join("; ")}` }; + } + + async #captureBucket(context: Readonly): Promise { + const config = await this.#input.clusterConfig(context); + const bucket = config.getString("cluster.cluster_s3_bucket", ""); + if (bucket === "") { + throw new GeneralException( + `cluster.cluster_s3_bucket is not set for ${context.clusterName}, so the before-state capture has nowhere to go.`, + ); + } + return bucket; + } + + /** + * Capture the exact before-state the record refers to. + * + * The capture is an object in the cluster bucket, and the observation names + * its key and digest, because the record holds references rather than copies. + */ + async #captureBeforeState(context: Readonly): Promise { + const { deps, reads } = this.#input; + const target = { awsRegion: context.awsRegion, awsProfile: context.awsProfile }; + const config = await this.#input.clusterConfig(context); + const bucket = await this.#captureBucket(context); + + const settings = await scanTable(deps, `${context.clusterName}.cluster-settings`); + const modules = await scanTable(deps, `${context.clusterName}.modules`); + // The prior enabled value of every queue profile: the reopen boundary restores these, and they + // live in their own table rather than in cluster settings. + const queueProfiles = await this.#queueProfiles(context, config); + const stacks = await this.#deployedStacks(context); + const stackSummaries: Record = {}; + for (const stack of stacks) { + stackSummaries[stack.stackName] = { + ...(await reads.stackSummary({ ...target, stackName: stack.stackName })), + digest: stack.digest, + }; + } + + const instances = await reads.clusterInstances({ ...target, clusterName: context.clusterName }); + const listeners = await reads.clusterListeners({ ...target, clusterName: context.clusterName }); + const targetGroupArns = [ + ...new Set( + listeners.flatMap((listener) => [ + ...listener.defaultTargetGroupArns, + ...listener.rules.flatMap((rule) => rule.targetGroupArns), + ]), + ), + ]; + const targetGroups = await reads.targetGroupHealth({ ...target, targetGroupArns }); + + const hostedZoneId = config.getString("cluster.route53.private_hosted_zone_id", ""); + const recordSets = hostedZoneId === "" + ? [] + : await reads.recordSets({ ...target, hostedZoneId }); + + const packages = await reads.bucketObjects({ ...target, bucket, prefix: "idea/bootstrap/" }); + let valuesFileSha256 = "absent"; + try { + valuesFileSha256 = sha256(await deps.s3.getObject({ Bucket: bucket, Key: "values/values.yml" })); + } catch (error) { + // A cluster installed before the bucket copy existed has no stored values file. Record + // that, because a restore needs to know which values file the run started from. + deps.out(`OBSERVED [OPERATION_STARTED] values/values.yml is not readable in the cluster bucket: ${error instanceof Error ? error.message : String(error)}`); + } + // The same accessor the status command uses, so a cluster with a generated load balancer name + // and one with a custom name are both reached. + let health: number | undefined; + try { + health = await deps.httpStatus(config.getClusterExternalEndpoint()); + } catch (error) { + deps.out(`OBSERVED [OPERATION_STARTED] the cluster external endpoint is not resolvable: ${error instanceof Error ? error.message : String(error)}`); + } + + const capture = { + release: this.#release(), + deploymentId: context.deploymentId, + clusterName: context.clusterName, + awsRegion: context.awsRegion, + moduleSet: context.moduleSet, + selectedModules: [...context.selectedModules], + targetBaseOs: context.targetBaseOs, + imageDigest: context.imageDigest, + capturedAt: new Date(deps.now()).toISOString(), + settings, + modules, + queueProfiles, + stacks: stackSummaries, + instances, + listeners, + targetGroups, + recordSets, + packages, + valuesFileSha256, + preRunHealthStatus: health, + }; + const body = JSON.stringify(capture); + const key = captureKey(context); + await deps.s3.putObject({ Bucket: bucket, Key: key, Body: body }); + + const stored = (await reads.bucketObjects({ ...target, bucket, prefix: key })) + .find((object) => object.key === key); + if (stored === undefined) { + return { + ok: false, + detail: `the before-state capture was written to s3://${bucket}/${key} but a read back did not find it`, + }; + } + return { + ok: true, + detail: [ + `captured s3://${bucket}/${key}`, + `sha256=${sha256(body)}`, + `settings=${settings.length}`, + `modules=${modules.length}`, + `queueProfiles=${queueProfiles.length}/${queueProfiles.filter((profile) => profile.enabled).length} enabled`, + `stacks=${stacks.length}`, + `instances=${instances.length}`, + `running=${instances.filter((instance) => instance.state === "running").length}`, + `listeners=${listeners.length}`, + `targetGroups=${targetGroups.length}`, + `recordSets=${recordSets.length}`, + `packages=${packages.length}`, + `valuesFile=${valuesFileSha256 === "absent" ? "absent" : `sha256:${valuesFileSha256.slice(0, 16)}`}`, + `preRunHealth=${health ?? "not checked"}`, + ].join("; "), + }; + } +} + +/** Configuration rows the migration owns, by the step that writes them. */ +const CONTAINER_ENABLED_KEY = "ecs.enabled"; +const CONTAINER_IMAGE_KEY = "ecs.image"; +const SCHEDULER_DESIRED_KEY = "ecs.tasks.scheduler.desired"; +const SCHEDULER_STABLE_NAME_KEY = "scheduler.use_stable_server_name"; +const SCHEDULER_RETAIN_DNS_KEY = "scheduler.retain_dns_record"; +const RETAIN_HOSTS_KEY = "ecs.retain_existing_hosts"; + +/** The tag value a scheduler compute node carries. */ +const COMPUTE_NODE_TYPE = "compute-node"; + +/** The endpoint-routing boundaries, and whether each one keeps the legacy hosts. */ +const ROUTING_STEPS: Readonly>> = { + CLUSTER_MANAGER_ROUTED: { moduleName: "cluster-manager", retainHosts: true }, + CLUSTER_MANAGER_LEGACY_REMOVED: { moduleName: "cluster-manager", retainHosts: false }, + VDC_ROUTED: { moduleName: "virtual-desktop-controller", retainHosts: true }, + VDC_LEGACY_REMOVED: { moduleName: "virtual-desktop-controller", retainHosts: false }, + SCHEDULER_ROUTED: { moduleName: "scheduler", retainHosts: true }, + SCHEDULER_LEGACY_REMOVED: { moduleName: "scheduler", retainHosts: false }, +}; + +/** Where one operation's before-state capture lives. */ +export function captureKey(context: Readonly): string { + return `values/migration/${context.deploymentId}/before-state.json`; +} + +/** Page one cluster table in full. */ +async function scanTable(deps: Deps, tableName: string): Promise>> { + const rows: Array> = []; + let startKey: Record | undefined; + do { + const page = await deps.scan({ TableName: tableName, ExclusiveStartKey: startKey }); + rows.push(...(page.Items ?? [])); + startKey = page.LastEvaluatedKey; + } while (startKey !== undefined); + return rows; +} + +// --------------------------------------------------------------------------------------------- +// factory +// --------------------------------------------------------------------------------------------- + +/** + * Build the live migration dependencies. + * + * `awsRegion` resolves the region the command selected, because the durable + * record store is created once and every other read takes its region from the + * migration context. + */ +export function createLiveMigrateDeps( + deps: Deps, + awsRegion: () => string, + awsProfile: () => string | undefined = () => process.env.AWS_PROFILE, +): MigrateDeps { + const upgrade = createLiveUpgradeDeps(deps); + const reads = liveMigrationAccountReads(); + + return { + stateObjects: liveMigrationStateObjects(awsRegion, awsProfile), + steps: new LiveMigrationSteps({ + deps, + reads, + async clusterConfig(context) { + return ClusterConfig.fromDynamoDb(context.clusterName, context.awsRegion, { + moduleSet: context.moduleSet, + scan: deps.scan, + }); + }, + async driftInput(context) { + return prepareUpgradeDriftInput(upgrade, { + clusterName: context.clusterName, + awsRegion: context.awsRegion, + awsProfile: context.awsProfile, + baseOs: context.targetBaseOs, + modules: [...context.selectedModules], + }); + }, + /** + * The batch server's own state, from the host it runs on. + * + * It refuses rather than answers once the server is a task: a host reply at + * that point would describe an instance that no longer serves the cluster, + * and a stale answer to this question is worse than no answer. The steps + * after the cutover wait on the container-side interface for that reason. + */ + async schedulerState(context) { + const config = await ClusterConfig.fromDynamoDb(context.clusterName, context.awsRegion, { + moduleSet: context.moduleSet, + scan: deps.scan, + }); + if (config.getBool("ecs.enabled", false)) { + throw new SchedulerStateUnreadableError( + "the batch server runs as a task on this cluster, so its state is not readable from a host. This boundary waits on the container-side read interface recorded in docs/port/requests/migrate-live.md section 1.", + ); + } + const instanceId = config.getString(`${config.moduleId("scheduler")}.instance_id`, ""); + if (instanceId === "") { + throw new SchedulerStateUnreadableError( + `no scheduler instance is recorded for ${context.clusterName}, so the batch server's state cannot be read`, + ); + } + return readBatchServerState( + liveSsmReadChannel({ + awsRegion: context.awsRegion, + awsProfile: context.awsProfile, + sleep: deps.sleep, + clientOptions: awsClientOptions, + }), + instanceId, + ); + }, + async trunkingEnabled(context) { + const accountSettings = upgrade.ecsAccountSettings; + if (accountSettings === undefined) { + throw new GeneralException( + "the container account-setting reader is not wired, so task network interface trunking cannot be checked", + ); + } + const settings = await accountSettings.listAccountSettings({ + awsRegion: context.awsRegion, + effectiveSettings: true, + name: "awsvpcTrunking", + }); + return settings.some((setting) => setting.name === "awsvpcTrunking" && setting.value === "enabled"); + }, + }), + uuid: deps.uuid, + targetVersion: () => ideaVersion(), + out: deps.out, + now: deps.now, + }; +} diff --git a/source/idea/ideactl/src/cli/live-operator-adapters.ts b/source/idea/ideactl/src/cli/live-operator-adapters.ts new file mode 100644 index 00000000..e4ac2490 --- /dev/null +++ b/source/idea/ideactl/src/cli/live-operator-adapters.ts @@ -0,0 +1,876 @@ +/** + * Live dependency factories for the operator command groups. + * + * Each factory receives the action options after parsing, so configuration + * reads and every SDK client use the profile selected for that action. + */ + +import { spawn } from "node:child_process"; +import { basename, dirname, join } from "node:path"; + +import yaml from "js-yaml"; + +import { ClusterConfig, ClusterConfigError, GeneralException } from "../config/cluster-config.ts"; +import { awsClientOptions } from "./aws-client-options.ts"; +import type { Deps } from "./cdk-invoker.ts"; +import { + CLUSTER_NAME_TAG, + MODULE_ID_TAG, + AUTOSCALING_GROUP_TAG, + NODE_TYPE_TAG, + type DeleteClusterDeps, + type DeleteClusterDepsFactory, + type DeleteClusterInstance, +} from "./commands/delete-cluster.ts"; +import type { RemainingOperatorCommandDeps } from "./commands/utils.ts"; + +interface AwsActionOptions { + clusterName?: string; + awsRegion?: string; + awsProfile?: string; + moduleSet?: string; +} + +function requiredRegion(options: AwsActionOptions): string { + if (options.awsRegion === undefined || options.awsRegion.trim() === "") { + throw new GeneralException("This command needs --aws-region (for example us-east-2). Pass it on the command line."); + } + return options.awsRegion; +} + +const clientOptions = (options: AwsActionOptions) => + awsClientOptions(requiredRegion(options), options.awsProfile); + +async function loadConfig(deps: Deps, options: AwsActionOptions): Promise { + if (options.clusterName === undefined || options.clusterName.trim() === "") { + throw new GeneralException("This command needs --cluster-name. Pass it on the command line."); + } + return ClusterConfig.fromDynamoDb(options.clusterName, requiredRegion(options), { + moduleSet: options.moduleSet, + scan: deps.scan, + }); +} + +async function scanAll( + deps: Deps, + tableName: string, +): Promise>> { + const rows: Array> = []; + let key: Record | undefined; + do { + const page = await deps.scan({ TableName: tableName, ExclusiveStartKey: key }); + rows.push(...(page.Items ?? [])); + key = page.LastEvaluatedKey; + } while (key !== undefined); + return rows; +} + +async function archiveDirectory(directory: string): Promise { + const archive = `${directory}.tar.gz`; + await new Promise((resolve, reject) => { + const process = spawn("tar", ["-czf", archive, "-C", dirname(directory), basename(directory)]); + process.once("error", reject); + process.once("close", (code) => { + if (code === 0) resolve(); + else reject(new GeneralException(`could not create support archive: tar exited with ${code ?? 1}`)); + }); + }); + return archive; +} + +/** + * Live dependencies for `delete-cluster` and `delete-backups`. + * + * Every client is built on first use from the one options helper the other command groups use, so + * the profile and region the action selected are the only credential path. The per-service + * `lazy...` helpers exist because this command reaches nine services: they hold the dynamic import + * and the client construction in one place instead of repeating both in thirty methods. + */ +async function deleteClusterDeps( + deps: Deps, + options: AwsActionOptions, +): Promise { + const lazyEc2 = async () => { + const sdk = await import("@aws-sdk/client-ec2"); + return { sdk, client: new sdk.EC2Client(await clientOptions(options)) }; + }; + const lazyCfn = async () => { + const sdk = await import("@aws-sdk/client-cloudformation"); + return { sdk, client: new sdk.CloudFormationClient(await clientOptions(options)) }; + }; + const lazySsm = async () => { + const sdk = await import("@aws-sdk/client-ssm"); + return { sdk, client: new sdk.SSMClient(await clientOptions(options)) }; + }; + const lazyCognito = async () => { + const sdk = await import("@aws-sdk/client-cognito-identity-provider"); + return { sdk, client: new sdk.CognitoIdentityProviderClient(await clientOptions(options)) }; + }; + const lazyDynamoDb = async () => { + const sdk = await import("@aws-sdk/client-dynamodb"); + return { sdk, client: new sdk.DynamoDBClient(await clientOptions(options)) }; + }; + const lazyCloudWatch = async () => { + const sdk = await import("@aws-sdk/client-cloudwatch"); + return { sdk, client: new sdk.CloudWatchClient(await clientOptions(options)) }; + }; + const lazyLogs = async () => { + const sdk = await import("@aws-sdk/client-cloudwatch-logs"); + return { sdk, client: new sdk.CloudWatchLogsClient(await clientOptions(options)) }; + }; + const lazyS3 = async () => { + const sdk = await import("@aws-sdk/client-s3"); + return { sdk, client: new sdk.S3Client(await clientOptions(options)) }; + }; + const lazyIam = async () => { + const sdk = await import("@aws-sdk/client-iam"); + return { sdk, client: new sdk.IAMClient(await clientOptions(options)) }; + }; + + /** `ResourceNotFoundException` for a table this cluster never created. */ + const isMissingTable = (error: unknown): boolean => + (error as { name?: string }).name === "ResourceNotFoundException"; + + return { + async loadConfig(input) { + // A cluster whose settings tables are already gone is still deletable: the command falls + // back to the generated names for every value it would have read. + try { + return await ClusterConfig.fromDynamoDb(input.clusterName, requiredRegion(options), { + moduleSet: options.moduleSet, + scan: deps.scan, + }); + } catch (error) { + // `ClusterConfigError` is what a cluster with no settings tables raises, which is the + // normal state of a half-deleted cluster and the one this command has to survive. + if (isMissingTable(error) || error instanceof ClusterConfigError || error instanceof GeneralException) { + return undefined; + } + throw error; + } + }, + async findInstances(input) { + const { sdk, client } = await lazyEc2(); + const found: DeleteClusterInstance[] = []; + let nextToken: string | undefined; + do { + const result = await client.send( + new sdk.DescribeInstancesCommand({ + Filters: input.filters.map((filter) => ({ Name: filter.name, Values: filter.values })), + NextToken: nextToken, + }), + ); + nextToken = result.NextToken; + for (const reservation of result.Reservations ?? []) { + for (const instance of reservation.Instances ?? []) { + if (instance.InstanceId === undefined) continue; + found.push({ + instanceId: instance.InstanceId, + state: instance.State?.Name ?? "unknown", + nodeType: instance.Tags?.find((tag) => tag.Key === NODE_TYPE_TAG)?.Value, + // Set by the platform on every instance an auto scaling group launches, and already + // in this response, so telling a group member from a standalone instance costs no + // extra call. + autoScalingGroupName: instance.Tags?.find((tag) => tag.Key === AUTOSCALING_GROUP_TAG)?.Value, + }); + } + } + } while (nextToken !== undefined && nextToken !== ""); + return found; + }, + async instanceTerminationProtection(instanceId) { + const { sdk, client } = await lazyEc2(); + const result = await client.send( + new sdk.DescribeInstanceAttributeCommand({ InstanceId: instanceId, Attribute: "disableApiTermination" }), + ); + return result.DisableApiTermination?.Value === true; + }, + async disableInstanceTerminationProtection(instanceId) { + const { sdk, client } = await lazyEc2(); + await client.send( + new sdk.ModifyInstanceAttributeCommand({ InstanceId: instanceId, DisableApiTermination: { Value: false } }), + ); + }, + async terminateInstance(input) { + const { sdk, client } = await lazyEc2(); + await client.send( + new sdk.TerminateInstancesCommand({ + InstanceIds: [input.instanceId], + ...(input.force ? { Force: true, SkipOsShutdown: input.skipOsShutdown } : {}), + }), + ); + }, + async getTaggedStacks(input) { + const sdk = await import("@aws-sdk/client-resource-groups-tagging-api"); + const client = new sdk.ResourceGroupsTaggingAPIClient(await clientOptions(options)); + const result = await client.send( + new sdk.GetResourcesCommand({ + TagFilters: input.tagFilters.map((filter) => ({ Key: filter.key, Values: filter.values })), + ResourceTypeFilters: input.resourceTypeFilters, + PaginationToken: input.paginationToken, + }), + ); + return { + stacks: (result.ResourceTagMappingList ?? []).flatMap((resource) => + resource.ResourceARN === undefined ? [] : [resource.ResourceARN], + ), + paginationToken: result.PaginationToken, + }; + }, + async describeStack(stackName) { + const { sdk, client } = await lazyCfn(); + const result = await client.send(new sdk.DescribeStacksCommand({ StackName: stackName })); + const stack = result.Stacks?.[0]; + if (stack?.StackName === undefined) throw new GeneralException(`stack not found: ${stackName}`); + return { + stackName: stack.StackName, + stackStatus: stack.StackStatus, + terminationProtection: stack.EnableTerminationProtection, + }; + }, + async stackFailedResources(stackName) { + const { sdk, client } = await lazyCfn(); + const failed: string[] = []; + let nextToken: string | undefined; + do { + const result = await client.send( + new sdk.ListStackResourcesCommand({ StackName: stackName, NextToken: nextToken }), + ); + nextToken = result.NextToken; + for (const resource of result.StackResourceSummaries ?? []) { + if (resource.ResourceStatus === "DELETE_FAILED" && resource.LogicalResourceId !== undefined) { + failed.push(resource.LogicalResourceId); + } + } + } while (nextToken !== undefined && nextToken !== ""); + return failed; + }, + async disableStackTerminationProtection(stackName) { + const { sdk, client } = await lazyCfn(); + await client.send( + new sdk.UpdateTerminationProtectionCommand({ StackName: stackName, EnableTerminationProtection: false }), + ); + }, + async deleteStack(stackName, retainResources) { + const { sdk, client } = await lazyCfn(); + await client.send( + new sdk.DeleteStackCommand({ + StackName: stackName, + ...(retainResources === undefined || retainResources.length === 0 + ? {} + : { RetainResources: retainResources }), + }), + ); + }, + async findAppInstance(input) { + const { sdk, client } = await lazyEc2(); + const result = await client.send( + new sdk.DescribeInstancesCommand({ + Filters: [ + { Name: "instance-state-name", Values: ["pending", "stopped", "running"] }, + { Name: `tag:${CLUSTER_NAME_TAG}`, Values: [input.clusterName] }, + { Name: `tag:${MODULE_ID_TAG}`, Values: [input.moduleId] }, + { Name: `tag:${NODE_TYPE_TAG}`, Values: ["app"] }, + ], + }), + ); + for (const reservation of result.Reservations ?? []) { + for (const instance of reservation.Instances ?? []) { + if (instance.InstanceId !== undefined && instance.State?.Name === "running") { + return { instanceId: instance.InstanceId, state: "running" }; + } + } + } + return undefined; + }, + async sendAppCleanup(input) { + // SendCommand rejects an empty target list, and a cluster whose application hosts are + // already gone has nothing to clean up. The empty command id reads back as no invocations. + if (input.instanceIds.length === 0) return ""; + const { sdk, client } = await lazySsm(); + const command = `sudo ideactl app-module-clean-up${input.deleteDatabases ? " --delete-databases" : ""}`; + const result = await client.send( + new sdk.SendCommandCommand({ + InstanceIds: input.instanceIds, + DocumentName: "AWS-RunShellScript", + Parameters: { commands: [command] }, + }), + ); + const commandId = result.Command?.CommandId; + if (commandId === undefined) throw new GeneralException("ssm:SendCommand returned no command id"); + return commandId; + }, + async appCleanupStatus(commandId) { + if (commandId === "") return []; + const { sdk, client } = await lazySsm(); + const result = await client.send( + new sdk.ListCommandInvocationsCommand({ CommandId: commandId, Details: false }), + ); + return (result.CommandInvocations ?? []).map((invocation) => ({ status: invocation.Status ?? "Pending" })); + }, + async findBedrockProjects(cluster) { + const { DynamoDBDocumentClient, ScanCommand } = await import("@aws-sdk/lib-dynamodb"); + const { client } = await lazyDynamoDb(); + const document = DynamoDBDocumentClient.from(client); + const projects: Array> = []; + let startKey: Record | undefined; + try { + do { + const page = await document.send( + new ScanCommand({ TableName: `${cluster}.projects`, ExclusiveStartKey: startKey }), + ); + startKey = page.LastEvaluatedKey; + for (const item of page.Items ?? []) { + const bedrock = (item as { bedrock?: Record }).bedrock ?? {}; + const hasRole = typeof bedrock["role_arn"] === "string" && bedrock["role_arn"] !== ""; + const hasProfile = + typeof bedrock["instance_profile_arn"] === "string" && bedrock["instance_profile_arn"] !== ""; + const inference = bedrock["inference_profile_arns"]; + const hasInference = + typeof inference === "object" && inference !== null && Object.keys(inference).length > 0; + if (hasRole || hasProfile || hasInference) projects.push(item); + } + } while (startKey !== undefined); + } catch (error) { + if (isMissingTable(error)) return []; + throw error; + } + return projects; + }, + async deleteBedrockProjectResources(input) { + const { sdk, client: iam } = await lazyIam(); + // Every step is best effort: the command reports a failure and carries on to the stack + // delete, where the boundary-detaching custom resource clears what blocks a policy delete. + const attempt = async (description: string, call: () => Promise): Promise => { + try { + await call(); + deps.out(description); + } catch (error) { + deps.err(`${description} failed: ${error instanceof Error ? error.message : String(error)}`); + } + }; + const inferenceProfileArns: string[] = []; + for (const project of input.projects) { + const bedrock = (project as { bedrock?: Record }).bedrock ?? {}; + const roleArn = typeof bedrock["role_arn"] === "string" ? bedrock["role_arn"] : ""; + const instanceProfileArn = + typeof bedrock["instance_profile_arn"] === "string" ? bedrock["instance_profile_arn"] : ""; + const roleName = roleArn === "" ? undefined : roleArn.split("/").pop(); + const instanceProfileName = instanceProfileArn === "" ? undefined : instanceProfileArn.split("/").pop(); + const inference = bedrock["inference_profile_arns"]; + if (typeof inference === "object" && inference !== null) { + for (const value of Object.values(inference as Record)) { + if (typeof value === "string" && value !== "") inferenceProfileArns.push(value); + } + } + + if (roleName !== undefined && roleName !== "") { + try { + const attached = await iam.send(new sdk.ListAttachedRolePoliciesCommand({ RoleName: roleName })); + for (const policy of attached.AttachedPolicies ?? []) { + if (policy.PolicyArn === undefined) continue; + await attempt(`detached ${policy.PolicyArn} from role ${roleName}`, () => + iam.send(new sdk.DetachRolePolicyCommand({ RoleName: roleName, PolicyArn: policy.PolicyArn })), + ); + } + } catch (error) { + deps.err(`could not list policies of role ${roleName}: ${error instanceof Error ? error.message : String(error)}`); + } + } + + if (instanceProfileName !== undefined && instanceProfileName !== "") { + try { + const profile = await iam.send( + new sdk.GetInstanceProfileCommand({ InstanceProfileName: instanceProfileName }), + ); + for (const role of profile.InstanceProfile?.Roles ?? []) { + if (role.RoleName === undefined) continue; + await attempt(`removed role ${role.RoleName} from instance profile ${instanceProfileName}`, () => + iam.send( + new sdk.RemoveRoleFromInstanceProfileCommand({ + InstanceProfileName: instanceProfileName, + RoleName: role.RoleName, + }), + ), + ); + } + } catch (error) { + deps.err(`could not read instance profile ${instanceProfileName}: ${error instanceof Error ? error.message : String(error)}`); + } + await attempt(`deleted instance profile ${instanceProfileName}`, () => + iam.send(new sdk.DeleteInstanceProfileCommand({ InstanceProfileName: instanceProfileName })), + ); + } + + if (roleName !== undefined && roleName !== "") { + await attempt(`deleted role ${roleName}`, () => iam.send(new sdk.DeleteRoleCommand({ RoleName: roleName }))); + } + } + + // Every policy under the cluster's project path, including one whose role is already gone. + let marker: string | undefined; + do { + const page = await iam.send( + new sdk.ListPoliciesCommand({ + PathPrefix: `/idea/${input.clusterName}/projects/`, + Scope: "Local", + Marker: marker, + }), + ); + marker = page.IsTruncated === true ? page.Marker : undefined; + for (const policy of page.Policies ?? []) { + const policyArn = policy.Arn; + if (policyArn === undefined) continue; + try { + const versions = await iam.send(new sdk.ListPolicyVersionsCommand({ PolicyArn: policyArn })); + for (const version of versions.Versions ?? []) { + if (version.IsDefaultVersion === true || version.VersionId === undefined) continue; + await attempt(`deleted policy version ${version.VersionId} of ${policyArn}`, () => + iam.send(new sdk.DeletePolicyVersionCommand({ PolicyArn: policyArn, VersionId: version.VersionId })), + ); + } + } catch (error) { + deps.err(`could not list versions of ${policyArn}: ${error instanceof Error ? error.message : String(error)}`); + } + await attempt(`deleted policy ${policyArn}`, () => + iam.send(new sdk.DeletePolicyCommand({ PolicyArn: policyArn })), + ); + } + } while (marker !== undefined && marker !== ""); + + if (inferenceProfileArns.length > 0) { + const bedrockSdk = await import("@aws-sdk/client-bedrock"); + const bedrock = new bedrockSdk.BedrockClient(await clientOptions(options)); + for (const profileArn of inferenceProfileArns) { + await attempt(`deleted inference profile ${profileArn}`, () => + bedrock.send( + new bedrockSdk.DeleteInferenceProfileCommand({ inferenceProfileIdentifier: profileArn.split("/").pop() }), + ), + ); + } + } + }, + async listUserPools(nextToken) { + const { sdk, client } = await lazyCognito(); + const result = await client.send(new sdk.ListUserPoolsCommand({ MaxResults: 50, NextToken: nextToken })); + return { + pools: (result.UserPools ?? []).flatMap((pool) => + pool.Id === undefined ? [] : [{ id: pool.Id, name: pool.Name ?? "" }], + ), + nextToken: result.NextToken, + }; + }, + async describeUserPool(userPoolId) { + const { sdk, client } = await lazyCognito(); + const result = await client.send(new sdk.DescribeUserPoolCommand({ UserPoolId: userPoolId })); + return { + deletionProtection: result.UserPool?.DeletionProtection, + tags: result.UserPool?.UserPoolTags, + }; + }, + async disableUserPoolDeletionProtection(userPoolId) { + const { sdk, client } = await lazyCognito(); + await client.send(new sdk.UpdateUserPoolCommand({ UserPoolId: userPoolId, DeletionProtection: "INACTIVE" })); + }, + async describeLambdaNetworkInterfaces(input) { + const { sdk, client } = await lazyEc2(); + const found: Array<{ networkInterfaceId: string; description?: string }> = []; + let nextToken: string | undefined; + do { + const result = await client.send( + new sdk.DescribeNetworkInterfacesCommand({ + Filters: [ + { Name: "description", Values: [`AWS Lambda VPC ENI-${input.clusterName}-*`] }, + { Name: "status", Values: ["available"] }, + ], + NextToken: nextToken, + }), + ); + nextToken = result.NextToken; + for (const networkInterface of result.NetworkInterfaces ?? []) { + if (networkInterface.NetworkInterfaceId === undefined) continue; + found.push({ + networkInterfaceId: networkInterface.NetworkInterfaceId, + description: networkInterface.Description, + }); + } + } while (nextToken !== undefined && nextToken !== ""); + return found; + }, + async deleteNetworkInterface(networkInterfaceId) { + const { sdk, client } = await lazyEc2(); + await client.send(new sdk.DeleteNetworkInterfaceCommand({ NetworkInterfaceId: networkInterfaceId })); + }, + async describeBackupVault(backupVaultName) { + const { BackupClient, DescribeBackupVaultCommand } = await import("@aws-sdk/client-backup"); + await new BackupClient(await clientOptions(options)).send(new DescribeBackupVaultCommand({ BackupVaultName: backupVaultName })); + }, + async listRecoveryPoints(backupVaultName) { + const { BackupClient, ListRecoveryPointsByBackupVaultCommand } = await import("@aws-sdk/client-backup"); + const result = await new BackupClient(await clientOptions(options)).send( + new ListRecoveryPointsByBackupVaultCommand({ BackupVaultName: backupVaultName }), + ); + return (result.RecoveryPoints ?? []).flatMap((point) => + point.RecoveryPointArn === undefined ? [] : [{ arn: point.RecoveryPointArn, status: point.Status }], + ); + }, + async deleteRecoveryPoint(input) { + const { BackupClient, DeleteRecoveryPointCommand } = await import("@aws-sdk/client-backup"); + await new BackupClient(await clientOptions(options)).send( + new DeleteRecoveryPointCommand({ BackupVaultName: input.backupVaultName, RecoveryPointArn: input.recoveryPointArn }), + ); + }, + async listTables(nextTableName) { + const { sdk, client } = await lazyDynamoDb(); + const result = await client.send(new sdk.ListTablesCommand({ ExclusiveStartTableName: nextTableName })); + return { tableNames: result.TableNames ?? [], nextTableName: result.LastEvaluatedTableName }; + }, + async deleteTable(tableName) { + const { sdk, client } = await lazyDynamoDb(); + await client.send(new sdk.DeleteTableCommand({ TableName: tableName })); + }, + async listDynamoDbAlarms(cluster) { + const { sdk, client } = await lazyCloudWatch(); + const alarms: Array<{ name: string; namespace: string; tableName?: string }> = []; + let nextToken: string | undefined; + do { + const result = await client.send( + new sdk.DescribeAlarmsCommand({ + AlarmNamePrefix: `TargetTracking-table/${cluster}`, + NextToken: nextToken, + }), + ); + nextToken = result.NextToken; + for (const alarm of result.MetricAlarms ?? []) { + if (alarm.AlarmName === undefined) continue; + alarms.push({ + name: alarm.AlarmName, + namespace: alarm.Namespace ?? "unknown-namespace", + tableName: alarm.Dimensions?.find((dimension) => dimension.Name === "TableName")?.Value, + }); + } + } while (nextToken !== undefined && nextToken !== ""); + return alarms; + }, + async deleteAlarms(alarmNames) { + if (alarmNames.length === 0) return; + const { sdk, client } = await lazyCloudWatch(); + await client.send(new sdk.DeleteAlarmsCommand({ AlarmNames: alarmNames })); + }, + async listLogGroups(prefix) { + const { sdk, client } = await lazyLogs(); + const groups: Array<{ name: string; size: number }> = []; + let nextToken: string | undefined; + do { + const result = await client.send( + new sdk.DescribeLogGroupsCommand({ logGroupNamePrefix: prefix, nextToken }), + ); + nextToken = result.nextToken; + for (const group of result.logGroups ?? []) { + if (group.logGroupName === undefined) continue; + groups.push({ name: group.logGroupName, size: group.storedBytes ?? 0 }); + } + } while (nextToken !== undefined && nextToken !== ""); + return groups; + }, + async deleteLogGroup(name) { + const { sdk, client } = await lazyLogs(); + await client.send(new sdk.DeleteLogGroupCommand({ logGroupName: name })); + }, + accountId: deps.accountId, + async bucketExists(name) { + const { sdk, client } = await lazyS3(); + try { + await client.send(new sdk.HeadBucketCommand({ Bucket: name })); + return true; + } catch (error) { + const status = (error as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode; + if (status === 404 || (error as { name?: string }).name === "NotFound") return false; + throw error; + } + }, + async deleteAllBucketObjectVersions(name) { + const { sdk, client } = await lazyS3(); + let keyMarker: string | undefined; + let versionIdMarker: string | undefined; + let deleted = 0; + do { + const page = await client.send( + new sdk.ListObjectVersionsCommand({ Bucket: name, KeyMarker: keyMarker, VersionIdMarker: versionIdMarker }), + ); + keyMarker = page.IsTruncated === true ? page.NextKeyMarker : undefined; + versionIdMarker = page.IsTruncated === true ? page.NextVersionIdMarker : undefined; + const objects = [...(page.Versions ?? []), ...(page.DeleteMarkers ?? [])].flatMap((entry) => + entry.Key === undefined ? [] : [{ Key: entry.Key, VersionId: entry.VersionId }], + ); + // DeleteObjects takes at most 1000 keys per call. + for (let index = 0; index < objects.length; index += 1000) { + await client.send( + new sdk.DeleteObjectsCommand({ + Bucket: name, + Delete: { Objects: objects.slice(index, index + 1000), Quiet: true }, + }), + ); + } + deleted += objects.length; + } while (keyMarker !== undefined || versionIdMarker !== undefined); + deps.out(`deleted ${deleted} object version(s) from bucket: ${name}`); + }, + async deleteBucket(name) { + const { sdk, client } = await lazyS3(); + await client.send(new sdk.DeleteBucketCommand({ Bucket: name })); + }, + prompt: async (message) => deps.prompt({ message, default: false }), + sleep: deps.sleep, + out: deps.out, + err: deps.err, + }; +} + +export function createLiveDeleteClusterDeps(deps: Deps): DeleteClusterDepsFactory { + return async (options) => deleteClusterDeps(deps, options); +} + +/** Create all runnable operator adapters with a profile-local AWS credential provider. */ +export function createLiveRemainingOperatorDeps(deps: Deps): RemainingOperatorCommandDeps { + return { + sso: async (options) => { + const config = await loadConfig(deps, options); + return { + config, + cognito: { + async getIdentityProviderByIdentifier(input) { + const { CognitoIdentityProviderClient, GetIdentityProviderByIdentifierCommand } = + await import("@aws-sdk/client-cognito-identity-provider"); + const result = await new CognitoIdentityProviderClient(await clientOptions(options)).send( + new GetIdentityProviderByIdentifierCommand(input as never), + ); + return result.IdentityProvider === undefined ? {} : { IdentityProvider: {} }; + }, + async createIdentityProvider(input) { + const { CognitoIdentityProviderClient, CreateIdentityProviderCommand } = + await import("@aws-sdk/client-cognito-identity-provider"); + await new CognitoIdentityProviderClient(await clientOptions(options)).send( + new CreateIdentityProviderCommand(input as never), + ); + }, + async updateIdentityProvider(input) { + const { CognitoIdentityProviderClient, UpdateIdentityProviderCommand } = + await import("@aws-sdk/client-cognito-identity-provider"); + await new CognitoIdentityProviderClient(await clientOptions(options)).send( + new UpdateIdentityProviderCommand(input as never), + ); + }, + async createUserPoolClient(input) { + const { CognitoIdentityProviderClient, CreateUserPoolClientCommand } = + await import("@aws-sdk/client-cognito-identity-provider"); + const result = await new CognitoIdentityProviderClient(await clientOptions(options)).send( + new CreateUserPoolClientCommand(input as never), + ); + return { + UserPoolClient: result.UserPoolClient === undefined ? undefined : { + ClientId: result.UserPoolClient.ClientId, + ClientSecret: result.UserPoolClient.ClientSecret, + }, + }; + }, + async updateUserPoolClient(input) { + const { CognitoIdentityProviderClient, UpdateUserPoolClientCommand } = + await import("@aws-sdk/client-cognito-identity-provider"); + const result = await new CognitoIdentityProviderClient(await clientOptions(options)).send( + new UpdateUserPoolClientCommand(input as never), + ); + return { + UserPoolClient: result.UserPoolClient === undefined ? undefined : { + ClientId: result.UserPoolClient.ClientId, + ClientSecret: result.UserPoolClient.ClientSecret, + }, + }; + }, + async listUsers(input) { + const { CognitoIdentityProviderClient, ListUsersCommand } = + await import("@aws-sdk/client-cognito-identity-provider"); + const result = await new CognitoIdentityProviderClient(await clientOptions(options)).send( + new ListUsersCommand(input), + ); + return { + Users: (result.Users ?? []).map((user) => ({ + Username: user.Username, + UserStatus: user.UserStatus, + Attributes: user.Attributes?.map((attribute) => ({ Name: attribute.Name, Value: attribute.Value })), + })), + PaginationToken: result.PaginationToken, + }; + }, + async adminLinkProviderForUser(input) { + const { AdminLinkProviderForUserCommand, CognitoIdentityProviderClient } = + await import("@aws-sdk/client-cognito-identity-provider"); + await new CognitoIdentityProviderClient(await clientOptions(options)).send( + new AdminLinkProviderForUserCommand(input as never), + ); + }, + }, + secrets: { + async describeSecret(input) { + const { DescribeSecretCommand, SecretsManagerClient } = await import("@aws-sdk/client-secrets-manager"); + const result = await new SecretsManagerClient(await clientOptions(options)).send( + new DescribeSecretCommand(input), + ); + return { ARN: result.ARN }; + }, + async createSecret(input) { + const { CreateSecretCommand, SecretsManagerClient } = await import("@aws-sdk/client-secrets-manager"); + const result = await new SecretsManagerClient(await clientOptions(options)).send( + new CreateSecretCommand(input as never), + ); + return { ARN: result.ARN }; + }, + async updateSecret(input) { + const { SecretsManagerClient, UpdateSecretCommand } = await import("@aws-sdk/client-secrets-manager"); + const result = await new SecretsManagerClient(await clientOptions(options)).send( + new UpdateSecretCommand(input as never), + ); + return { ARN: result.ARN }; + }, + }, + async setConfigEntry(key, value) { + const writer = await deps.configWriter({ + clusterName: options.clusterName, + awsRegion: requiredRegion(options), + awsProfile: options.awsProfile, + }); + await writer.setConfigEntry(key, value); + }, + sleep: deps.sleep, + out: deps.out, + }; + }, + directoryService: async (options) => ({ + secrets: { + async createSecret(input) { + const { CreateSecretCommand, SecretsManagerClient } = await import("@aws-sdk/client-secrets-manager"); + const result = await new SecretsManagerClient(await clientOptions(options)).send( + new CreateSecretCommand(input as never), + ); + return { ARN: result.ARN }; + }, + }, + out: deps.out, + }), + sharedStorage: async () => { + throw new GeneralException( + "shared-storage requires the EFS and FSx client adapters, which are not ported", + ); + }, + utils: async (options) => { + const config = options.clusterName === undefined ? new ClusterConfig([]) : await loadConfig(deps, options); + return { + config, + api: { + async getParametersByPath(input) { + const { GetParametersByPathCommand, SSMClient } = await import("@aws-sdk/client-ssm"); + const result = await new SSMClient(await clientOptions(options)).send( + new GetParametersByPathCommand({ ...input, Recursive: false }), + ); + return { + Parameters: result.Parameters?.map((parameter) => ({ Value: parameter.Value })), + NextToken: result.NextToken, + }; + }, + async describeVpcEndpointServices(input) { + const { DescribeVpcEndpointServicesCommand, EC2Client } = await import("@aws-sdk/client-ec2"); + const result = await new EC2Client(await clientOptions(options)).send( + new DescribeVpcEndpointServicesCommand(input as never), + ); + return { + ServiceDetails: result.ServiceDetails?.map((detail) => ({ + ServiceName: detail.ServiceName, + ServiceType: detail.ServiceType?.map((type) => ({ ServiceType: type.ServiceType })), + AvailabilityZones: detail.AvailabilityZones, + })), + }; + }, + async getManagedPrefixListEntries(input) { + const { EC2Client, GetManagedPrefixListEntriesCommand } = await import("@aws-sdk/client-ec2"); + const result = await new EC2Client(await clientOptions(options)).send( + new GetManagedPrefixListEntriesCommand(input), + ); + return { + Entries: result.Entries?.map((entry) => ({ Cidr: entry.Cidr, Description: entry.Description })), + NextToken: result.NextToken, + }; + }, + async describeManagedPrefixLists(input) { + const { DescribeManagedPrefixListsCommand, EC2Client } = await import("@aws-sdk/client-ec2"); + const result = await new EC2Client(await clientOptions(options)).send( + new DescribeManagedPrefixListsCommand(input), + ); + return { PrefixLists: result.PrefixLists?.map((list) => ({ Version: list.Version })) }; + }, + async modifyManagedPrefixList(input) { + const { EC2Client, ModifyManagedPrefixListCommand } = await import("@aws-sdk/client-ec2"); + await new EC2Client(await clientOptions(options)).send(new ModifyManagedPrefixListCommand(input)); + }, + }, + async dnsSuffix() { + return requiredRegion(options).startsWith("cn-") ? "amazonaws.com.cn" : "amazonaws.com"; + }, + async syncGlobalSettings(input) { + const clusterName = options.clusterName; + if (clusterName === undefined || clusterName.trim() === "") { + throw new GeneralException("cluster name is required for global settings update"); + } + const writer = await deps.configWriter({ + clusterName, + awsRegion: requiredRegion(options), + awsProfile: options.awsProfile, + }); + await writer.deleteConfigEntries(input.deletePrefix); + await writer.syncClusterSettingsInDb(input.entries, true); + }, + async exportConfig(input) { + const [settings, modules] = await Promise.all([ + scanAll(deps, `${input.clusterName}.cluster-settings`), + scanAll(deps, `${input.clusterName}.modules`), + ]); + const config = yaml.dump(settings, { noRefs: true, sortKeys: false }); + const moduleConfig = yaml.dump(modules, { noRefs: true, sortKeys: false }); + await import("node:fs/promises").then(({ mkdir, writeFile }) => + mkdir(input.configDir, { recursive: true }).then(async () => { + await writeFile(join(input.configDir, "config.yml"), config); + await writeFile(join(input.configDir, "modules.yml"), moduleConfig); + }), + ); + }, + now: () => new Date(), + prompt: async (message) => (await deps.prompt({ message, default: false })) === true, + out: deps.out, + }; + }, + support: async (options) => ({ + async databaseConfig() { + const clusterName = options.clusterName; + if (clusterName === undefined) throw new GeneralException("cluster name is required for support database export"); + const [settings, modules] = await Promise.all([ + scanAll(deps, `${clusterName}.cluster-settings`), + scanAll(deps, `${clusterName}.modules`), + ]); + return { + configYaml: yaml.dump(settings, { noRefs: true, sortKeys: false }), + modulesYaml: yaml.dump(modules, { noRefs: true, sortKeys: false }), + }; + }, + now: () => new Date(), + archive: archiveDirectory, + out: deps.out, + }), + integrationTests: async (options) => ({ + config: await loadConfig(deps, options), + casesForModule() { + throw new GeneralException("integration test case registry is not ported"); + }, + out: deps.out, + err: deps.err, + }), + }; +} diff --git a/source/idea/ideactl/src/cli/main.ts b/source/idea/ideactl/src/cli/main.ts new file mode 100644 index 00000000..ad73cef5 --- /dev/null +++ b/source/idea/ideactl/src/cli/main.ts @@ -0,0 +1,602 @@ +#!/usr/bin/env node +/** + * `ideactl`: the administrator CLI. + * + * Port of `app_main.py`'s command tree and of `main_wrapper`'s exit behaviour. The commands + * themselves live under `commands/`; this file owns the program, the exit codes, `quick-setup` and + * the live `Deps` (the only place in the CLI that constructs an AWS client). + */ + +import { randomUUID } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { createInterface } from 'node:readline/promises'; +import { pathToFileURL } from 'node:url'; + +import { Command } from 'commander'; + +import { ClusterConfig, ConfigKeyNotFound, GeneralException, type ScanPage } from '../config/cluster-config.ts'; +import { loadValuesFile, resourcePath } from '../config/values.ts'; +import { ideaVersion } from '../version.ts'; +import { + ExitWithCode, + liveSpawn, + type ConfigWriter, + type ConfigWriterOptions, + type Deps, + type PromptChoice, +} from './cdk-invoker.ts'; +import { registerCdkCommands } from './commands/cdk.ts'; +import { + configGenerate, + configUpdate, + pyStr, + registerConfigCommands, + renderTable, + scanSettings, +} from './commands/config.ts'; +import { registerDeleteClusterCommands } from './commands/delete-cluster.ts'; +import { registerDeployCommands, runBootstrap, runDeploy } from './commands/deploy.ts'; +import { registerReplaceCommands } from './commands/replace.ts'; +import { registerMigrateCommands } from './commands/migrate.ts'; +import { checkClusterStatus, connectionInfo, liveHttpStatus, modulesTable, registerStatusCommands } from './commands/status.ts'; +import { createLiveUpgradeDeps, liveEcsAccountSettings, registerUpgradeCommands } from './commands/upgrade.ts'; +import { registerRemainingOperatorCommands } from './commands/utils.ts'; +import { + AwsProfileCredentialsError, + awsClientOptions, + formatAwsIdentity, +} from "./aws-client-options.ts"; +import { DeploymentHelper } from './deployment-helper.ts'; +import { createLiveMigrateDeps } from "./live-migrate-adapters.ts"; +import { createLiveDeleteClusterDeps, createLiveRemainingOperatorDeps } from "./live-operator-adapters.ts"; + +const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); +let actionProfile: string | undefined; +const initialAwsProfile = process.env.AWS_PROFILE; +const initialAwsDefaultRegion = process.env.AWS_DEFAULT_REGION; + +function selectActionProfile(profile: string | undefined): void { + actionProfile = profile === undefined || profile.trim() === "" ? undefined : profile; + if (actionProfile === undefined) { + if (initialAwsProfile === undefined) delete process.env.AWS_PROFILE; + else process.env.AWS_PROFILE = initialAwsProfile; + } else { + process.env.AWS_PROFILE = actionProfile; + } +} + +/** + * Bind the region the command selected, the same way the profile is bound. + * + * Every client the live dependencies build resolves its region from the + * environment, so a `--aws-region` that is read only for the identity banner + * leaves the table scan, the change-set calls and the values-file reads with no + * region at all. + */ +function selectActionRegion(awsRegion: string | undefined): void { + const selected = awsRegion === undefined || awsRegion.trim() === "" ? undefined : awsRegion; + if (selected === undefined) { + if (initialAwsDefaultRegion === undefined) delete process.env.AWS_DEFAULT_REGION; + else process.env.AWS_DEFAULT_REGION = initialAwsDefaultRegion; + } else { + process.env.AWS_DEFAULT_REGION = selected; + } +} + +/** The region every live client uses, after the pre-action hook has bound it. */ +function environmentRegion(): string { + return process.env.AWS_DEFAULT_REGION ?? process.env.AWS_REGION ?? ''; +} + +/** Terminal red, as `click.secho(fg='red', bold=True)` prints it. */ +function red(message: string): string { + return process.stderr.isTTY === true ? `${message}` : message; +} + +// --------------------------------------------------------------------------------------------- +// live deps +// --------------------------------------------------------------------------------------------- + +/** + * The real effects. Every AWS client is imported lazily so a command that needs no credentials + * (`about`, `quick-setup-help`, `--help`) loads none of them. + */ +export function liveDeps(): Deps { + const region = environmentRegion; + const callerIdentity = async (options: { + awsRegion: string; + awsProfile?: string; + }): Promise<{ account: string; arn: string }> => { + const { STSClient, GetCallerIdentityCommand } = await import("@aws-sdk/client-sts"); + const result = await new STSClient( + await awsClientOptions(options.awsRegion, options.awsProfile), + ).send(new GetCallerIdentityCommand({})); + if (result.Account === undefined || result.Arn === undefined) { + const profile = options.awsProfile; + const resolvedRegion = options.awsRegion; + const where = [ + profile === undefined || profile.trim() === "" ? undefined : `profile ${profile}`, + resolvedRegion.trim() === "" ? undefined : `region ${resolvedRegion}`, + ] + .filter((part): part is string => part !== undefined) + .join(" in "); + throw new GeneralException( + where === "" + ? "sts:GetCallerIdentity returned no account. The credentials did not resolve to an account. Pass --aws-profile and --aws-region, then retry." + : `sts:GetCallerIdentity returned no account for ${where}. The credentials did not resolve to an account. Pass --aws-profile and --aws-region, then retry.`, + ); + } + return { account: result.Account, arn: result.Arn }; + }; + + const scan = async (input: { TableName: string; ExclusiveStartKey?: Record }): Promise => { + const { DynamoDBClient } = await import('@aws-sdk/client-dynamodb'); + const { DynamoDBDocumentClient, ScanCommand } = await import('@aws-sdk/lib-dynamodb'); + const doc = DynamoDBDocumentClient.from(new DynamoDBClient(await awsClientOptions(region()))); + return doc.send(new ScanCommand(input)); + }; + + return { + spawn: liveSpawn, + scan, + // `deploy` and `quick-setup` run the container pre-flight as soon as the deployment includes + // the container module, so the reader it needs belongs in the live dependency set rather than + // only in the upgrade command group's. + ecsAccountSettings: liveEcsAccountSettings(), + cfn: { + async describeChangeSet(input) { + const { CloudFormationClient, DescribeChangeSetCommand } = await import('@aws-sdk/client-cloudformation'); + const client = new CloudFormationClient(await awsClientOptions(region())); + return client.send(new DescribeChangeSetCommand(input)); + }, + async executeChangeSet(input) { + const { CloudFormationClient, ExecuteChangeSetCommand } = await import('@aws-sdk/client-cloudformation'); + const client = new CloudFormationClient(await awsClientOptions(region())); + await client.send(new ExecuteChangeSetCommand(input)); + }, + async describeStack(stackName) { + const { CloudFormationClient, DescribeStacksCommand } = await import('@aws-sdk/client-cloudformation'); + const client = new CloudFormationClient(await awsClientOptions(region())); + const result = await client.send(new DescribeStacksCommand({ StackName: stackName })); + const stack = result.Stacks?.[0]; + if (stack === undefined) throw new GeneralException(`stack not found: ${stackName}`); + return stack; + }, + }, + s3: { + async putObject(input) { + const { S3Client, PutObjectCommand } = await import('@aws-sdk/client-s3'); + await new S3Client(await awsClientOptions(region())).send(new PutObjectCommand(input)); + }, + async getObject(input) { + const { S3Client, GetObjectCommand } = await import('@aws-sdk/client-s3'); + const result = await new S3Client(await awsClientOptions(region())).send(new GetObjectCommand(input)); + return (await result.Body?.transformToString()) ?? ''; + }, + }, + async configWriter(options: ConfigWriterOptions): Promise { + const { DynamoDBClient } = await import('@aws-sdk/client-dynamodb'); + const { ClusterConfigDb } = await import('../config/cluster-config-db.ts'); + return ClusterConfigDb.open({ + clusterName: options.clusterName, + awsRegion: options.awsRegion, + client: new DynamoDBClient(await awsClientOptions(options.awsRegion, options.awsProfile)), + dynamodbKmsKeyId: options.dynamodbKmsKeyId, + createDatabase: options.createDatabase, + logger: (message) => console.log(message), + }); + }, + callerIdentity, + async accountId() { + return (await callerIdentity({ awsRegion: region(), awsProfile: actionProfile })).account; + }, + httpStatus: liveHttpStatus, + sleep, + now: () => Date.now(), + uuid: () => randomUUID(), + out: (line) => console.log(line), + err: (line) => console.error(red(line)), + async prompt(choice: PromptChoice) { + const rl = createInterface({ input: process.stdin, output: process.stdout }); + try { + const suffix = choice.choices === undefined ? ' [Y/n] ' : ` [${choice.choices.join('/')}] `; + const answer = (await rl.question(`${choice.message}${suffix}`)).trim(); + if (choice.choices !== undefined) { + const matched = choice.choices.find((option) => option.toLowerCase() === answer.toLowerCase()); + return matched ?? String(choice.default ?? choice.choices[0]); + } + if (answer === '') return choice.default !== false; + return ['y', 'yes'].includes(answer.toLowerCase()); + } finally { + rl.close(); + } + }, + }; +} + +// --------------------------------------------------------------------------------------------- +// quick-setup +// --------------------------------------------------------------------------------------------- + +export interface QuickSetupOptions { + valuesFile?: string; + existingResources?: boolean; + terminationProtection?: string; + deploymentId?: string; + optimizeDeployment?: boolean; + force?: boolean; + skipConfig?: boolean; + rollback?: boolean; + moduleSet: string; + allowReplacement?: string[]; +} + +/** `quick_setup` (`app_main.py:1644-1795`), step for step. */ +export async function quickSetup(deps: Deps, options: QuickSetupOptions): Promise { + let values: Record; + if (options.skipConfig === true) { + if (options.valuesFile === undefined) { + deps.err('--values-file is required when --skip-config flag is provided.'); + throw new ExitWithCode(1); + } + values = loadValuesFile(options.valuesFile); + } else { + deps.out(`ideactl ${ideaVersion()}`); + values = await configGenerate(deps, { + valuesFile: options.valuesFile, + existingResources: options.existingResources, + force: options.force, + }); + } + + const clusterName = String(values['cluster_name'] ?? ''); + const awsRegion = String(values['aws_region'] ?? ''); + const awsProfile = values['aws_profile'] === undefined ? undefined : String(values['aws_profile']); + selectActionProfile(awsProfile); + // quick-setup takes its region from the values file rather than from an option, so the hook has + // nothing to bind and the clients below would have no region. + selectActionRegion(awsRegion); + if (deps.callerIdentity !== undefined) { + const identity = await deps.callerIdentity({ awsRegion, awsProfile }); + deps.out(formatAwsIdentity(identity, awsProfile)); + } + + if (options.skipConfig !== true) { + await configUpdate(deps, { + clusterName, + awsRegion, + awsProfile, + moduleSet: options.moduleSet, + force: options.force, + }); + } + + const settings = await scanSettings(deps, clusterName); + deps.out( + renderTable(['Key', 'Value', 'Version'], settings.map((row) => [row.key, pyStr(row.value), String(row.version ?? 0)])), + ); + + let config = await ClusterConfig.fromDynamoDb(clusterName, awsRegion, { + moduleSet: options.moduleSet, + scan: deps.scan, + }); + deps.out(modulesTable(config.modules())); + + if (options.force !== true) { + const confirm = await deps.prompt({ + message: 'Are you sure you want to deploy above IDEA modules with applicable configuration settings?', + default: true, + }); + if (confirm !== true && confirm !== 'Yes') { + deps.out('Deployment aborted!'); + throw new ExitWithCode(0); + } + } + + await runBootstrap(deps, { + clusterName, + awsRegion, + awsProfile, + terminationProtection: options.terminationProtection, + moduleSet: options.moduleSet, + }); + + const helper = await DeploymentHelper.open({ + clusterName, + awsRegion, + moduleSet: options.moduleSet, + awsProfile, + allModules: true, + upgrade: false, + forceBuildBootstrap: true, + optimizeDeployment: options.optimizeDeployment === true, + deploymentId: options.deploymentId, + deps, + }); + const moduleIds = helper.getDeploymentOrder(); + if (moduleIds.length === 0) { + deps.out('all modules are already deployed. skipping deployment.'); + } else { + const order = options.optimizeDeployment === true ? helper.getOptimizedDeploymentOrder() : moduleIds; + deps.out(`deploying modules: ${JSON.stringify(order)}`); + await runDeploy(deps, moduleIds, { + clusterName, + awsRegion, + awsProfile, + terminationProtection: options.terminationProtection, + deploymentId: options.deploymentId, + rollback: options.rollback, + optimizeDeployment: options.optimizeDeployment, + moduleSet: options.moduleSet, + allowReplacement: options.allowReplacement, + }); + } + + await checkClusterStatus(deps, { + clusterName, + awsRegion, + awsProfile, + wait: true, + waitTimeout: 30 * 60, + moduleSet: options.moduleSet, + }); + + config = await ClusterConfig.fromDynamoDb(clusterName, awsRegion, { + moduleSet: options.moduleSet, + scan: deps.scan, + }); + deps.out(modulesTable(config.modules())); + deps.out('--- Cluster Connection Info ---'); + for (const entry of connectionInfo(config, awsRegion)) deps.out(`${entry.key}: ${entry.value}`); +} + +// --------------------------------------------------------------------------------------------- +// program +// --------------------------------------------------------------------------------------------- + +export function buildProgram(deps: Deps): Command { + // The shell wrapper this tool replaces defaults the security scan off for deploys, and the +// suppression metadata is emitted whether or not the aspect runs. Keep the same default so +// running from the image behaves like running from the wrapper. +process.env.IDEA_ADMIN_ENABLE_CDK_NAG_SCAN ??= 'false'; + +const program = new Command('ideactl') + .description('IDEA cluster administration') + .version(ideaVersion()) + .helpOption('-h, --help', 'display help for command') + .showHelpAfterError() + // Set before the subcommands are registered so every one of them inherits it: commander + // copies settings at `.command()` time, and a subcommand that still calls `process.exit` + // would take the process down before `run` could map the exit code. + .exitOverride(); + + program.hook("preAction", async (_command, actionCommand) => { + const options = actionCommand.opts(); + const selected = typeof options.awsProfile === "string" ? options.awsProfile : undefined; + selectActionProfile(selected); + + const optionRegion = options.awsRegion; + const positionalRegion = + actionCommand.name() === "check-aws-services" ? actionCommand.args[0] : undefined; + const awsRegion = + typeof optionRegion === "string" + ? optionRegion + : typeof positionalRegion === "string" + ? positionalRegion + : undefined; + selectActionRegion(awsRegion); + if (awsRegion !== undefined && deps.callerIdentity !== undefined) { + const identity = await deps.callerIdentity({ awsRegion, awsProfile: selected }); + deps.out(formatAwsIdentity(identity, selected)); + } + }); + + program + .command('about') + .description('print the release version') + .option('--no-banner', 'print the version without the banner') + .action(() => { + deps.out(`ideactl ${ideaVersion()}`); + }); + + program + .command('quick-setup-help') + .description('display quick-setup help') + .action(() => { + deps.out(readFileSync(resourcePath('config/values.yml'), 'utf-8')); + }); + + program + .command('quick-setup') + .description('Install a new cluster') + .option('--values-file ', 'path to values.yml file') + .option('--existing-resources', 'Install IDEA using existing resources') + .option('--termination-protection ', 'enable/disable termination protection for all stacks', 'true') + .option('--deployment-id ', 'Deployment Id') + .option('--optimize-deployment', 'Deploy applicable stacks in parallel.') + .option('--force', 'Skip all confirmation prompts') + .option( + '--skip-config', + 'Skip config generation and update steps. --values-file is required when this flag is provided.', + ) + .option('--rollback', 'Rollback stack to stable state on failure. Default.', true) + .option('--no-rollback', 'Do not roll back on failure, to iterate more rapidly.') + .option('--module-set ', 'Name of the ModuleSet. Default: default', 'default') + .option( + '--allow-replacement ', + 'Accept a change-set entry the deploy guard would refuse, by logical ID. Repeatable.', + (value: string, previous: string[] = []) => [...previous, value], + ) + .action(async (options: QuickSetupOptions) => { + await quickSetup(deps, options); + }); + + registerConfigCommands(program, deps); + registerCdkCommands(program, deps); + registerDeployCommands(program, deps); + registerReplaceCommands(program, deps); + registerStatusCommands(program, deps); + registerUpgradeCommands(program, createLiveUpgradeDeps(deps)); + registerMigrateCommands(program, createLiveMigrateDeps(deps, environmentRegion)); + registerDeleteClusterCommands(program, createLiveDeleteClusterDeps(deps)); + registerRemainingOperatorCommands(program, createLiveRemainingOperatorDeps(deps)); + + return program; +} + +/** + * CLI exit behaviour: with no arguments print help and exit 0. Operator errors print one red + * line and exit 1. A stack is printed only when IDEA_DEBUG=1. + */ +export async function run(argv: string[] = process.argv.slice(2), deps: Deps = liveDeps()): Promise { + const program = buildProgram(deps); + try { + await program.parseAsync(argv.length === 0 ? ['--help'] : argv, { from: 'user' }); + return 0; + } catch (error) { + if (error instanceof ExitWithCode) { + if (error.message !== '') deps.err(error.message); + return error.code; + } + if (isCommanderExit(error)) return (error as { exitCode: number }).exitCode; + deps.err(operatorMessage(error, argv)); + if (process.env.IDEA_DEBUG === '1') console.error(error); + return 1; + } +} + +function isCommanderExit(error: unknown): boolean { + return (error as { code?: string })?.code?.startsWith('commander.') === true; +} + +/** One operator-facing sentence for a caught failure. */ +function operatorMessage(error: unknown, argv: string[]): string { + if (isClusterConfigNotInitialized(error)) { + return formatUninitialisedCluster(error as Error, argv); + } + if (isResourceNotFound(error)) { + return formatMissingTables(argv); + } + if (error instanceof ConfigKeyNotFound) { + return formatConfigKeyNotFound(error, argv); + } + if (error instanceof AwsProfileCredentialsError) { + return error.message; + } + if (isCredentialsError(error)) { + return formatCredentialsError(error as Error, argv); + } + if (error instanceof Error) return error.message; + return String(error); +} + +function isClusterConfigNotInitialized(error: unknown): boolean { + return ( + error instanceof Error && + (error.name === 'ClusterConfigDbError' || + error.message.startsWith('Configuration tables not found for cluster')) + ); +} + +function isResourceNotFound(error: unknown): boolean { + return (error as { name?: string })?.name === 'ResourceNotFoundException'; +} + +function isCredentialsError(error: unknown): boolean { + const name = (error as { name?: string })?.name ?? ''; + const message = (error as Error)?.message ?? ''; + return ( + name === 'CredentialsProviderError' || + /Profile .* (could not be found|not found)/i.test(message) || + /Could not resolve credentials using profile:/i.test(message) + ); +} + +function flagValue(argv: string[], flag: string): string | undefined { + const index = argv.indexOf(flag); + if (index === -1) return undefined; + const value = argv[index + 1]; + if (value === undefined || value.startsWith('-')) return undefined; + return value; +} + +function formatUninitialisedCluster(error: Error, argv: string[]): string { + if (error.message.includes('Create them with ideactl config update')) return error.message; + const cluster = + error.message.match(/cluster:\s*(\S+)/)?.[1] ?? + flagValue(argv, '--cluster-name') ?? + 'the cluster'; + const region = + flagValue(argv, '--aws-region') ?? process.env.AWS_DEFAULT_REGION ?? process.env.AWS_REGION ?? ''; + if (region === '') { + return ( + `Configuration tables not found for cluster ${cluster}. Create them with ideactl config update ` + + `--cluster-name ${cluster} --aws-region , or confirm the cluster was installed in this account and region.` + ); + } + return ( + `Configuration tables not found for cluster ${cluster} in ${region}. Create them with ideactl config update ` + + `--cluster-name ${cluster} --aws-region ${region}, or confirm the cluster was installed in this account and region.` + ); +} + +function formatMissingTables(argv: string[]): string { + const cluster = flagValue(argv, '--cluster-name') ?? ''; + const region = + flagValue(argv, '--aws-region') ?? process.env.AWS_DEFAULT_REGION ?? process.env.AWS_REGION ?? ''; + return ( + `No configuration tables for cluster ${cluster} in ${region} (looked for ${cluster}.modules and ${cluster}.cluster-settings). ` + + `Install with ideactl quick-setup, or run ideactl config update --cluster-name ${cluster} --aws-region ${region}. ` + + 'If the cluster already exists, check --aws-region and --aws-profile.' + ); +} + +function formatConfigKeyNotFound(error: ConfigKeyNotFound, argv: string[]): string { + const key = error.message.match(/, key:\s*(.+)$/)?.[1] ?? error.message; + const cluster = flagValue(argv, '--cluster-name') ?? ''; + const region = flagValue(argv, '--aws-region') ?? ''; + return ( + `Configuration key ${key} is missing for this cluster. Show nearby keys with ideactl config show ` + + `--cluster-name ${cluster} --aws-region ${region}, or set it with ideactl config set.` + ); +} + +function formatCredentialsError(error: Error, argv: string[]): string { + const profileFromSdk = error.message.match(/profile:\s*\[([^\]]+)\]/i)?.[1]; + const profile = + profileFromSdk ?? actionProfile ?? flagValue(argv, '--aws-profile') ?? process.env.AWS_PROFILE; + if ( + profile !== undefined && + profile !== '' && + (/Could not resolve credentials using profile:/i.test(error.message) || + /Profile .* (could not be found|not found)/i.test(error.message)) + ) { + return ( + `AWS profile ${profile} was not found in the shared config/credentials files. ` + + 'Create the profile, or pass an existing name with --aws-profile. AWS_PROFILE is also read.' + ); + } + const region = + flagValue(argv, '--aws-region') ?? process.env.AWS_DEFAULT_REGION ?? process.env.AWS_REGION ?? ''; + if (region === '') { + return 'No AWS credentials were loaded. Export keys, start a federated session, or pass --aws-profile. Then retry.'; + } + return ( + `No AWS credentials were loaded for region ${region}. Export keys, start a federated session, or pass --aws-profile. Then retry.` + ); +} + +const entryPoint = process.argv[1]; +const isMainModule = entryPoint !== undefined && import.meta.url === pathToFileURL(entryPoint).href; + +if (isMainModule) { + run().then( + (code) => { + if (code !== 0) process.exitCode = code; + }, + (error: unknown) => { + console.error(error); + process.exitCode = 1; + }, + ); +} diff --git a/source/idea/ideactl/src/cli/preflight.ts b/source/idea/ideactl/src/cli/preflight.ts new file mode 100644 index 00000000..8a547a30 --- /dev/null +++ b/source/idea/ideactl/src/cli/preflight.ts @@ -0,0 +1,289 @@ +/** + * Collect and run read-only checks before a command can mutate a cluster. + */ + +/** Severity displayed with each pre-flight result. */ +export type PreflightSeverity = "error" | "warning"; + +/** Cluster and command values shared by every pre-flight check. */ +export interface PreflightContext { + command: string; + account: string; + region: string; + cluster: string; + profile?: string; +} + +/** The result returned by one pre-flight predicate. */ +export type PreflightOutcome = + | { passed: true } + | { passed: false; message: string }; + +/** One named check and the commands that require it. */ +export interface PreflightCheck { + name: string; + description: string; + severity: PreflightSeverity; + commands: readonly string[]; + run(context: Readonly): Promise; +} + +/** One completed check in a grouped report. */ +export interface PreflightResult { + name: string; + description: string; + severity: PreflightSeverity; + passed: boolean; + message?: string; +} + +/** All checks selected for one command invocation. */ +export interface PreflightReport { + context: Readonly; + results: readonly PreflightResult[]; + passed: boolean; +} + +/** Read-only predicate used by checks with a boolean result. */ +export type BooleanPreflightProbe = ( + context: Readonly, +) => Promise; + +/** Target-template comparison evidence supplied by the parity adapter. */ +export interface TemplateComparisonEvidence { + matches: boolean; + remedyCommand: string; +} + +/** Drift evidence supplied by the configuration preview adapter. */ +export interface ConfigurationDriftEvidence { + reportHash: string; + lossKeys: readonly string[]; + acceptedReportHash?: string; +} + +const MIGRATION_COMMANDS = ["upgrade-cluster", "migrate"] as const; +const ECS_COMMANDS = ["deploy", ...MIGRATION_COMMANDS] as const; + +/** Reject empty identifiers before they enter operator-facing output. */ +function requireText(value: string, label: string): void { + if (value.trim() === "") { + throw new TypeError(`${label} must not be empty`); + } +} + +/** Validate a check when it is registered so runner failures stay actionable. */ +function validateCheck(check: PreflightCheck): void { + requireText(check.name, "Pre-flight check name"); + requireText(check.description, `Description for ${check.name}`); + if (check.severity !== "error" && check.severity !== "warning") { + throw new TypeError(`Invalid severity for pre-flight check ${check.name}`); + } + if (check.commands.length === 0) { + throw new TypeError(`Pre-flight check ${check.name} must name at least one command`); + } + for (const command of check.commands) { + requireText(command, `Command for ${check.name}`); + } +} + +/** Validate invocation values before checks use them in probes or messages. */ +function validateContext(context: Readonly): void { + requireText(context.command, "Pre-flight command"); + requireText(context.account, "Account"); + requireText(context.region, "Region"); + requireText(context.cluster, "Cluster"); + if (context.profile !== undefined) { + requireText(context.profile, "Profile"); + } +} + +/** Convert an unexpected probe failure into an operator-facing refusal. */ +function probeFailure(check: PreflightCheck, context: Readonly, error: unknown): PreflightResult { + const detail = error instanceof Error ? error.message : String(error); + return { + name: check.name, + description: check.description, + severity: check.severity, + passed: false, + message: `Could not evaluate ${check.name} for account ${context.account} in region ${context.region}: ${detail}. Resolve the read failure, then rerun ${context.command}.`, + }; +} + +/** + * Registry for checks contributed by command and feature owners. + * + * Registration order is preserved in reports. + */ +export class PreflightRegistry { + readonly #checks = new Map(); + + /** Add one uniquely named check. */ + register(check: PreflightCheck): this { + validateCheck(check); + if (this.#checks.has(check.name)) { + throw new TypeError(`Duplicate pre-flight check name: ${check.name}`); + } + this.#checks.set(check.name, check); + return this; + } + + /** Return checks required by the named command. */ + forCommand(command: string): readonly PreflightCheck[] { + requireText(command, "Pre-flight command"); + return [...this.#checks.values()].filter((check) => check.commands.includes(command)); + } +} + +/** + * Run every relevant check and retain every failure in one report. + */ +export async function runPreflight( + registry: PreflightRegistry, + context: Readonly, +): Promise { + validateContext(context); + const checks = registry.forCommand(context.command); + + // All probes are read-only, so one failed predicate must not suppress its peers. + const results = await Promise.all( + checks.map(async (check): Promise => { + try { + const outcome = await check.run(context); + if (outcome.passed) { + return { + name: check.name, + description: check.description, + severity: check.severity, + passed: true, + }; + } + requireText(outcome.message, `Failure message for ${check.name}`); + return { + name: check.name, + description: check.description, + severity: check.severity, + passed: false, + message: outcome.message, + }; + } catch (error) { + return probeFailure(check, context, error); + } + }), + ); + + return { + context: { ...context }, + results, + passed: results.every((result) => result.passed), + }; +} + +/** Render a compact grouped report suitable for terminal output or a journal. */ +export function renderPreflightReport(report: Readonly): string { + const failed = report.results.filter((result) => !result.passed).length; + const lines = [ + `Pre-flight checks for ${report.context.command} on ${report.context.cluster}, account ${report.context.account}, region ${report.context.region}`, + ]; + + for (const result of report.results) { + lines.push(`${result.passed ? "PASS" : "FAIL"} [${result.severity}] ${result.name}: ${result.description}`); + if (result.message !== undefined) { + lines.push(...result.message.split("\n").map((line) => ` ${line}`)); + } + } + lines.push(`${report.passed ? "PASS" : "FAIL"}: ${report.results.length - failed} passed, ${failed} failed`); + return lines.join("\n"); +} + +/** Build the exact one-time command that enables task network interface trunking. */ +export function awsvpcTrunkingRemedy(context: Readonly): string { + return [ + "aws", + "ecs", + "put-account-setting-default", + "--name", + "awsvpcTrunking", + "--value", + "enabled", + "--region", + context.region, + ...(context.profile === undefined ? [] : ["--profile", context.profile]), + ].join(" "); +} + +/** Create the account-wide task network interface trunking prerequisite. */ +export function createAwsvpcTrunkingCheck( + probe: BooleanPreflightProbe, + commands: readonly string[] = ECS_COMMANDS, +): PreflightCheck { + return { + name: "awsvpc-trunking", + description: "Account-wide task network interface trunking is enabled", + severity: "error", + commands, + async run(context): Promise { + if (await probe(context)) { + return { passed: true }; + } + return { + passed: false, + message: `Task network interface trunking is not enabled for account ${context.account} in region ${context.region}. Run this command once, then rerun ${context.command}: ${awsvpcTrunkingRemedy(context)}`, + }; + }, + }; +} + +/** Create the target-cluster template comparison prerequisite. */ +export function createTemplateComparisonCheck( + probe: (context: Readonly) => Promise, + commands: readonly string[] = MIGRATION_COMMANDS, +): PreflightCheck { + return { + name: "template-comparison", + description: "Target release templates match the current cluster", + severity: "error", + commands, + async run(context): Promise { + const evidence = await probe(context); + if (evidence.matches) { + return { passed: true }; + } + requireText(evidence.remedyCommand, "Template comparison remedy command"); + return { + passed: false, + message: `Template comparison is not green for cluster ${context.cluster}, account ${context.account}, region ${context.region}. Run ${evidence.remedyCommand}, resolve every reported difference, then rerun ${context.command}.`, + }; + }, + }; +} + +/** Create the check that blocks only configuration values the run would lose. */ +export function createConfigurationDriftCheck( + probe: (context: Readonly) => Promise, + commands: readonly string[] = MIGRATION_COMMANDS, +): PreflightCheck { + return { + name: "configuration-drift", + description: "Configuration writes will not lose unacknowledged operator edits", + severity: "error", + commands, + async run(context): Promise { + const evidence = await probe(context); + for (const key of evidence.lossKeys) { + requireText(key, "Configuration drift key"); + } + if (evidence.lossKeys.length === 0) { + return { passed: true }; + } + requireText(evidence.reportHash, "Configuration drift report hash"); + if (evidence.acceptedReportHash === evidence.reportHash) { + return { passed: true }; + } + return { + passed: false, + message: `Configuration changes would lose operator edits for account ${context.account} in region ${context.region}: ${evidence.lossKeys.join(", ")}. Preserve those typed values, or rerun ${context.command} with --accept-drift ${evidence.reportHash}.`, + }; + }, + }; +} diff --git a/source/idea/ideactl/src/cli/prompts.ts b/source/idea/ideactl/src/cli/prompts.ts new file mode 100644 index 00000000..c9dcd918 --- /dev/null +++ b/source/idea/ideactl/src/cli/prompts.ts @@ -0,0 +1,121 @@ +/** + * Terminal question types and prompt drivers used by installer flows. + * + * The driver boundary keeps question collection independent from a real terminal. Callers can + * supply scripted answers in tests or use the readline-backed driver in the command line tool. + */ + +import { createInterface } from "node:readline/promises"; +import { stdin, stdout } from "node:process"; + +/** One option displayed by a select or checkbox question. */ +export interface InstallerChoice { + title: string; + value: string; + disabled?: boolean; +} + +/** Question shapes declared by the installer parameter YAML. */ +export type InstallerPromptType = "text" | "select" | "checkbox" | "confirm"; + +/** A normalised parameter declaration ready for a terminal driver. */ +export interface InstallerQuestion { + name: string; + title: string; + description: string; + promptType: InstallerPromptType; + multiple: boolean; + defaultValue?: unknown; + choices: InstallerChoice[]; + helpText?: string; +} + +/** Collects one answer and optionally displays validation feedback. */ +export interface PromptDriver { + ask(question: InstallerQuestion): Promise; + report(message: string): void; +} + +/** Parses a comma-separated terminal selection as choice values or one-based choice indexes. */ +function parseSelection(answer: string, choices: readonly InstallerChoice[], multiple: boolean): string | string[] { + const tokens = multiple ? answer.split(",") : [answer]; + const values = tokens + .map((token) => token.trim()) + .filter((token) => token !== "") + .map((token) => { + const index = Number.parseInt(token, 10); + if (/^\d+$/.test(token) && index >= 1 && index <= choices.length) { + return choices[index - 1]?.value ?? token; + } + return token; + }); + return multiple ? values : (values[0] ?? ""); +} + +/** Builds a compact terminal suffix without putting choice labels into the answer format. */ +function suffix(question: InstallerQuestion): string { + if (question.promptType === "confirm") { + return question.defaultValue === false ? " [y/N] " : " [Y/n] "; + } + if (question.choices.length > 0) return question.multiple ? " [comma-separated values] " : " [value] "; + return question.defaultValue === undefined ? " " : ` [${String(question.defaultValue)}] `; +} + +/** + * Uses stdin/stdout directly. Answers are intentionally returned untyped because validation and + * type conversion are owned by the parameter runner. + */ +export class TerminalPromptDriver implements PromptDriver { + async ask(question: InstallerQuestion): Promise { + if (question.description !== "") stdout.write(`${question.description}\n`); + if (question.helpText !== undefined && question.helpText !== "") stdout.write(`${question.helpText}\n`); + if (question.choices.length > 0) { + for (const [index, choice] of question.choices.entries()) { + const unavailable = choice.disabled === true ? " (unavailable)" : ""; + stdout.write(`${index + 1}. ${choice.title === "" ? choice.value : choice.title}${unavailable}\n`); + } + } + + const reader = createInterface({ input: stdin, output: stdout }); + try { + const answer = (await reader.question(`${question.title}${suffix(question)}`)).trim(); + if (answer === "") return question.defaultValue; + if (question.promptType === "confirm") return answer; + if (question.promptType === "select" || question.promptType === "checkbox") { + return parseSelection(answer, question.choices, question.multiple); + } + return answer; + } finally { + reader.close(); + } + } + + report(message: string): void { + stdout.write(`${message}\n`); + } +} + +/** + * Test-only answer source. An array supplies successive retry answers, while a scalar supplies + * the answer once. An absent answer lets the runner apply the declared default. + */ +export class ScriptedPromptDriver implements PromptDriver { + readonly answers: Map; + readonly messages: string[] = []; + + constructor(answers: Record) { + this.answers = new Map( + Object.entries(answers).map(([name, value]) => [name, Array.isArray(value) ? [...value] : [value]]), + ); + } + + async ask(question: InstallerQuestion): Promise { + const answers = this.answers.get(question.name); + if (answers === undefined || answers.length === 0) return question.defaultValue; + return answers.shift(); + } + + report(message: string): void { + this.messages.push(message); + } +} diff --git a/source/idea/ideactl/src/cli/scheduler-state-read.ts b/source/idea/ideactl/src/cli/scheduler-state-read.ts new file mode 100644 index 00000000..bb7ef256 --- /dev/null +++ b/source/idea/ideactl/src/cli/scheduler-state-read.ts @@ -0,0 +1,356 @@ +/** + * Read the batch server's own state, read-only, over the systems manager channel. + * + * The admission boundary needs three facts about the batch server: scheduling + * stopped, every queue disabled, and no work still in flight. Before the cutover + * the batch server runs on a virtual machine that the account already manages, so + * those facts are readable today without a new interface in the scheduler module. + * + * Three properties make this an observation path rather than a command path. + * + * 1. What reaches the host is a fixed document. Its content is the constant in + * this file, it declares no parameters, and the command names only that + * document, so neither this tool nor its caller can put a character on the + * host. An operator running the migration needs `ssm:SendCommand` on this one + * document, which cannot run anything else, rather than on the shell document, + * which can run anything. + * 2. The content is verified before it is used. A document already present under + * the expected name is read back and compared byte for byte, so a document + * that was replaced with one that writes is refused rather than run. + * 3. It fails closed. Every unreadable answer throws. Nothing here returns a + * default, because the values that would be defaulted, scheduling off and no + * jobs, are exactly the values that would let the boundary pass. + * + * After the cutover the batch server is a task and nothing in this file reaches + * it. That is the caller's decision to make, and the caller refuses there. + */ + +import { createHash } from "node:crypto"; + +import type { AwsClientOptions } from "./aws-client-options.ts"; + +/** The only thing this tool ever asks a batch server host to do. */ +export const SCHEDULER_READ_DOCUMENT_CONTENT = JSON.stringify( + { + schemaVersion: "2.2", + description: + "Report the batch server's scheduling flag, per-queue enablement and job state counts. Reads only: no parameters, no writes.", + mainSteps: [ + { + action: "aws:runShellScript", + name: "readBatchServerState", + inputs: { + runCommand: [ + "set -eu", + "/opt/pbs/bin/qstat -B -f -F json", + "echo ---QUEUES---", + "/opt/pbs/bin/qstat -Q -f -F json", + ], + }, + }, + ], + }, + undefined, + 2, +); + +/** + * The document name carries the content digest, so a release that changes the + * probe uses a new document instead of disagreeing with one already in place. + */ +export const SCHEDULER_READ_DOCUMENT_NAME = `idea-scheduler-read-state-${ + createHash("sha256").update(SCHEDULER_READ_DOCUMENT_CONTENT).digest("hex").slice(0, 12) +}`; + +/** One batch server queue as the server itself reports it. */ +export interface BatchServerQueue { + name: string; + enabled: boolean; + started: boolean; +} + +/** What the batch server reports about itself at the moment of the call. */ +export interface BatchServerState { + /** The name the running server answers under, which is not always the configured one. */ + serverName: string; + scheduling: boolean; + queues: BatchServerQueue[]; + /** Every state the server counts, as reported, so nothing is dropped by grouping. */ + stateCounts: Readonly>; + queuedJobs: number; + provisioningJobs: number; + runningJobs: number; +} + +/** Raised when the batch server's state cannot be established. Never swallowed. */ +export class SchedulerStateUnreadableError extends Error {} + +/** One completed document invocation. */ +export interface SsmInvocation { + status: string; + responseCode: number; + stdout: string; + stderr: string; +} + +/** The systems manager surface this read needs, as a seam tests drive without a client. */ +export interface SsmReadChannel { + /** The stored content of a document, or undefined when no document has that name. */ + documentContent(name: string): Promise; + createDocument(name: string, content: string): Promise; + /** Run one document on one instance and wait for a terminal result. */ + runDocument(name: string, instanceId: string): Promise; +} + +const QUEUE_SEPARATOR = "---QUEUES---"; + +/** PBS reports its booleans as the strings True and False. Anything else is unreadable. */ +function readFlag(value: unknown, label: string): boolean { + if (typeof value === "boolean") return value; + if (value === "True") return true; + if (value === "False") return false; + throw new SchedulerStateUnreadableError( + `the batch server reported ${label}=${JSON.stringify(value)}, which is neither True nor False, so its state is not established`, + ); +} + +/** `Transit:0 Queued:0 Held:0 Waiting:0 Running:0 Exiting:0 Begun:0` */ +function parseStateCounts(value: unknown): Record { + if (typeof value !== "string" || value.trim() === "") { + throw new SchedulerStateUnreadableError( + "the batch server reported no state_count, so the number of jobs in flight is not established", + ); + } + const counts: Record = {}; + for (const field of value.trim().split(/\s+/)) { + const separator = field.indexOf(":"); + const name = field.slice(0, separator); + const count = Number(field.slice(separator + 1)); + if (separator <= 0 || !Number.isSafeInteger(count) || count < 0) { + throw new SchedulerStateUnreadableError( + `the batch server reported an unreadable state_count field: ${JSON.stringify(field)}`, + ); + } + counts[name] = count; + } + if (Object.keys(counts).length === 0) { + throw new SchedulerStateUnreadableError("the batch server reported an empty state_count"); + } + return counts; +} + +function sumStates(counts: Readonly>, states: readonly string[]): number { + return states.reduce((total, state) => total + (counts[state] ?? 0), 0); +} + +function parseJsonSection(text: string, label: string): Record { + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch (error) { + throw new SchedulerStateUnreadableError( + `the ${label} reply from the batch server is not JSON (${error instanceof Error ? error.message : String(error)})`, + ); + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + throw new SchedulerStateUnreadableError(`the ${label} reply from the batch server is not an object`); + } + return parsed as Record; +} + +/** + * Turn the probe's output into the three facts the boundary needs. + * + * `total_jobs` is deliberately not used: with job history enabled it counts + * finished jobs too, so a drained server reports a non-zero total. The per-state + * counts are the live figures. + */ +export function parseBatchServerState(stdout: string): BatchServerState { + const separator = stdout.indexOf(QUEUE_SEPARATOR); + if (separator < 0) { + throw new SchedulerStateUnreadableError( + "the batch server probe returned no queue section, so its output is incomplete", + ); + } + const serverReply = parseJsonSection(stdout.slice(0, separator), "server"); + const queueReply = parseJsonSection(stdout.slice(separator + QUEUE_SEPARATOR.length), "queue"); + + const servers = serverReply["Server"]; + if (typeof servers !== "object" || servers === null || Array.isArray(servers)) { + throw new SchedulerStateUnreadableError("the batch server reply names no server, so it did not answer"); + } + const [serverName, serverEntry] = Object.entries(servers as Record)[0] ?? []; + if (serverName === undefined || typeof serverEntry !== "object" || serverEntry === null) { + throw new SchedulerStateUnreadableError("the batch server reply names no server, so it did not answer"); + } + const server = serverEntry as Record; + + const queueEntries = queueReply["Queue"]; + if (typeof queueEntries !== "object" || queueEntries === null || Array.isArray(queueEntries)) { + throw new SchedulerStateUnreadableError( + "the batch server reported no queue list, so per-queue admission is not established", + ); + } + const queues = Object.entries(queueEntries as Record).map(([name, value]) => { + if (typeof value !== "object" || value === null) { + throw new SchedulerStateUnreadableError(`the batch server reported queue ${name} without any attributes`); + } + const attributes = value as Record; + return { + name, + enabled: readFlag(attributes["enabled"], `queue ${name} enabled`), + started: readFlag(attributes["started"], `queue ${name} started`), + }; + }); + + const stateCounts = parseStateCounts(server["state_count"]); + return { + serverName, + scheduling: readFlag(server["scheduling"], "scheduling"), + queues, + stateCounts, + // Held is counted with queued rather than ignored: a held job is work the server still holds. + queuedJobs: sumStates(stateCounts, ["Queued", "Waiting", "Transit", "Held"]), + provisioningJobs: sumStates(stateCounts, ["Begun"]), + runningJobs: sumStates(stateCounts, ["Running", "Exiting"]), + }; +} + +/** + * Ensure the fixed document is in place and is the one this release wrote. + * + * A document under the expected name whose content differs is refused rather + * than repaired: the only reasons for a difference are a document this release + * did not write and one that was edited, and neither is safe to run. + */ +export async function ensureSchedulerReadDocument(channel: SsmReadChannel): Promise<"present" | "created"> { + const stored = await channel.documentContent(SCHEDULER_READ_DOCUMENT_NAME); + if (stored === undefined) { + await channel.createDocument(SCHEDULER_READ_DOCUMENT_NAME, SCHEDULER_READ_DOCUMENT_CONTENT); + const written = await channel.documentContent(SCHEDULER_READ_DOCUMENT_NAME); + if (written !== SCHEDULER_READ_DOCUMENT_CONTENT) { + throw new SchedulerStateUnreadableError( + `the read-only document ${SCHEDULER_READ_DOCUMENT_NAME} did not read back as written, so what would run on the batch server host is not known`, + ); + } + return "created"; + } + if (stored !== SCHEDULER_READ_DOCUMENT_CONTENT) { + throw new SchedulerStateUnreadableError( + `the document ${SCHEDULER_READ_DOCUMENT_NAME} exists with content this release did not write, so what it would run on the batch server host is not known. Inspect it, and delete it if it is not the read-only probe.`, + ); + } + return "present"; +} + +/** Read the batch server's state, or throw. There is no third answer. */ +export async function readBatchServerState( + channel: SsmReadChannel, + instanceId: string, +): Promise { + await ensureSchedulerReadDocument(channel); + const invocation = await channel.runDocument(SCHEDULER_READ_DOCUMENT_NAME, instanceId); + if (invocation.status !== "Success" || invocation.responseCode !== 0) { + throw new SchedulerStateUnreadableError( + `the batch server state read on ${instanceId} ended ${invocation.status} with exit code ${invocation.responseCode}: ${ + invocation.stderr.replaceAll(/\s+/g, " ").trim() || "no error output" + }`, + ); + } + return parseBatchServerState(invocation.stdout); +} + +/** One printable line naming every fact the boundary rests on. */ +export function renderBatchServerState(state: BatchServerState): string { + const enabled = state.queues.filter((queue) => queue.enabled).map((queue) => queue.name); + const counts = Object.entries(state.stateCounts) + .map(([name, count]) => `${name}:${count}`) + .join(" "); + return [ + `server=${state.serverName}`, + `scheduling=${state.scheduling}`, + `queues=${state.queues.length}`, + `enabled=${enabled.length === 0 ? "none" : enabled.join(",")}`, + `state_count=${counts}`, + ].join("; "); +} + +/** Where the read happens. */ +export interface SsmReadTarget { + awsRegion: string; + awsProfile?: string; + sleep(ms: number): Promise; + /** Client options, so this file shares the command tree's one credential path. */ + clientOptions(awsRegion: string, awsProfile?: string): Promise; +} + +const TERMINAL_STATUSES = new Set(["Success", "Cancelled", "TimedOut", "Failed"]); +const POLL_INTERVAL_MS = 2_000; +const POLL_LIMIT = 45; + +/** The live channel. It calls three read actions and one document create, and nothing else. */ +export function liveSsmReadChannel(target: SsmReadTarget): SsmReadChannel { + const client = async () => { + const sdk = await import("@aws-sdk/client-ssm"); + return { sdk, client: new sdk.SSMClient(await target.clientOptions(target.awsRegion, target.awsProfile)) }; + }; + + return { + async documentContent(name) { + const { sdk, client: ssm } = await client(); + try { + const result = await ssm.send(new sdk.GetDocumentCommand({ Name: name, DocumentFormat: "JSON" })); + return result.Content; + } catch (error) { + if ((error as { name?: string })?.name === "InvalidDocument") return undefined; + throw error; + } + }, + async createDocument(name, content) { + const { sdk, client: ssm } = await client(); + await ssm.send( + new sdk.CreateDocumentCommand({ + Name: name, + Content: content, + DocumentType: "Command", + DocumentFormat: "JSON", + TargetType: "/AWS::EC2::Instance", + }), + ); + }, + async runDocument(name, instanceId) { + const { sdk, client: ssm } = await client(); + const sent = await ssm.send( + new sdk.SendCommandCommand({ DocumentName: name, InstanceIds: [instanceId] }), + ); + const commandId = sent.Command?.CommandId; + if (commandId === undefined) { + throw new SchedulerStateUnreadableError("the batch server state read returned no command id"); + } + for (let attempt = 0; attempt < POLL_LIMIT; attempt += 1) { + await target.sleep(POLL_INTERVAL_MS); + let result; + try { + result = await ssm.send( + new sdk.GetCommandInvocationCommand({ CommandId: commandId, InstanceId: instanceId }), + ); + } catch (error) { + // The invocation is not registered for a moment after the command is accepted. + if ((error as { name?: string })?.name === "InvocationDoesNotExist") continue; + throw error; + } + const status = result.Status ?? "Pending"; + if (!TERMINAL_STATUSES.has(status)) continue; + return { + status, + responseCode: result.ResponseCode ?? -1, + stdout: result.StandardOutputContent ?? "", + stderr: result.StandardErrorContent ?? "", + }; + } + throw new SchedulerStateUnreadableError( + `the batch server state read on ${instanceId} did not finish within ${(POLL_INTERVAL_MS * POLL_LIMIT) / 1000} seconds`, + ); + }, + }; +} diff --git a/source/idea/ideactl/src/config/arn-builder.ts b/source/idea/ideactl/src/config/arn-builder.ts new file mode 100644 index 00000000..ad22bd49 --- /dev/null +++ b/source/idea/ideactl/src/config/arn-builder.ts @@ -0,0 +1,342 @@ +/** + * Literal ARN construction from cluster config. Port of ideasdk `context/arn_builder.py`. + * + * Partition, region, account id and dns suffix are config values, never CDK pseudo-parameters: + * the rendered policy documents in the live templates carry literal ARNs. + */ + +import type { ClusterConfig } from './cluster-config.ts'; + +export interface BuildArnInput { + partition?: string; + service?: string; + region?: string; + accountId?: string; + resource?: string; + resourceType?: string; + resourceId?: string; + resourceDelimiter?: string; +} + +const MODULE_CLUSTER_MANAGER = 'cluster-manager'; +const MODULE_DIRECTORYSERVICE = 'directoryservice'; + +export class ArnBuilder { + readonly config: ClusterConfig; + + constructor(config: ClusterConfig) { + this.config = config; + } + + static buildArn(input: BuildArnInput): string { + const { partition, service, region, accountId, resource, resourceType, resourceId } = input; + const delimiter = input.resourceDelimiter ?? '/'; + let arn = `arn:${partition}:${service}:${region}:${accountId}`; + if (resource !== undefined) { + arn += `:${resource}`; + } else if (resourceType === undefined) { + arn += `:${resourceId}`; + } else { + arn += `:${resourceType}${delimiter}${resourceId}`; + } + return arn; + } + + getArn(service: string, resource: string, awsAccountId?: string, awsRegion?: string): string { + return ArnBuilder.buildArn({ + partition: this.config.getString('cluster.aws.partition'), + service, + region: awsRegion ?? this.config.getString('cluster.aws.region'), + accountId: awsAccountId ?? this.config.getString('cluster.aws.account_id'), + resource, + }); + } + + private clusterName(): string | undefined { + return this.config.getString('cluster.cluster_name'); + } + + private region(): string | undefined { + return this.config.getString('cluster.aws.region'); + } + + private dnsSuffix(): string | undefined { + return this.config.getString('cluster.aws.dns_suffix'); + } + + get vpcArn(): string { + return this.getArn('ec2', `vpc/${this.config.getString('cluster.network.vpc_id')}`); + } + + getLogGroupArn(suffix = '*'): string { + return this.getArn('logs', `log-group:/${this.clusterName()}${suffix}`); + } + + getLogStreamArn(): string { + return this.getArn('logs', `log-group:/${this.clusterName()}*:log-stream:*`); + } + + getLambdaLogGroupArn(suffix = '*'): string { + return this.getArn('logs', `log-group:/aws/lambda/${this.clusterName()}${suffix}`); + } + + get lambdaLogStreamArn(): string { + return this.getArn('logs', `log-group:/aws/lambda/${this.clusterName()}*:log-stream:*`); + } + + get ec2CommonArns(): string[] { + return [ + this.getArn('ec2', 'subnet/*', '*', '*'), + this.getArn('ec2', 'key-pair/*', undefined, '*'), + this.getArn('ec2', 'instance/*', undefined, '*'), + this.getArn('ec2', 'snapshot/*', '*', '*'), + this.getArn('ec2', 'launch-template/*', undefined, '*'), + this.getArn('ec2', 'volume/*', undefined, '*'), + this.getArn('ec2', 'security-group/*', undefined, '*'), + this.getArn('ec2', 'placement-group/*', undefined, '*'), + this.getArn('ec2', 'network-interface/*', undefined, '*'), + this.getArn('ec2', 'spot-instances-request/*', '*', '*'), + this.getArn('ec2', 'image/*', '*', '*'), + ]; + } + + get s3GlobalArns(): string[] { + return [ + this.getArn('s3', `dcv-license.${this.region()}/*`, '', ''), + this.getArn('s3', 'ec2-linux-nvidia-drivers/*', '', ''), + this.getArn('s3', 'ec2-linux-nvidia-drivers', '', ''), + this.getArn('s3', 'ec2-windows-nvidia-drivers/*', '', ''), + this.getArn('s3', 'ec2-windows-nvidia-drivers', '', ''), + this.getArn('s3', 'nvidia-gaming/*', '', ''), + this.getArn('s3', 'nvidia-gaming-drivers', '', ''), + this.getArn('s3', 'nvidia-gaming-drivers/*', '', ''), + this.getArn('s3', 'ec2-amd-linux-drivers/*', '', ''), + this.getArn('s3', 'ec2-amd-linux-drivers', '', ''), + this.getArn('s3', 'ec2-amd-windows-drivers/*', '', ''), + this.getArn('s3', 'ec2-amd-windows-drivers', '', ''), + ]; + } + + get albListenerRuleArn(): string { + return this.getArn('elasticloadbalancing', `listener-rule/app/${this.clusterName()}*/*/*`); + } + + get albListenerArn(): string { + return this.getArn('elasticloadbalancing', `listener/app/${this.clusterName()}*/*/*`); + } + + get targetGroupArn(): string { + return this.getArn('elasticloadbalancing', 'targetgroup/soca*/*'); + } + + getLambdaArn(suffix = '*'): string { + return this.getArn('lambda', `function:${this.clusterName()}-${suffix}`); + } + + get serviceRoleArns(): string[] { + // the service linked role path is literally 'aws-service-role' in every partition, + // including aws-us-gov and aws-cn. only the arn prefix is partition specific. + const partition = this.config.getString('cluster.aws.partition'); + const accountId = this.config.getString('cluster.aws.account_id'); + const roleArn = (service: string): string => + `arn:${partition}:iam::${accountId}:role/aws-service-role/${service}`; + const suffix = this.dnsSuffix(); + return [ + roleArn(`s3.data-source.lustre.fsx.${suffix}/*`), + roleArn(`autoscaling.${suffix}/*`), + roleArn(`spotfleet.${suffix}/*`), + roleArn(`fsx.${suffix}/*`), + ]; + } + + /** iam path for per-project instance roles. such roles are created at runtime, not by cdk. */ + get projectRolePath(): string { + return `/idea/${this.config.getString('cluster.cluster_name', undefined, { required: true })}/projects/`; + } + + getProjectRoleArn(roleName = '*'): string { + return this.getArn('iam', `role${this.projectRolePath}${roleName}`, undefined, ''); + } + + getProjectInstanceProfileArn(name = '*'): string { + return this.getArn('iam', `instance-profile${this.projectRolePath}${name}`, undefined, ''); + } + + getProjectPolicyArn(name = '*'): string { + return this.getArn('iam', `policy${this.projectRolePath}${name}`, undefined, ''); + } + + getProjectPermissionsBoundaryArn(): string { + const clusterName = this.config.getString('cluster.cluster_name', undefined, { required: true }); + const awsRegion = this.config.getString('cluster.aws.region', undefined, { required: true }); + const moduleId = this.config.moduleId(MODULE_CLUSTER_MANAGER); + return this.getArn('iam', `policy/${clusterName}-${awsRegion}-${moduleId}-project-boundary`, undefined, ''); + } + + get bedrockInvocationLogGroupName(): string { + const clusterName = this.config.getString('cluster.cluster_name', undefined, { required: true }); + const moduleId = this.config.moduleId(MODULE_CLUSTER_MANAGER); + return `/${clusterName}/${moduleId}/bedrock-invocations`; + } + + get bedrockInvocationLogGroupArn(): string { + return this.getArn('logs', `log-group:${this.bedrockInvocationLogGroupName}`); + } + + get bedrockApplicationInferenceProfileArn(): string { + return this.getArn('bedrock', 'application-inference-profile/*'); + } + + get bedrockSystemInferenceProfileArn(): string { + return this.getArn('bedrock', 'inference-profile/*'); + } + + get bedrockAnySystemInferenceProfileArn(): string { + // region and account wildcarded: a deny on system profiles has to cover every region a + // caller could reach, not only the cluster's own. + return this.getArn('bedrock', 'inference-profile/*', '*', '*'); + } + + get bedrockFoundationModelArn(): string { + return this.getArn('bedrock', 'foundation-model/*', '', '*'); + } + + get sesArn(): string { + return this.getArn('ses', 'identity/*', undefined, '*'); + } + + get dcvLicenseS3BucketArns(): string[] { + return [this.getArn('s3', 'dcv-license.*/*', '', ''), this.getArn('s3', 'dcv-license.*', '', '')]; + } + + get s3BucketArns(): string[] { + const bucket = this.config.getString('cluster.cluster_s3_bucket'); + return [this.getArn('s3', `${bucket}/*`, '', ''), this.getArn('s3', `${bucket}`, '', '')]; + } + + getSsmArn(resourceId: string): string { + return ArnBuilder.buildArn({ + partition: this.config.getString('cluster.aws.partition'), + service: 'ssm', + region: '', + accountId: '', + resourceType: '', + resourceId, + resourceDelimiter: ':', + }); + } + + get clusterConfigDdbArn(): string[] { + const cluster = this.clusterName(); + return [ + this.getArn('dynamodb', `table/${cluster}.cluster-settings`, undefined, this.region()), + this.getArn('dynamodb', `table/${cluster}.cluster-settings/stream/*`, undefined, this.region()), + this.getArn('dynamodb', `table/${cluster}.modules`, undefined, this.region()), + ]; + } + + getDdbTableArn(tableNameSuffix: string): string { + return this.getArn('dynamodb', `table/${this.clusterName()}.${tableNameSuffix}`, undefined, this.region()); + } + + getAdAutomationDdbTableArn(): string { + return this.getDdbTableArn('ad-automation'); + } + + getAdAutomationSqsQueueArn(): string { + return this.getSqsArn(`${this.config.moduleId(MODULE_DIRECTORYSERVICE)}-ad-automation.fifo`); + } + + getKinesisArn(): string { + return this.getArn('kinesis', `stream/${this.clusterName()}-*`, undefined, this.region()); + } + + getSnsArn(topicNameSuffix: string): string { + return this.getArn('sns', `${this.clusterName()}-${topicNameSuffix}`, undefined, this.region()); + } + + getSqsArn(queueNameSuffix: string): string { + return this.getArn('sqs', `${this.clusterName()}-${queueNameSuffix}`, undefined, this.region()); + } + + getRoute53HostedzoneArn(): string { + const partition = this.config.getString('cluster.aws.partition', undefined, { required: true }); + return `arn:${partition}:route53:::hostedzone/*`; + } + + private kmsKeyArnFor(keyIdSetting: string): string { + return this.getArn('kms', `key/${this.config.getString(keyIdSetting)}`, undefined, this.region()); + } + + get kmsSecretsmanagerKeyArn(): string { + return this.kmsKeyArnFor('cluster.secretsmanager.kms_key_id'); + } + + get kmsSqsKeyArn(): string { + return this.kmsKeyArnFor('cluster.sqs.kms_key_id'); + } + + get kmsSnsKeyArn(): string { + return this.kmsKeyArnFor('cluster.sns.kms_key_id'); + } + + get kmsDynamodbKeyArn(): string { + return this.kmsKeyArnFor('cluster.dynamodb.kms_key_id'); + } + + get kmsEbsKeyArn(): string { + return this.kmsKeyArnFor('cluster.ebs.kms_key_id'); + } + + get kmsBackupKeyArn(): string { + return this.kmsKeyArnFor('cluster.backups.backup_vault.kms_key_id'); + } + + get kmsOpensearchKeyArn(): string { + return this.kmsKeyArnFor('analytics.opensearch.kms_key_id'); + } + + get kmsKinesisKeyArn(): string { + return this.kmsKeyArnFor('analytics.kinesis.kms_key_id'); + } + + /** one arn per service whose kms_key_id is set; insertion order is Python's dict order. */ + get kmsKeyArn(): string[] { + const settings = [ + 'cluster.secretsmanager.kms_key_id', + 'cluster.sqs.kms_key_id', + 'cluster.sns.kms_key_id', + 'cluster.dynamodb.kms_key_id', + 'cluster.ebs.kms_key_id', + 'cluster.backups.backup_vault.kms_key_id', + 'analytics.opensearch.kms_key_id', + 'analytics.kinesis.kms_key_id', + ]; + return settings + .filter((setting) => this.config.getString(setting) !== undefined) + .map((setting) => this.kmsKeyArnFor(setting)); + } + + get userPoolArn(): string { + return this.getArn( + 'cognito-idp', + `userpool/${this.config.getString('identity-provider.cognito.user_pool_id')}`, + undefined, + this.region(), + ); + } + + getDirectoryServiceArn(): string { + const directoryId = this.config.getString('directoryservice.directory_id', undefined, { required: true }); + return this.getArn('ds', `directory/${directoryId}`, undefined, this.region()); + } + + getDdbApplicationAutoscalingServiceRoleArn(): string { + return this.getArn( + 'iam', + 'role/aws-service-role/dynamodb.application-autoscaling.amazonaws.com/AWSServiceRoleForApplicationAutoScaling_DynamoDBTable', + undefined, + '', + ); + } +} diff --git a/source/idea/ideactl/src/config/cluster-config-db.ts b/source/idea/ideactl/src/config/cluster-config-db.ts new file mode 100644 index 00000000..84fba288 --- /dev/null +++ b/source/idea/ideactl/src/config/cluster-config-db.ts @@ -0,0 +1,362 @@ +/** + * Write side of the cluster configuration tables. + * + * Port of the write half of ideasdk `config/cluster_config_db.py`: table creation (the two tables + * are made with the SDK, not CloudFormation), the add-only `sync_modules_in_db` / + * `sync_cluster_settings_in_db`, `set_config_entry` and `delete_config_entries`. + * + * Behaviour that is contract, not accident, and is reproduced verbatim: + * - `sync_modules_in_db` never updates an existing `module_id` row (the `type` of a module is + * frozen once written) and validates `type` only for rows it is about to create; + * - `sync_cluster_settings_in_db` skips any key that already exists unless `overwrite`, and + * never deletes; deletion is only ever `delete_config_entries(prefix)`, a full scan plus + * `startsWith`; + * - every write is `SET #value=:value ADD #version :version` with `:version = 1`, so `version` + * is a per-key write counter that starts at 1 and increments, not a release version; + * - values keep their JSON types on the way in: bool -> BOOL, number -> N (as a string), string + * -> S, list -> L (an empty list stays an empty `L`, it is NOT a NULL), null -> NULL, object + * -> M. Python converts floats to `Decimal(str(v))` for boto3's sake; the JS document client + * already writes `N: String(v)`, which is the same wire value. + * + * The read side lives in `cluster-config.ts` and is not duplicated here. + */ + +import { + CreateTableCommand, + DescribeTableCommand, + type DynamoDBClient, + type SSESpecification, +} from '@aws-sdk/client-dynamodb'; +import { + DeleteCommand, + DynamoDBDocumentClient, + GetCommand, + ScanCommand, + UpdateCommand, +} from '@aws-sdk/lib-dynamodb'; + +import type { ModuleInfo } from './cluster-config.ts'; +import { isEmpty } from './cluster-config.ts'; + +const IDEA_TAG_PREFIX = 'idea:'; +const IDEA_TAG_CLUSTER_NAME = `${IDEA_TAG_PREFIX}ClusterName`; +const IDEA_TAG_BACKUP_PLAN = `${IDEA_TAG_PREFIX}BackupPlan`; +const MODULE_CLUSTER = 'cluster'; + +export const SUPPORTED_MODULE_TYPES = ['app', 'stack', 'config']; + +/** `exceptions.cluster_config_error` / `CLUSTER_CONFIG_NOT_INITIALIZED` / `invalid_params`. */ +export class ClusterConfigDbError extends Error { + constructor(message: string) { + super(message); + this.name = 'ClusterConfigDbError'; + } +} + +export interface ConfigEntry { + key: string; + value?: unknown; +} + +/** One entry of a module set, as `read_modules_from_files` produces it from `idea.yml`. */ +export interface ModuleSpec { + id: string; + name: string; + type: string; + [key: string]: unknown; +} + +export interface ClusterConfigDbOptions { + clusterName: string; + /** caller owns the client, so tests can point it at DynamoDB Local */ + client: DynamoDBClient; + /** Region named when the configuration tables are missing. */ + awsRegion?: string; + /** `cluster.dynamodb.kms_key_id`; SSE is configured only when this is set */ + dynamodbKmsKeyId?: string | null; + /** only `config update` passes true; everything else fails if the tables are missing */ + createDatabase?: boolean; + logger?: (message: string) => void; +} + +function isResourceNotFound(e: unknown): boolean { + return (e as { name?: string })?.name === 'ResourceNotFoundException'; +} + +function configurationTablesNotFound(clusterName: string, awsRegion: string): string { + if (awsRegion === '') { + return ( + `Configuration tables not found for cluster ${clusterName}. Create them with ideactl config update ` + + `--cluster-name ${clusterName} --aws-region , or confirm the cluster was installed in this account and region.` + ); + } + return ( + `Configuration tables not found for cluster ${clusterName} in ${awsRegion}. Create them with ideactl config update ` + + `--cluster-name ${clusterName} --aws-region ${awsRegion}, or confirm the cluster was installed in this account and region.` + ); +} + +async function dynamoRegion(client: DynamoDBClient): Promise { + const configured = client.config.region; + if (typeof configured === 'string' && configured !== '') return configured; + if (typeof configured === 'function') { + const value = await configured(); + if (typeof value === 'string' && value !== '') return value; + } + return undefined; +} + +const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); + +export class ClusterConfigDb { + readonly clusterName: string; + readonly awsRegion: string; + readonly dynamodbKmsKeyId: string | null; + readonly createDatabase: boolean; + private readonly client: DynamoDBClient; + private readonly doc: DynamoDBDocumentClient; + private readonly logger: (message: string) => void; + + private constructor(options: ClusterConfigDbOptions, awsRegion: string) { + this.clusterName = options.clusterName; + this.awsRegion = awsRegion; + this.client = options.client; + this.doc = DynamoDBDocumentClient.from(options.client); + this.dynamodbKmsKeyId = isEmpty(options.dynamodbKmsKeyId) ? null : (options.dynamodbKmsKeyId as string); + this.createDatabase = options.createDatabase === true; + this.logger = options.logger ?? (() => {}); + } + + /** + * `ClusterConfigDB.__init__`: validate, then either assert both tables exist or create them. + * A constructor cannot await, so the async half of `__init__` lives here. + */ + static async open(options: ClusterConfigDbOptions): Promise { + if (isEmpty(options.clusterName)) { + throw new ClusterConfigDbError('cluster_name is required'); + } + if (options.clusterName.length > 11) { + throw new ClusterConfigDbError( + `cluster_name: ${options.clusterName} cannot be more than 11 characters. current: ${options.clusterName.length}`, + ); + } + const awsRegion = options.awsRegion ?? (await dynamoRegion(options.client)) ?? ''; + const db = new ClusterConfigDb(options, awsRegion); + if (!db.createDatabase) await db.checkTableCreated(); + await db.getOrCreateModulesTable(); + await db.getOrCreateClusterSettingsTable(); + return db; + } + + get modulesTableName(): string { + return `${this.clusterName}.modules`; + } + + get clusterSettingsTableName(): string { + return `${this.clusterName}.cluster-settings`; + } + + async checkTableCreated(): Promise { + try { + await this.client.send(new DescribeTableCommand({ TableName: this.modulesTableName })); + await this.client.send(new DescribeTableCommand({ TableName: this.clusterSettingsTableName })); + return true; + } catch (e) { + if (isResourceNotFound(e)) { + throw new ClusterConfigDbError(configurationTablesNotFound(this.clusterName, this.awsRegion)); + } + throw e; + } + } + + /** `describe_table` until ACTIVE, creating the table on the first ResourceNotFoundException. */ + private async waitOrCreate(tableName: string, create: () => Promise): Promise { + for (;;) { + try { + const result = await this.client.send(new DescribeTableCommand({ TableName: tableName })); + if (result.Table?.TableStatus === 'ACTIVE') return; + await sleep(2000); + } catch (e) { + if (!isResourceNotFound(e) || !this.createDatabase) throw e; + await create(); + } + } + } + + private sseSpecification(): SSESpecification | undefined { + if (this.dynamodbKmsKeyId === null) return undefined; + return { Enabled: true, SSEType: 'KMS', KMSMasterKeyId: this.dynamodbKmsKeyId }; + } + + private async getOrCreateModulesTable(): Promise { + await this.waitOrCreate(this.modulesTableName, async () => { + if (this.dynamodbKmsKeyId !== null) { + this.logger(`detected cluster.dynamodb.kms_key_id is set to: ${this.dynamodbKmsKeyId}`); + } + this.logger(`creating cluster config dynamodb table: ${this.modulesTableName}`); + await this.client.send( + new CreateTableCommand({ + TableName: this.modulesTableName, + AttributeDefinitions: [{ AttributeName: 'module_id', AttributeType: 'S' }], + KeySchema: [{ AttributeName: 'module_id', KeyType: 'HASH' }], + BillingMode: 'PAY_PER_REQUEST', + SSESpecification: this.sseSpecification(), + Tags: [ + { Key: IDEA_TAG_CLUSTER_NAME, Value: this.clusterName }, + { Key: IDEA_TAG_BACKUP_PLAN, Value: `${this.clusterName}-${MODULE_CLUSTER}` }, + ], + }), + ); + }); + } + + private async getOrCreateClusterSettingsTable(): Promise { + await this.waitOrCreate(this.clusterSettingsTableName, async () => { + this.logger(`creating cluster config dynamodb table: ${this.clusterSettingsTableName}`); + await this.client.send( + new CreateTableCommand({ + TableName: this.clusterSettingsTableName, + AttributeDefinitions: [{ AttributeName: 'key', AttributeType: 'S' }], + KeySchema: [{ AttributeName: 'key', KeyType: 'HASH' }], + BillingMode: 'PAY_PER_REQUEST', + StreamSpecification: { StreamEnabled: true, StreamViewType: 'NEW_AND_OLD_IMAGES' }, + SSESpecification: this.sseSpecification(), + Tags: [{ Key: IDEA_TAG_CLUSTER_NAME, Value: this.clusterName }], + }), + ); + }); + } + + async getConfigEntry(key: string): Promise | undefined> { + const result = await this.doc.send( + new GetCommand({ TableName: this.clusterSettingsTableName, Key: { key } }), + ); + return result.Item; + } + + async getModuleInfo(moduleId: string): Promise { + const result = await this.doc.send( + new GetCommand({ TableName: this.modulesTableName, Key: { module_id: moduleId } }), + ); + return result.Item as ModuleInfo | undefined; + } + + /** `SET #value=:value ADD #version :version`. `version` counts writes; it never resets. */ + async setConfigEntry(key: string, value: unknown): Promise { + this.logger(`updating config: ${key} = ${String(value)}`); + await this.doc.send( + new UpdateCommand({ + TableName: this.clusterSettingsTableName, + Key: { key }, + UpdateExpression: 'SET #value=:value ADD #version :version', + ExpressionAttributeNames: { '#value': 'value', '#version': 'version' }, + // Python has no `undefined`; a missing value is Python's None, i.e. NULL. + ExpressionAttributeValues: { ':value': value === undefined ? null : value, ':version': 1 }, + }), + ); + } + + /** Add-only unless `overwrite`. Never deletes, never touches keys absent from `entries`. */ + async syncClusterSettingsInDb(entries: ConfigEntry[], overwrite = false): Promise { + this.logger(`sync config entries to db. overwrite: ${overwrite}`); + for (const entry of entries) { + if (!overwrite) { + const existing = await this.getConfigEntry(entry.key); + if (existing !== undefined) { + this.logger(`entry already exists for key: ${entry.key}, skip.`); + continue; + } + } + await this.setConfigEntry(entry.key, entry.value); + } + } + + /** + * Add-only, with no update path at all: an existing `module_id` keeps its `type`, `status`, + * `stack_name` and `version`. Python validates the type in a first pass over every module and + * writes in a second pass, so an invalid type on a NEW module aborts before anything is written + * while an invalid type on an EXISTING module is never noticed. + */ + async syncModulesInDb(modules: ModuleSpec[]): Promise { + this.logger('sync modules in db ...'); + + const modulesToCreate: ModuleSpec[] = []; + for (const module of modules) { + const existing = await this.getModuleInfo(module.id); + if (existing !== undefined) { + this.logger(`module: ${module.id}, name: ${module.name} already exists. skip.`); + continue; + } + if (!SUPPORTED_MODULE_TYPES.includes(module.type)) { + throw new ClusterConfigDbError( + `invalid type: ${module.type} for module_id: ${module.id}. ` + + `supported module types: ${SUPPORTED_MODULE_TYPES.join(', ')}`, + ); + } + modulesToCreate.push(module); + } + + for (const module of modulesToCreate) { + const status = module.type === 'config' ? 'deployed' : 'not-deployed'; + this.logger(`creating module entry for module: ${module.name}, module_id: ${module.id}`); + await this.doc.send( + new UpdateCommand({ + TableName: this.modulesTableName, + Key: { module_id: module.id }, + UpdateExpression: + 'SET #name=:name, #status=:status, #stack_name=:stack_name, #version=:version, #type=:type', + ExpressionAttributeNames: { + '#name': 'name', + '#status': 'status', + '#stack_name': 'stack_name', + '#version': 'version', + '#type': 'type', + }, + ExpressionAttributeValues: { + ':name': module.name, + ':type': module.type, + ':status': status, + ':stack_name': null, + ':version': null, + }, + }), + ); + } + } + + /** Full scan + `startsWith`, then a delete per match. `config delete `. */ + async deleteConfigEntries(configKeyPrefix: string): Promise { + if (isEmpty(configKeyPrefix)) { + throw new ClusterConfigDbError('config_key_prefix is required'); + } + + this.logger(`searching for config entries with prefix: ${configKeyPrefix}`); + const toDelete: Array> = []; + let startKey: Record | undefined; + do { + const result = await this.doc.send( + new ScanCommand({ TableName: this.clusterSettingsTableName, ExclusiveStartKey: startKey }), + ); + for (const item of result.Items ?? []) { + if (String(item['key'] ?? '').startsWith(configKeyPrefix)) toDelete.push(item); + } + startKey = result.LastEvaluatedKey; + } while (startKey !== undefined); + + if (toDelete.length === 0) { + this.logger(`no config entries found matching config prefix: ${configKeyPrefix}`); + return; + } + this.logger(`found ${toDelete.length} config entries matching: ${configKeyPrefix}`); + for (const item of toDelete) { + this.logger(`deleting config entry - ${String(item['key'])} = ${String(item['value'])}`); + await this.doc.send( + new DeleteCommand({ + TableName: this.clusterSettingsTableName, + Key: { key: item['key'] }, + }), + ); + } + this.logger(`deleted ${toDelete.length} config entries`); + } +} diff --git a/source/idea/ideactl/src/config/cluster-config.ts b/source/idea/ideactl/src/config/cluster-config.ts new file mode 100644 index 00000000..5cd41193 --- /dev/null +++ b/source/idea/ideactl/src/config/cluster-config.ts @@ -0,0 +1,552 @@ +/** Read cluster configuration from the settings and modules tables. */ + +const DEFAULT_MODULE_SET = 'default'; + +export interface ModuleInfo { + module_id: string; + name: string; + type: string; + status?: string; + stack_name?: string | null; + version?: string | null; + /** joined in memory by `get_cluster_modules`, never stored in the table */ + title?: string; + /** joined in memory by `get_cluster_modules`, never stored in the table */ + deployment_priority?: number; + [key: string]: unknown; +} + +/** `utils/module_metadata.py` MODULE_METADATA, in file order. */ +export const MODULE_METADATA: ReadonlyArray<{ + name: string; + title: string; + type: string; + deployment_priority: number; +}> = [ + { name: 'global-settings', title: 'Global Settings', type: 'config', deployment_priority: 0 }, + { name: 'bootstrap', title: 'Bootstrap', type: 'stack', deployment_priority: 1 }, + { name: 'cluster', title: 'Cluster', type: 'stack', deployment_priority: 2 }, + { name: 'analytics', title: 'Analytics', type: 'stack', deployment_priority: 3 }, + { name: 'metrics', title: 'Metrics & Monitoring', type: 'stack', deployment_priority: 3 }, + { name: 'identity-provider', title: 'Identity Provider', type: 'stack', deployment_priority: 3 }, + { name: 'directoryservice', title: 'Directory Service', type: 'stack', deployment_priority: 3 }, + { name: 'shared-storage', title: 'Shared Storage', type: 'stack', deployment_priority: 4 }, + { name: "ecs", title: "ECS", type: "stack", deployment_priority: 4.5 }, + { name: 'cluster-manager', title: 'Cluster Manager', type: 'app', deployment_priority: 5 }, + { name: 'virtual-desktop-controller', title: 'eVDI', type: 'app', deployment_priority: 6 }, + { name: 'scheduler', title: 'Scale-Out Computing', type: 'app', deployment_priority: 6 }, + { name: 'bastion-host', title: 'Bastion Host', type: 'stack', deployment_priority: 7 }, +]; + +const MODULE_METADATA_BY_NAME = new Map(MODULE_METADATA.map((entry) => [entry.name, entry])); + +export interface GetOptions { + /** raise instead of returning the default when the key is absent */ + required?: boolean; + /** override the module id the key's module-name prefix maps to */ + moduleId?: string; +} + +export interface ClusterConfigOptions { + moduleSet?: string; + /** The current module maps its own name to its ID without a module_sets row. */ + moduleId?: string; +} + +/** One page of a DynamoDB scan, already unmarshalled by the document client. */ +export interface ScanPage { + Items?: Array>; + LastEvaluatedKey?: Record; +} + +/** The one AWS call `fromDynamoDb` makes, as a function so tests can supply their own. */ +export type TableScanner = (input: { + TableName: string; + ExclusiveStartKey?: Record; +}) => Promise; + +export interface FromDynamoDbOptions extends ClusterConfigOptions { + scan?: TableScanner; +} + +/** `errorcodes.CONFIG_KEY_NOT_FOUND` */ +export class ConfigKeyNotFound extends Error {} +/** `errorcodes.CONFIG_TYPE_ERROR` */ +export class ConfigTypeError extends Error {} +/** `exceptions.cluster_config_error` */ +export class ClusterConfigError extends Error {} +/** `exceptions.general_exception` */ +export class GeneralException extends Error {} + +/** `ModelUtils.is_empty` (idea-data-model/model_utils.py:37-56). Numbers and booleans are never empty. */ +export function isEmpty(value: unknown): boolean { + if (value === null || value === undefined) return true; + if (typeof value === 'string') return value.trim().length === 0; + if (Array.isArray(value)) return value.length === 0; + if (value instanceof Map || value instanceof Set) return value.size === 0; + if (value instanceof Uint8Array) return value.length === 0; + if (typeof value === 'object') return Object.keys(value).length === 0; + return false; +} + +/** `soca_config.is_null_value`: an empty list is a real value, every other empty value is null. */ +export function isNullValue(value: unknown): boolean { + return isEmpty(value) && !Array.isArray(value); +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** One DynamoDB typed attribute value, as the AWS CLI prints it. */ +type AttributeValue = Record; + +/** boto3's Table resource unmarshalling: N becomes a number, everything else its JSON shape. */ +export function unmarshallAttribute(attr: AttributeValue): unknown { + const [type] = Object.keys(attr); + const raw = attr[type as string]; + switch (type) { + case 'S': + return raw as string; + case 'N': + return Number(raw as string); + case 'BOOL': + return raw as boolean; + case 'NULL': + return null; + case 'L': + return (raw as AttributeValue[]).map(unmarshallAttribute); + case 'M': + return unmarshallItem(raw as Record); + case 'SS': + return raw as string[]; + case 'NS': + return (raw as string[]).map(Number); + default: + throw new ConfigTypeError(`unsupported dynamodb attribute type: ${type}`); + } +} + +function unmarshallItem(item: Record): Record { + const out: Record = {}; + for (const [key, attr] of Object.entries(item)) out[key] = unmarshallAttribute(attr); + return out; +} + +/** Numeric lists interleave each number with the original list. */ +export function checkAndConvertDecimalValue(value: unknown): unknown { + if (!Array.isArray(value) || value.length === 0 || typeof value[0] !== 'number') return value; + const converted: unknown[] = []; + for (const item of value) { + if (typeof item !== 'number') { + throw new ConfigTypeError( + `invalid literal for int() with base 10: '${pyStr(item)}' (mixed list with a numeric first element)`, + ); + } + converted.push(item); + converted.push(value); + } + return converted; +} + +/** `SocaConfig.put` + pyhocon `ConfigTree.put`: build the path, null-normalise the leaf. */ +function putKey(tree: Record, key: string, value: unknown): void { + const parts = key.split('.'); + let node = tree; + for (let i = 0; i < parts.length - 1; i += 1) { + const part = parts[i] as string; + const next = node[part]; + if (isPlainObject(next)) { + node = next; + } else { + const created: Record = {}; + node[part] = created; + node = created; + } + } + node[parts[parts.length - 1] as string] = isNullValue(value) ? null : value; +} + +/** `int(value)` */ +function toInt(value: unknown, key: string): number { + if (typeof value === 'boolean') return value ? 1 : 0; + if (typeof value === 'number') return Math.trunc(value); + if (typeof value === 'string' && /^\s*[+-]?\d+\s*$/.test(value)) return Number(value.trim()); + throw new ConfigTypeError(`${key} has type '${pyTypeName(value)}' rather than 'int'`); +} + +/** `float(value)` */ +function toFloat(value: unknown, key: string): number { + if (typeof value === 'boolean') return value ? 1 : 0; + if (typeof value === 'number') return value; + if (typeof value === 'string' && value.trim() !== '' && Number.isFinite(Number(value))) { + return Number(value); + } + throw new ConfigTypeError(`${key} has type '${pyTypeName(value)}' rather than 'float'`); +} + +const BOOL_CONVERSIONS: Record = { + true: true, + yes: true, + on: true, + false: false, + no: false, + off: false, +}; + +/** + * pyhocon's `re.match('^[1-9][0-9]*$|0', key)` in `get_list`: either the whole key is a non-zero + * integer, or it starts with a `0`. `re.match` anchors at the start, hence the second branch. + */ +const NUMERIC_TREE_KEY = /^(?:[1-9][0-9]*$|0)/; + +/** One `.cluster-settings` row, with the Decimal post-processing Python applies. */ +function toConfigEntry(row: Record): { key: string; value?: unknown } { + const key = row['key']; + if (typeof key !== 'string') { + throw new ConfigTypeError(`cluster-settings row without a string key: ${JSON.stringify(row)}`); + } + return { key, value: checkAndConvertDecimalValue(row['value']) }; +} + +/** One `.modules` row. Python does not post-process these. */ +function toModuleInfo(row: Record): ModuleInfo { + const { module_id: moduleId, name, type } = row; + if (typeof moduleId !== 'string' || typeof name !== 'string' || typeof type !== 'string') { + throw new ConfigTypeError(`modules row without module_id/name/type: ${JSON.stringify(row)}`); + } + return { ...row, module_id: moduleId, name, type }; +} + +/** The live scanner. Imported lazily so that reading a fixture never loads the AWS SDK. */ +async function defaultTableScanner(region: string): Promise { + const { DynamoDBClient } = await import('@aws-sdk/client-dynamodb'); + const { DynamoDBDocumentClient, ScanCommand } = await import('@aws-sdk/lib-dynamodb'); + const doc = DynamoDBDocumentClient.from(new DynamoDBClient({ region })); + return (input) => doc.send(new ScanCommand(input)); +} + +export class ClusterConfig { + private readonly tree: Record; + private readonly moduleList: ModuleInfo[]; + readonly moduleSet: string; + /** ID of the current module. */ + currentModuleId: string | undefined; + moduleInfo: ModuleInfo | undefined; + + constructor( + entries: Array<{ key: string; value?: unknown }>, + modules: ModuleInfo[] = [], + options: ClusterConfigOptions = {}, + ) { + this.tree = {}; + // Sorted keys let `a.b.c` replace a scalar at `a.b`. + const sorted = [...entries].sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0)); + for (const entry of sorted) putKey(this.tree, entry.key, entry.value); + this.moduleList = modules; + this.moduleSet = isEmpty(options.moduleSet) ? DEFAULT_MODULE_SET : (options.moduleSet as string); + if (!isEmpty(options.moduleId)) this.setModuleId(options.moduleId as string); + } + + /** Raw `aws dynamodb scan` output for the two tables, as captured by the parity fixtures. */ + static fromFile(scanJson: string, modulesJson?: string, options: ClusterConfigOptions = {}): ClusterConfig { + const scan = JSON.parse(scanJson) as { Items?: Array> }; + const entries = (scan.Items ?? []).map((item) => toConfigEntry(unmarshallItem(item))); + let modules: ModuleInfo[] = []; + if (modulesJson !== undefined) { + const modulesScan = JSON.parse(modulesJson) as { Items?: Array> }; + modules = (modulesScan.Items ?? []).map((item) => toModuleInfo(unmarshallItem(item))); + } + return new ClusterConfig(entries, modules, options); + } + + /** + * `ClusterConfigDB.get_config_entries` + `get_cluster_modules`: a full scan of both tables, + * following `LastEvaluatedKey`. `options.scan` replaces the one AWS call, for tests and for + * callers that already hold a client. + */ + static async fromDynamoDb( + cluster: string, + region: string, + options: FromDynamoDbOptions = {}, + ): Promise { + const scan = options.scan ?? (await defaultTableScanner(region)); + + const scanAll = async (tableName: string): Promise>> => { + const rows: Array> = []; + let startKey: Record | undefined; + do { + const page = await scan({ TableName: tableName, ExclusiveStartKey: startKey }); + rows.push(...(page.Items ?? [])); + startKey = page.LastEvaluatedKey; + } while (startKey !== undefined); + return rows; + }; + + try { + const entries = (await scanAll(`${cluster}.cluster-settings`)).map(toConfigEntry); + const modules = (await scanAll(`${cluster}.modules`)).map(toModuleInfo); + return new ClusterConfig(entries, modules, options); + } catch (error) { + if ((error as { name?: string }).name === 'ResourceNotFoundException') { + throw new ClusterConfigError( + `No configuration tables for cluster ${cluster} in ${region} (looked for ${cluster}.modules and ${cluster}.cluster-settings). ` + + `Install with ideactl quick-setup, or run ideactl config update --cluster-name ${cluster} --aws-region ${region}. ` + + 'If the cluster already exists, check --aws-region and --aws-profile.', + ); + } + throw error; + } + } + + /** `ClusterConfigDB.get_cluster_modules`: the table rows, joined with the module metadata. */ + modules(): ModuleInfo[] { + return this.moduleList.map((module) => { + const metadata = MODULE_METADATA_BY_NAME.get(module.name); + if (metadata === undefined) { + throw new GeneralException(`module not found for name: ${module.name}`); + } + return { ...module, title: metadata.title, deployment_priority: metadata.deployment_priority }; + }); + } + + moduleInfoById(moduleId: string): ModuleInfo | undefined { + return this.moduleList.find((module) => module.module_id === moduleId); + } + + setModuleId(moduleId: string): void { + const info = this.moduleInfoById(moduleId); + if (info === undefined) throw new GeneralException(`module not found for module_id: ${moduleId}`); + this.currentModuleId = moduleId; + this.moduleInfo = info; + } + + /** `ClusterConfig.get_module_id`, required, so a missing module_sets row raises. */ + moduleId(moduleName: string): string { + const value = this.getString( + `global-settings.module_sets.${this.moduleSet}.${moduleName}.module_id`, + undefined, + { required: true }, + ); + if (value === undefined) { + throw new ConfigKeyNotFound( + `'${moduleName}', key: global-settings.module_sets.${this.moduleSet}.${moduleName}.module_id`, + ); + } + return value; + } + + isModuleEnabled(moduleName: string): boolean { + return !isEmpty( + this.getString(`global-settings.module_sets.${this.moduleSet}.${moduleName}.module_id`), + ); + } + + /** `ClusterConfig.get_real_key` (cluster_config.py:95-117). */ + getRealKey(key: string, moduleId?: string): string { + const parts = key.split('.'); + const moduleName = parts[0] as string; + if (moduleName === 'global-settings') return key; + + let resolved = moduleId; + if (isEmpty(resolved)) { + if (this.moduleInfo !== undefined && this.moduleInfo.name === moduleName) { + resolved = this.moduleInfo.module_id; + } else { + resolved = this.rawGetString( + `global-settings.module_sets.${this.moduleSet}.${moduleName}.module_id`, + ); + } + } + if (isEmpty(resolved)) resolved = moduleName; + // single-segment keys produce a trailing '.', exactly as Python's '.'.join([]) does + return `${resolved}.${parts.slice(1).join('.')}`; + } + + // --- pyhocon layer, on already-rewritten keys --------------------------------------------- + + private rawGet(key: string, required: boolean): unknown { + // pyhocon parses a key with `re.findall(r'"[^"]+"|[^\.]+', key)`, and `[^\.]+` needs at least + // one non-dot character, so an empty segment is never a path element. `get_real_key` turns a + // single-segment key into '.', which pyhocon reads as [''] and resolves + // to the whole module subtree; 'a..b' collapses the same way. + const parts = key.split('.').filter((part) => part !== ''); + let node: unknown = this.tree; + for (const part of parts) { + if (!isPlainObject(node) || !(part in node)) { + if (required) throw new ConfigKeyNotFound(`'${part}', key: ${key}`); + return undefined; + } + node = node[part]; + } + return node; + } + + private rawGetString(key: string): string | undefined { + const value = this.rawGet(key, false); + const asString = stringify(value); + return isEmpty(asString) ? undefined : asString; + } + + // --- SocaConfig getters, on module-name keys ---------------------------------------------- + + get(key: string, defaultValue?: T, options: GetOptions = {}): T { + const value = this.rawGet(this.getRealKey(key, options.moduleId), options.required === true); + return (isNullValue(value) ? defaultValue : value) as T; + } + + getString(key: string): string | undefined; + getString(key: string, defaultValue: string, options?: GetOptions): string; + getString(key: string, defaultValue?: string, options?: GetOptions): string | undefined; + getString(key: string, defaultValue?: string, options: GetOptions = {}): string | undefined { + const value = this.rawGet(this.getRealKey(key, options.moduleId), options.required === true); + const asString = value === undefined || value === null ? defaultValue : stringify(value); + return isEmpty(asString) ? defaultValue : asString; + } + + getBool(key: string): boolean | undefined; + getBool(key: string, defaultValue: boolean, options?: GetOptions): boolean; + getBool(key: string, defaultValue?: boolean, options?: GetOptions): boolean | undefined; + getBool(key: string, defaultValue?: boolean, options: GetOptions = {}): boolean | undefined { + const realKey = this.getRealKey(key, options.moduleId); + const value = this.rawGet(realKey, options.required === true); + if (value === undefined || value === null) return defaultValue; + const asString = (stringify(value) as string).toLowerCase(); + if (!(asString in BOOL_CONVERSIONS)) { + throw new ConfigTypeError(`${realKey} does not translate to a Boolean value`); + } + return BOOL_CONVERSIONS[asString]; + } + + getInt(key: string): number | undefined; + getInt(key: string, defaultValue: number, options?: GetOptions): number; + getInt(key: string, defaultValue?: number, options?: GetOptions): number | undefined; + getInt(key: string, defaultValue?: number, options: GetOptions = {}): number | undefined { + const realKey = this.getRealKey(key, options.moduleId); + const value = this.rawGet(realKey, options.required === true); + if (value === undefined || value === null) return defaultValue; + return toInt(value, realKey); + } + + getFloat(key: string): number | undefined; + getFloat(key: string, defaultValue: number, options?: GetOptions): number; + getFloat(key: string, defaultValue?: number, options?: GetOptions): number | undefined; + getFloat(key: string, defaultValue?: number, options: GetOptions = {}): number | undefined { + const realKey = this.getRealKey(key, options.moduleId); + const value = this.rawGet(realKey, options.required === true); + if (value === undefined || value === null) return defaultValue; + return toFloat(value, realKey); + } + + getList(key: string): T[] | undefined; + getList(key: string, defaultValue: T[], options?: GetOptions): T[]; + getList(key: string, defaultValue?: T[], options?: GetOptions): T[] | undefined; + getList(key: string, defaultValue?: T[], options: GetOptions = {}): T[] | undefined { + const realKey = this.getRealKey(key, options.moduleId); + const value = this.rawGet(realKey, options.required === true); + if (value === undefined || value === null) return defaultValue; + // [] is a real value: is_null_value() lets it through where every other empty value falls back + if (Array.isArray(value)) return value as T[]; + // a tree with none but numeric keys is a list to pyhocon: its values, in sorted key order + if (isPlainObject(value)) { + return Object.entries(value) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([entryKey, entryValue]) => { + if (!NUMERIC_TREE_KEY.test(entryKey)) { + throw new ConfigTypeError(`${realKey} does not translate to a list`); + } + return entryValue as T; + }); + } + throw new ConfigTypeError(`${realKey} has type '${pyTypeName(value)}' rather than 'list'`); + } + + /** + * `SocaConfig.get_config`: a subtree. A missing, NULL or empty subtree reads as the default; + * a stored scalar or list is a CONFIG_TYPE_ERROR, not a fallback. + */ + getConfig(key: string, defaultValue?: Record, options: GetOptions = {}): Record | undefined { + const realKey = this.getRealKey(key, options.moduleId); + const value = this.rawGet(realKey, options.required === true); + if (value === undefined || value === null) return defaultValue; + if (!isPlainObject(value)) { + throw new ConfigTypeError(`${realKey} has type '${pyTypeName(value)}' rather than 'config'`); + } + return isEmpty(value) ? defaultValue : value; + } + + getClusterExternalEndpoint(): string { + const clusterModuleId = this.moduleId('cluster'); + const dns = + this.getString(`${clusterModuleId}.load_balancers.external_alb.certificates.custom_dns_name`) ?? + this.getString(`${clusterModuleId}.load_balancers.external_alb.load_balancer_dns_name`); + if (isEmpty(dns)) throw new ClusterConfigError('cluster external endpoint not found'); + return `https://${dns}`; + } + + getClusterInternalEndpoint(): string { + const clusterModuleId = this.moduleId('cluster'); + const dns = + this.getString(`${clusterModuleId}.load_balancers.internal_alb.certificates.custom_dns_name`) ?? + // Check the alternate `custom_dns_name` path. + this.getString(`${clusterModuleId}.load_balancers.internal_alb.custom_dns_name`) ?? + this.getString(`${clusterModuleId}.load_balancers.internal_alb.load_balancer_dns_name`); + if (isEmpty(dns)) throw new ClusterConfigError('cluster internal endpoint not found'); + return `https://${dns}`; + } +} + +/** The name Python's `type(value).__name__` gives a value read out of the tree. */ +function pyTypeName(value: unknown): string { + if (value === undefined || value === null) return 'NoneType'; + if (typeof value === 'boolean') return 'bool'; + if (typeof value === 'number') return Number.isInteger(value) ? 'int' : 'float'; + if (typeof value === 'string') return 'str'; + if (Array.isArray(value)) return 'list'; + return 'ConfigTree'; +} + +/** + * Python `repr()` of a string: single quotes, unless the string holds a `'` and no `"`. + * Backslashes, the quote in use and the C0 controls are escaped; printable non-ASCII is not. + */ +function pyReprString(value: string): string { + const quote = value.includes("'") && !value.includes('"') ? '"' : "'"; + let out = ''; + for (const char of value) { + if (char === '\\') out += '\\\\'; + else if (char === quote) out += `\\${quote}`; + else if (char === '\n') out += '\\n'; + else if (char === '\r') out += '\\r'; + else if (char === '\t') out += '\\t'; + else if (char < ' ' || char === '\x7f') out += `\\x${char.charCodeAt(0).toString(16).padStart(2, '0')}`; + else out += char; + } + return `${quote}${out}${quote}`; +} + +/** Format configuration values with Python representation rules. */ +function pyRepr(value: unknown): string { + if (value === undefined || value === null) return 'None'; + if (typeof value === 'boolean') return value ? 'True' : 'False'; + if (typeof value === 'string') return pyReprString(value); + if (typeof value === 'number') return String(value); + if (Array.isArray(value)) return `[${value.map(pyRepr).join(', ')}]`; + const entries = Object.entries(value as Record) + .map(([key, entry]) => `${pyReprString(key)}: ${pyRepr(entry)}`) + .join(', '); + return `ConfigTree({${entries}})`; +} + +/** Python `str()`: a string is itself, a bool is lowercased by pyhocon, everything else is repr. */ +function pyStr(value: unknown): string { + if (typeof value === 'string') return value; + if (typeof value === 'boolean') return value ? 'true' : 'false'; + return pyRepr(value); +} + +/** pyhocon `ConfigTree.get_string`: `str(value)`, with booleans lowercased. */ +function stringify(value: unknown): string | undefined { + if (value === undefined || value === null) return undefined; + return pyStr(value); +} diff --git a/source/idea/ideactl/src/config/generator.ts b/source/idea/ideactl/src/config/generator.ts new file mode 100644 index 00000000..588fc173 --- /dev/null +++ b/source/idea/ideactl/src/config/generator.ts @@ -0,0 +1,248 @@ +/** + * The config generator: `values.yml` -> rendered `config//*.yml` -> flat key/value + * entries for the `.cluster-settings` table. + * Port of `ConfigGenerator.generate_config_from_templates`, `read_modules_from_files`, + * `read_config_from_files` and `traverse_config` (`ideaadministrator/app/config_generator.py`). + * + * The rendered files are the operator-editable form and are never diffed textually: what has to + * match Python byte for byte is the parsed result, so the YAML load has to agree with + * `yaml.safe_load` on every scalar the templates emit. + */ + +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import yaml from 'js-yaml'; + +import { GeneralException, isNullValue } from './cluster-config.ts'; +import { jinjaEnv, renderTemplate, toYaml } from './jinja.ts'; +import { + buildContext, + configTemplatesDirs, + loadValuesFile, + SUPPORTED_OS, + type BuildContextOptions, + type UserValues, +} from './values.ts'; + +export interface ModuleEntry { + name: string; + id: string; + type: string; + config_files: string[]; +} + +export interface ConfigEntry { + key: string; + value: unknown; +} + +/** + * `Utils.from_yaml` = PyYAML `safe_load`. The core schema is the closest js-yaml gets: it resolves + * `~`/`null` and `true|True|false|False`, and unlike the default schema it leaves date-like and + * sexagesimal-looking scalars as strings, which is what the templates' quoted values need. + */ +export function loadYaml(text: string): unknown { + return yaml.load(text, { schema: yaml.CORE_SCHEMA }); +} + +export interface GenerateOptions extends BuildContextOptions { + templatesDir?: string | string[]; +} + +const CONTAINER_MODULE: ModuleEntry = { + name: 'ecs', + id: 'ecs', + type: 'stack', + config_files: ['settings.yml'], +}; + +const MODULE_NAME_GLOBAL_SETTINGS = 'global-settings'; + +/** The two lines every module-set entry sits under. */ +const MODULE_SETS_HEADER = 'module_sets:\n default:\n'; + +/** + * `idea.yml` in the administrator's tree does not list the container module, so it is spliced in + * ahead of `cluster-manager`, where its deployment priority puts it. Ordering inside this list is + * cosmetic (`deploymentOrder` sorts by priority) but it keeps the generated file readable. + */ +function withContainerModule(modules: ModuleEntry[]): ModuleEntry[] { + if (modules.some((module) => module.id === CONTAINER_MODULE.id)) return modules; + const before = modules.findIndex((module) => module.name === 'cluster-manager'); + if (before < 0) { + throw new GeneralException('idea.yml lists no cluster-manager module to order ecs against'); + } + return [...modules.slice(0, before), { ...CONTAINER_MODULE }, ...modules.slice(before)]; +} + +/** + * Adds the container module's name-to-id mapping to the rendered default module set. The rendered + * text is kept rather than reparsed and dumped so the operator's comments survive; the anchor is + * the file's own two opening lines, and a template that no longer starts with them stops the run. + */ +function withContainerModuleSet(rendered: string): string { + if (!rendered.startsWith(MODULE_SETS_HEADER)) { + throw new GeneralException( + `${MODULE_NAME_GLOBAL_SETTINGS}/settings.yml does not open with the default module set`, + ); + } + return rendered.replace( + MODULE_SETS_HEADER, + `${MODULE_SETS_HEADER} ${CONTAINER_MODULE.id}:\n module_id: ${CONTAINER_MODULE.id}\n`, + ); +} + +/** + * Renders every template into `configDir` and returns the module list from `idea.yml`. + * Template lookup is by module NAME, output directory by module ID - they differ only for + * `virtual-desktop-controller`, whose id is `vdc`. + */ +export function generateConfigFromTemplates( + values: UserValues, + configDir: string, + options: GenerateOptions = {}, +): ModuleEntry[] { + const context = buildContext(values, options); + const env = jinjaEnv(options.templatesDir ?? configTemplatesDirs()); + const renderContext = { ...context, utils: { to_yaml: toYaml } }; + const containers = context.enable_ecs === true; + + mkdirSync(configDir, { recursive: true }); + + const ideaConfig = loadYaml(renderTemplate(env, 'idea.yml', renderContext)) as { + modules: ModuleEntry[]; + }; + if (containers) ideaConfig.modules = withContainerModule(ideaConfig.modules); + const modules = ideaConfig.modules; + + for (const module of modules) { + for (const file of module.config_files) { + let settings = renderTemplate(env, `${module.name}/${file}`, { + ...renderContext, + module_id: module.id, + module_name: module.name, + supported_base_os: SUPPORTED_OS, + }); + if (containers && module.name === MODULE_NAME_GLOBAL_SETTINGS) { + settings = withContainerModuleSet(settings); + } + const settingsFile = join(configDir, module.id, file); + mkdirSync(dirname(settingsFile), { recursive: true }); + writeFileSync(settingsFile, settings); + } + } + + // idea.yml is written back as a normalised dump: no comments, no jinja, and it is this file + // that `config update` and every later read parse. + writeFileSync(join(configDir, 'idea.yml'), toYaml(ideaConfig)); + return modules; +} + +/** Convenience wrapper: read `values.yml` from disk and generate. */ +export function generateConfig( + valuesFile: string, + configDir: string, + options: GenerateOptions = {}, +): ModuleEntry[] { + return generateConfigFromTemplates(loadValuesFile(valuesFile), configDir, options); +} + +/** + * Python indexes and spreads the parsed documents directly, so an empty or non-mapping file stops + * the run with a `TypeError`. Silently treating one as `{}` would let an operator who truncated a + * settings file push a config that is missing a whole module's keys. + */ +function asMapping(value: unknown, file: string): Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new GeneralException(`${file}: expected a YAML mapping, got ${describeYaml(value)}`); + } + return value as Record; +} + +function describeYaml(value: unknown): string { + if (value === null || value === undefined) return 'an empty document'; + if (Array.isArray(value)) return 'a list'; + return `a ${typeof value}`; +} + +export function readModulesFromFiles(configDir: string): ModuleEntry[] { + const file = join(configDir, 'idea.yml'); + const ideaConfig = asMapping(loadYaml(readFileSync(file, 'utf-8')), file); + if (!Array.isArray(ideaConfig.modules)) { + throw new GeneralException(`${file}: expected a modules list`); + } + return ideaConfig.modules as ModuleEntry[]; +} + +/** + * The generated tree as one dict keyed by module id. A module's files are merged shallowly with + * the later file winning - only `cluster` has two files, and their top-level keys are disjoint. + */ +export function readConfigFromFiles(configDir: string): Record { + const config: Record = {}; + for (const module of readModulesFromFiles(configDir)) { + let moduleSettings: Record = {}; + for (const file of module.config_files) { + const settingsFile = join(configDir, module.id, file); + const settings = asMapping(loadYaml(readFileSync(settingsFile, 'utf-8')), settingsFile); + moduleSettings = { ...moduleSettings, ...settings }; + } + config[module.id] = moduleSettings; + } + return config; +} + +/** + * Flattens a nested config into dotted `key` / `value` entries. + * + * The null normalisation happens BEFORE the dict test, exactly as in Python: an empty dict is + * therefore emitted as one NULL leaf rather than recursed into. An empty list stays a list, so it + * reaches DynamoDB as `L` and reads back as `[]` instead of the default. + */ +export function traverseConfig( + entries: ConfigEntry[], + prefix: string, + config: Record, + filterKeyPrefix?: string, +): void { + for (const key of Object.keys(config)) { + if (key.includes('.') || key.includes(':')) { + throw new GeneralException( + `Config key name: ${key} under: ${prefix} cannot contain a dot(.), colon(:) or comma(,)`, + ); + } + + let value = config[key]; + if (isNullValue(value)) value = null; + + const pathPrefix = prefix.length > 0 ? `${prefix}.${key}` : key; + + if (typeof value === 'object' && value !== null && !Array.isArray(value)) { + traverseConfig(entries, pathPrefix, value as Record, filterKeyPrefix); + } else { + if (filterKeyPrefix !== undefined && filterKeyPrefix.length > 0) { + if (!pathPrefix.startsWith(filterKeyPrefix)) continue; + } + entries.push({ key: pathPrefix, value }); + } + } +} + +/** `convert_config_to_key_value_pairs`: the generated tree as flat entries, in template order. */ +export function convertConfigToKeyValuePairs( + configDir: string, + keyPrefix?: string, +): ConfigEntry[] { + const entries: ConfigEntry[] = []; + traverseConfig(entries, '', readConfigFromFiles(configDir), keyPrefix); + return entries; +} + +/** The same entries as a plain object, which is the shape the golden fixtures are captured in. */ +export function flattenConfigDir(configDir: string, keyPrefix?: string): Record { + const flat: Record = {}; + for (const entry of convertConfigToKeyValuePairs(configDir, keyPrefix)) { + flat[entry.key] = entry.value; + } + return flat; +} diff --git a/source/idea/ideactl/src/config/jinja.ts b/source/idea/ideactl/src/config/jinja.ts new file mode 100644 index 00000000..4d1e3d28 --- /dev/null +++ b/source/idea/ideactl/src/config/jinja.ts @@ -0,0 +1,871 @@ +/** + * The nunjucks environment that stands in for the Python `Jinja2Utils.env_using_file_system_loader` + * environment, shared by the config generator, the IAM policy renderer and the bootstrap package + * builder. + * + * nunjucks is close enough to Jinja2 for these templates once the differences below are patched. + * Two kinds of patch live here: + * + * 1. Runtime shims (globals and filters): the `True`/`False`/`None` names Python templates use as + * literals, `lower`/`upper` on a boolean (nunjucks' own filters throw on a non-string), `indent` + * (nunjucks indents the first line and blank lines, Jinja2 3.x indents neither), `tojson` + * (Jinja2's `htmlsafe_json_dumps`, which nunjucks does not have at all), `plus` (Python `+` on + * two lists), `pyindex` (Python negative indexing) and `utils.to_yaml`, which has to emit exactly + * what PyYAML's `yaml.dump` emits because the result is spliced into YAML that is then parsed. + * + * 2. A source rewrite in the loader, for Python forms nunjucks either cannot parse or parses into + * something else without complaining. The dangerous member of that set is `x in ('a', 'b')`: + * nunjucks reads the tuple as a parenthesised expression whose value is its last element, so + * `in` degrades to a substring test that is usually false, every gated block silently + * disappears, and nothing raises. `list + list`, `seq[-1]`, `dict.items()`, `str.lower()` and + * friends, `'sep'.join(seq)` and `{% with %}` are in the same set. + * + * The rewrite walks the source the way a lexer does. It knows the three delimiter kinds + * (`{{ }}`, `{% %}`, `{# #}`), it copies a `{% raw %}` body and a comment through untouched, and + * inside a tag it masks every string literal before any expression rewrite runs, so text that is + * data rather than code is never edited. The `{{-`, `-}}`, `{%-` and `-%}` markers are carried + * across unchanged; the `{%+` spelling is dropped, because it only cancels `lstrip_blocks` and + * `trim_blocks`, which this environment leaves off, and nunjucks does not accept it. + * + * A Python form that has no equivalent this layer can produce raises at load time rather than + * rendering different bytes: see `unsupported`. + * + * Divergence this layer leaves to its callers: Python treats an empty list and an empty dict as + * false, JavaScript treats both as true, and nunjucks compiles conditions straight to JavaScript + * truthiness (`if (expr)`, `a || b`, `a ? b : c` in compiler.js), so no filter or global can + * intercept it. The rewrite could reach it, by turning every truthiness boundary into a call: `a or + * b` into `a if truthy(a) else b`, `a and b` into `b if truthy(a) else a`, `{% if e %}` into + * `{% if truthy(e) %}`. That is not done, because it evaluates an operand twice, so a template + * calling `config.get_list()` in a condition would call it twice, and every boundary would have to + * be found for the result to be trustworthy. Context builders hand empty containers to the renderer + * as `undefined` instead. `pythonTruthy` is exported for that. + */ + +import nunjucks from 'nunjucks'; +import { dump } from 'js-yaml'; + +/** `Utils.to_yaml`: `yaml.dump(json_round_trip(payload), sort_keys=False, width=140)`. */ +export function toYaml(value: unknown): string { + return dump(JSON.parse(JSON.stringify(value ?? null)), { + noRefs: true, + lineWidth: 140, + sortKeys: false, + noCompatMode: true, + }); +} + +/** + * Jinja2 3.x `do_indent(s, width=4, first=False, blank=False)`, including the trailing-newline + * quirk: a newline is appended before splitting, so a value ending in `\n` keeps its final + * (unindented) empty line. A string `width` is the indentation itself, as it is in Jinja2 3.x. + */ +export function jinjaIndent(value: unknown, width: number | string = 4, first = false, blank = false): string { + const indention = typeof width === 'string' ? width : ' '.repeat(width); + const source = `${pythonText(value, 'indent')}\n`; + const lines = source.split('\n'); + lines.pop(); // splitlines(): the final newline does not open a new line + let rv: string; + if (blank) { + rv = lines.join(`\n${indention}`); + } else { + const head = lines.shift() ?? ''; + rv = lines.length ? `${head}\n${lines.map((line) => (line ? indention + line : line)).join('\n')}` : head; + } + return first ? indention + rv : rv; +} + +/** + * Python truthiness for the values a template context carries: an empty string, an empty list, an + * empty dict, `0`, `false`, `null` and `undefined` are all false. JavaScript disagrees about the + * two empty containers, and nunjucks has no hook to correct it, so a context builder whose + * template branches on a list or a dict has to apply this itself. + */ +export function pythonTruthy(value: unknown): boolean { + if (value === undefined || value === null || value === false || value === '' || value === 0) return false; + if (Array.isArray(value)) return value.length > 0; + if (typeof value === 'object') return Object.keys(value as object).length > 0; + return Boolean(value); +} + +/** Python compares strings by code point; JavaScript `<` compares UTF-16 code units. */ +function compareByCodePoint(left: string, right: string): number { + const a = Array.from(left); + const b = Array.from(right); + for (let index = 0; index < Math.min(a.length, b.length); index += 1) { + const difference = (a[index].codePointAt(0) as number) - (b[index].codePointAt(0) as number); + if (difference !== 0) return difference; + } + return a.length - b.length; +} + +/** Python `json.dumps(value, sort_keys=True)`: sorted keys, `', '`/`': '` separators, ASCII only. */ +function pythonJsonDumps(value: unknown): string { + // JSON.stringify already escapes everything Python's ensure_ascii does, bar the non-ASCII range. + const nonAscii = new RegExp('[\\u007f-\\uffff]', 'g'); + const escapeString = (text: string): string => + JSON.stringify(text).replace(nonAscii, (character) => `\\u${character.charCodeAt(0).toString(16).padStart(4, '0')}`); + const write = (node: unknown): string => { + if (node === null || node === undefined) return 'null'; + if (typeof node === 'string') return escapeString(node); + if (typeof node === 'boolean') return node ? 'true' : 'false'; + if (typeof node === 'number') { + // Python's default JSON policy writes the three non-finite floats as bare words. + if (Number.isNaN(node)) return 'NaN'; + if (node === Infinity) return 'Infinity'; + if (node === -Infinity) return '-Infinity'; + // Negative zero is the one JavaScript number that has to have come from a float, so Python's + // float repr applies. A whole float such as `1.0` is indistinguishable from `1` here. + if (Object.is(node, -0)) return '-0.0'; + return JSON.stringify(node); + } + if (Array.isArray(node)) return `[${node.map(write).join(', ')}]`; + const entries = Object.entries(node as Record) + .filter(([, item]) => item !== undefined) + .sort(([left], [right]) => compareByCodePoint(left, right)); + return `{${entries.map(([key, item]) => `${escapeString(key)}: ${write(item)}`).join(', ')}}`; + }; + return write(value); +} + +/** + * Jinja2 `tojson` = `htmlsafe_json_dumps`: `json.dumps` under the environment's default + * `sort_keys=True` policy, then `<`, `>`, `&` and `'` replaced by their `\u` escapes so the result + * is safe inside a `