diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 22fb405e..1a787edc 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -41,6 +41,14 @@ env: AWS_ECS_STAGING_SECRETS_ARN: ${{ secrets.AWS_ECS_STAGING_SECRETS_ARN }} AWS_ECS_STAGING_API_SERVICE_NAME: knowhere-api-staging AWS_ECS_STAGING_WORKER_SERVICE_NAME: knowhere-worker-staging + AWS_ECS_PROD_CLUSTER_NAME: knowhere-fargate + AWS_ECS_PROD_REGION: us-east-1 + AWS_ECS_PROD_EXECUTION_ROLE_ARN: ${{ secrets.AWS_ECS_PROD_EXECUTION_ROLE_ARN }} + AWS_ECS_PROD_API_TASK_ROLE_ARN: ${{ secrets.AWS_ECS_PROD_API_TASK_ROLE_ARN }} + AWS_ECS_PROD_WORKER_TASK_ROLE_ARN: ${{ secrets.AWS_ECS_PROD_WORKER_TASK_ROLE_ARN }} + AWS_ECS_PROD_SECRETS_ARN: ${{ secrets.AWS_ECS_PROD_SECRETS_ARN }} + AWS_ECS_PROD_API_SERVICE_NAME: knowhere-api-prod + AWS_ECS_PROD_WORKER_SERVICE_NAME: knowhere-worker-prod jobs: build-and-publish: @@ -79,6 +87,22 @@ jobs: with: persist-credentials: false + # Production releases are cut from main. This keeps the new ECS path + # and the retained EKS rollback image on the same reviewed source line. + - name: Verify production release source + if: ${{ github.event_name == 'release' }} + env: + RELEASE_TAG: ${{ github.event.release.tag_name }} + shell: bash + run: | + set -euo pipefail + git fetch origin main --no-tags + release_commit="$(git rev-parse "${RELEASE_TAG}^{commit}")" + if ! git merge-base --is-ancestor "${release_commit}" origin/main; then + echo "::error::Release ${RELEASE_TAG} does not point to a commit on main" + exit 1 + fi + - name: Decide build context id: context shell: bash @@ -299,6 +323,62 @@ jobs: "$image_uri" \ -m alembic upgrade heads + migrate-ecs-production: + name: Run production Aurora database migration + runs-on: ubuntu-latest + needs: build-and-publish + if: ${{ github.event_name == 'release' && github.event.action == 'published' }} + permissions: + contents: read + + steps: + - name: Validate production migration configuration + shell: bash + run: | + set -euo pipefail + if [ -z "${{ secrets.AWS_ACCESS_KEY_ID }}" ] || [ -z "${{ secrets.AWS_SECRET_ACCESS_KEY }}" ]; then + echo "::error::Production migration requires AWS ECR credentials." + exit 1 + fi + if [ -z "${{ secrets.PRODUCTION_MIGRATION_DATABASE_URL }}" ]; then + echo "::error::PRODUCTION_MIGRATION_DATABASE_URL is not configured." + exit 1 + fi + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_ECS_PROD_REGION }} + + - name: Login to ECR + shell: bash + run: | + set -euo pipefail + aws ecr get-login-password --region "${AWS_ECS_PROD_REGION}" \ + | docker login --username AWS --password-stdin "${ECR_REGISTRY}" + + - name: Run migration against production Aurora + env: + DATABASE_URL: ${{ secrets.PRODUCTION_MIGRATION_DATABASE_URL }} + S3_BUCKET_NAME: knowhere-storage-prod + S3_TEMP_PATH: /tmp + TMP_PATH: /tmp/aismart_bid + shell: bash + run: | + set -euo pipefail + image_uri="${ECR_REGISTRY}/${ECR_REPOSITORY}/knowhere-backend:${{ github.event.release.tag_name }}-prod" + docker run --rm \ + --env DATABASE_URL \ + --env S3_BUCKET_NAME \ + --env TMP_PATH \ + --env S3_TEMP_PATH \ + --env DB_SSL_MODE=disable \ + --entrypoint python \ + "${image_uri}" \ + -m alembic upgrade heads + migrate-eks-staging: name: Run EKS rollback database migration runs-on: ubuntu-latest @@ -556,11 +636,29 @@ jobs: EXECUTION_ROLE_ARN: ${{ env.AWS_ECS_STAGING_EXECUTION_ROLE_ARN }} API_TASK_ROLE_ARN: ${{ env.AWS_ECS_STAGING_API_TASK_ROLE_ARN }} WORKER_TASK_ROLE_ARN: ${{ env.AWS_ECS_STAGING_WORKER_TASK_ROLE_ARN }} - STAGING_SECRETS_ARN: ${{ env.AWS_ECS_STAGING_SECRETS_ARN }} + SECRETS_ARN: ${{ env.AWS_ECS_STAGING_SECRETS_ARN }} + DEPLOYMENT_ENVIRONMENT: staging + RUNTIME_ENVIRONMENT: staging + APP_ENV: staging + DB_SSL_MODE: require + API_DB_POOL_SIZE: "5" + API_DB_MAX_OVERFLOW: "5" + WORKER_DB_SYNC_POOL_SIZE: "2" + WORKER_DB_SYNC_MAX_OVERFLOW: "2" + S3_BUCKET_NAME: knowhere-storage-staging + INTERNAL_DASHBOARD_ENDPOINT: https://staging.knowhereto.ai + FRONTEND_URL: https://staging.knowhereto.ai + API_WEBHOOK_ENDPOINT: https://api-staging.knowhereto.ai/v1/internal/s3-events + SNS_TOPIC_ARN: arn:aws:sns:us-east-1:107424103509:knowhere-staging-s3-events + QSTASH_CALLBACK_BASE_URL: https://api-staging.knowhereto.ai/api/v1 + WORKER_CPU: "2048" + WORKER_MEMORY: "4096" shell: bash run: | set -euo pipefail - python3 deploy/ecs/render_task_definitions.py --output-dir "$RUNNER_TEMP/ecs-task-definitions" + python3 deploy/ecs/render_task_definitions.py \ + --environment staging \ + --output-dir "$RUNNER_TEMP/ecs-task-definitions" - name: Register ECS task definitions id: task-definitions @@ -628,13 +726,230 @@ jobs: echo "API image: ${{ steps.images.outputs.api_image }}" echo "Worker image: ${{ steps.images.outputs.worker_image }}" + deploy-ecs-production: + name: Deploy production services to ECS + runs-on: ubuntu-latest + needs: [build-and-publish, migrate-ecs-production] + if: >- + ${{ always() && github.event_name == 'release' && + needs.build-and-publish.result == 'success' && + needs['migrate-ecs-production'].result == 'success' }} + permissions: + contents: read + + steps: + - name: Checkout released source + uses: actions/checkout@v4 + with: + persist-credentials: false + ref: ${{ github.event.release.tag_name }} + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_ECS_PROD_REGION }} + + - name: Validate ECS production prerequisites + shell: bash + run: | + set -euo pipefail + + required_values=( + AWS_ECS_PROD_EXECUTION_ROLE_ARN + AWS_ECS_PROD_API_TASK_ROLE_ARN + AWS_ECS_PROD_WORKER_TASK_ROLE_ARN + AWS_ECS_PROD_SECRETS_ARN + ) + missing=() + for variable_name in "${required_values[@]}"; do + if [ -z "${!variable_name}" ]; then + missing+=("${variable_name}") + fi + done + if [ "${#missing[@]}" -gt 0 ]; then + printf 'Missing ECS production configuration: %s\n' "${missing[*]}" >&2 + exit 1 + fi + + cluster_status="$(aws ecs describe-clusters \ + --cluster "${AWS_ECS_PROD_CLUSTER_NAME}" \ + --query 'clusters[0].status' --output text)" + if [ "${cluster_status}" != "ACTIVE" ]; then + echo "::error::ECS cluster ${AWS_ECS_PROD_CLUSTER_NAME} is not ACTIVE (status: ${cluster_status})." + exit 1 + fi + + for role_arn in \ + "${AWS_ECS_PROD_EXECUTION_ROLE_ARN}" \ + "${AWS_ECS_PROD_API_TASK_ROLE_ARN}" \ + "${AWS_ECS_PROD_WORKER_TASK_ROLE_ARN}"; do + aws iam get-role --role-name "${role_arn##*/}" --query 'Role.Arn' --output text >/dev/null + done + + aws secretsmanager describe-secret \ + --secret-id "${AWS_ECS_PROD_SECRETS_ARN}" \ + --query 'ARN' --output text >/dev/null + + for log_group in /ecs/knowhere-api-prod /ecs/knowhere-worker-prod; do + found="$(aws logs describe-log-groups \ + --log-group-name-prefix "${log_group}" \ + --query 'logGroups[0].logGroupName' --output text)" + if [ "${found}" != "${log_group}" ]; then + echo "::error::Required CloudWatch log group ${log_group} does not exist." + exit 1 + fi + done + + for service in \ + "${AWS_ECS_PROD_API_SERVICE_NAME}" \ + "${AWS_ECS_PROD_WORKER_SERVICE_NAME}"; do + service_json="$(aws ecs describe-services \ + --cluster "${AWS_ECS_PROD_CLUSTER_NAME}" \ + --services "${service}" --output json)" + status="$(jq -r '.services[0].status // "MISSING"' <<<"${service_json}")" + if [ "${status}" != "ACTIVE" ]; then + echo "::error::Required ECS service ${service} is not ACTIVE (status: ${status})." + exit 1 + fi + network_count="$(jq '[.services[0].networkConfiguration.awsvpcConfiguration.subnets // [] | length] | add' <<<"${service_json}")" + security_group_count="$(jq '[.services[0].networkConfiguration.awsvpcConfiguration.securityGroups // [] | length] | add' <<<"${service_json}")" + if [ "${network_count}" -lt 1 ] || [ "${security_group_count}" -lt 1 ]; then + echo "::error::ECS service ${service} has no usable awsvpc subnet/security-group configuration." + exit 1 + fi + done + + api_load_balancer_count="$(aws ecs describe-services \ + --cluster "${AWS_ECS_PROD_CLUSTER_NAME}" \ + --services "${AWS_ECS_PROD_API_SERVICE_NAME}" \ + --query 'length(services[0].loadBalancers)' --output text)" + if [ "${api_load_balancer_count}" -lt 1 ]; then + echo "::error::Production API ECS service has no load balancer target configured." + exit 1 + fi + + - name: Resolve immutable ECR image digests + id: images + shell: bash + run: | + set -euo pipefail + image_tag="${{ github.event.release.tag_name }}-prod" + + resolve_image() { + local repository="$1" + local output_name="$2" + local digest + digest="$(aws ecr describe-images \ + --repository-name "${ECR_REPOSITORY}/${repository}" \ + --image-ids "imageTag=${image_tag}" \ + --query 'imageDetails[0].imageDigest' --output text)" + if [ -z "${digest}" ] || [ "${digest}" = "None" ]; then + echo "::error::No ECR image found for ${repository}:${image_tag}." + exit 1 + fi + echo "${output_name}=${ECR_REGISTRY}/${ECR_REPOSITORY}/${repository}@${digest}" >> "${GITHUB_OUTPUT}" + } + + resolve_image knowhere-backend api_image + resolve_image knowhere-worker worker_image + + - name: Render ECS task definitions + env: + API_IMAGE: ${{ steps.images.outputs.api_image }} + WORKER_IMAGE: ${{ steps.images.outputs.worker_image }} + EXECUTION_ROLE_ARN: ${{ env.AWS_ECS_PROD_EXECUTION_ROLE_ARN }} + API_TASK_ROLE_ARN: ${{ env.AWS_ECS_PROD_API_TASK_ROLE_ARN }} + WORKER_TASK_ROLE_ARN: ${{ env.AWS_ECS_PROD_WORKER_TASK_ROLE_ARN }} + SECRETS_ARN: ${{ env.AWS_ECS_PROD_SECRETS_ARN }} + DEPLOYMENT_ENVIRONMENT: prod + RUNTIME_ENVIRONMENT: production + APP_ENV: production + DB_SSL_MODE: disable + API_DB_POOL_SIZE: "50" + API_DB_MAX_OVERFLOW: "50" + WORKER_DB_SYNC_POOL_SIZE: "5" + WORKER_DB_SYNC_MAX_OVERFLOW: "5" + S3_BUCKET_NAME: knowhere-storage-prod + INTERNAL_DASHBOARD_ENDPOINT: https://knowhereto.ai + FRONTEND_URL: https://knowhereto.ai + API_WEBHOOK_ENDPOINT: https://api.knowhereto.ai/v1/internal/s3-events + SNS_TOPIC_ARN: arn:aws:sns:us-east-1:107424103509:knowhere-prod-s3-events + QSTASH_CALLBACK_BASE_URL: https://api.knowhereto.ai/api/v1 + WORKER_CPU: "2048" + WORKER_MEMORY: "4096" + shell: bash + run: | + set -euo pipefail + python3 deploy/ecs/render_task_definitions.py \ + --environment production \ + --output-dir "${RUNNER_TEMP}/ecs-task-definitions" + + - name: Register ECS task definitions + id: task-definitions + shell: bash + run: | + set -euo pipefail + api_task_definition_arn="$(aws ecs register-task-definition \ + --cli-input-json "file://${RUNNER_TEMP}/ecs-task-definitions/knowhere-api-prod.json" \ + --query 'taskDefinition.taskDefinitionArn' --output text)" + worker_task_definition_arn="$(aws ecs register-task-definition \ + --cli-input-json "file://${RUNNER_TEMP}/ecs-task-definitions/knowhere-worker-prod.json" \ + --query 'taskDefinition.taskDefinitionArn' --output text)" + echo "api_task_definition_arn=${api_task_definition_arn}" >> "${GITHUB_OUTPUT}" + echo "worker_task_definition_arn=${worker_task_definition_arn}" >> "${GITHUB_OUTPUT}" + + - name: Update API ECS service on on-demand Fargate + shell: bash + run: | + set -euo pipefail + aws ecs update-service \ + --cluster "${AWS_ECS_PROD_CLUSTER_NAME}" \ + --service "${AWS_ECS_PROD_API_SERVICE_NAME}" \ + --task-definition "${{ steps.task-definitions.outputs.api_task_definition_arn }}" \ + --capacity-provider-strategy capacityProvider=FARGATE,weight=1 + + - name: Update worker ECS service on Fargate Spot + shell: bash + run: | + set -euo pipefail + aws ecs update-service \ + --cluster "${AWS_ECS_PROD_CLUSTER_NAME}" \ + --service "${AWS_ECS_PROD_WORKER_SERVICE_NAME}" \ + --task-definition "${{ steps.task-definitions.outputs.worker_task_definition_arn }}" \ + --capacity-provider-strategy capacityProvider=FARGATE_SPOT,weight=1 + + - name: Wait for API ECS service to stabilize + shell: bash + run: | + aws ecs wait services-stable \ + --cluster "${AWS_ECS_PROD_CLUSTER_NAME}" \ + --services "${AWS_ECS_PROD_API_SERVICE_NAME}" + + - name: Wait for worker ECS service to stabilize + shell: bash + run: | + aws ecs wait services-stable \ + --cluster "${AWS_ECS_PROD_CLUSTER_NAME}" \ + --services "${AWS_ECS_PROD_WORKER_SERVICE_NAME}" + + - name: Summarize ECS production deployment + shell: bash + run: | + echo "Environment: production" + echo "Cluster: ${AWS_ECS_PROD_CLUSTER_NAME}" + echo "API image: ${{ steps.images.outputs.api_image }}" + echo "Worker image: ${{ steps.images.outputs.worker_image }}" + deploy-eks: runs-on: ubuntu-latest - needs: [build-and-publish, migrate-eks-staging] + needs: [build-and-publish, migrate-eks-staging, migrate-ecs-production] if: >- ${{ always() && github.event_name != 'pull_request' && needs.build-and-publish.result == 'success' && (needs['migrate-eks-staging'].result == 'success' || needs['migrate-eks-staging'].result == 'skipped') && + (needs['migrate-ecs-production'].result == 'success' || needs['migrate-ecs-production'].result == 'skipped') && (github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.deployment_target == 'eks-staging-rollback')) }} @@ -761,10 +1076,12 @@ jobs: release: name: Attach deployment release assets runs-on: ubuntu-latest - needs: deploy-eks + needs: [deploy-eks, deploy-ecs-production] if: >- ${{ github.event_name == 'release' && - github.event.action == 'published' }} + github.event.action == 'published' && + needs.deploy-eks.result == 'success' && + needs.deploy-ecs-production.result == 'success' }} permissions: contents: write steps: diff --git a/deploy/ecs/README.md b/deploy/ecs/README.md index 564611fb..e19200af 100644 --- a/deploy/ecs/README.md +++ b/deploy/ecs/README.md @@ -2,9 +2,9 @@ These files are deployment templates for the shared `knowhere-fargate` cluster. They do not contain secret values and are not registered automatically. -The staging templates intentionally omit `S3_ACCESS_KEY_ID` and `S3_SECRET_ACCESS_KEY`. With `S3_TYPE=s3`, boto3 obtains temporary authenticated credentials from the ECS task role. +The templates intentionally omit `S3_ACCESS_KEY_ID` and `S3_SECRET_ACCESS_KEY`. With `S3_TYPE=s3`, boto3 obtains temporary authenticated credentials from the ECS task role. The same templates render staging or production values; the release workflow selects production only for a published release whose tag points to `main`. -The single staging Secrets Manager secret supplied to the renderer must be a JSON secret with these keys: +The environment-specific Secrets Manager secret supplied to the renderer must be a JSON secret with these keys: - API: `DATABASE_URL`, `REDIS_HOST`, `REDIS_PORT`, `REDIS_PASSWORD`, `CELERY_REDIS_URL`, `SECRET_KEY`, `DS_KEY`, `ALI_API_KEYS`, `ARK_API_KEY`, `GPT_API_KEY`, `MINERU_API_KEYS`, `STRIPE_SECRET_KEY`, `STRIPE_WEBHOOK_SECRET`, `WEBHOOK_MASTER_KEY`, `LOGFIRE_TOKEN`, `QSTASH_TOKEN`, `QSTASH_CURRENT_SIGNING_KEY`, `QSTASH_NEXT_SIGNING_KEY` - Worker: the API keys above plus `CELERY_REDIS_PASSWORD` and `ILOVEAPI_KEYS` @@ -17,8 +17,24 @@ WORKER_IMAGE=107424103509.dkr.ecr.us-east-1.amazonaws.com/knowhere/knowhere-work EXECUTION_ROLE_ARN=arn:aws:iam::107424103509:role/knowhere-fargate-staging-execution-role \ API_TASK_ROLE_ARN=arn:aws:iam::107424103509:role/knowhere-api-staging-task-role \ WORKER_TASK_ROLE_ARN=arn:aws:iam::107424103509:role/knowhere-worker-staging-task-role \ -STAGING_SECRETS_ARN=arn:aws:secretsmanager:us-east-1:107424103509:secret:knowhere/staging/runtime-... \ -python deploy/ecs/render_task_definitions.py --output-dir /tmp/knowhere-ecs-rendered +SECRETS_ARN=arn:aws:secretsmanager:us-east-1:107424103509:secret:knowhere/staging/runtime-... \ +DEPLOYMENT_ENVIRONMENT=staging \ +RUNTIME_ENVIRONMENT=staging \ +APP_ENV=staging \ +DB_SSL_MODE=require \ +API_DB_POOL_SIZE=5 \ +API_DB_MAX_OVERFLOW=5 \ +WORKER_DB_SYNC_POOL_SIZE=2 \ +WORKER_DB_SYNC_MAX_OVERFLOW=2 \ +S3_BUCKET_NAME=knowhere-storage-staging \ +INTERNAL_DASHBOARD_ENDPOINT=https://staging.knowhereto.ai \ +FRONTEND_URL=https://staging.knowhereto.ai \ +API_WEBHOOK_ENDPOINT=https://api-staging.knowhereto.ai/v1/internal/s3-events \ +SNS_TOPIC_ARN=arn:aws:sns:us-east-1:107424103509:knowhere-staging-s3-events \ +QSTASH_CALLBACK_BASE_URL=https://api-staging.knowhereto.ai/api/v1 \ +WORKER_CPU=2048 \ +WORKER_MEMORY=4096 \ +python deploy/ecs/render_task_definitions.py --environment staging --output-dir /tmp/knowhere-ecs-rendered ``` The output directory is deployment-only and must not be committed. The renderer fails on missing inputs, unresolved placeholders, or either long-lived S3 credential variable. @@ -33,6 +49,26 @@ The staging workflow in `.github/workflows/build-images.yml` expects these GitHu Before an ECS staging deployment, an operator must create and verify the ECS services, network configuration, API load-balancer target, CloudWatch log groups, and runtime secret. The workflow validates those resources and fails without registering or updating a service when any prerequisite is missing. It does not create or delete AWS resources. +## Production workflow prerequisites + +The production release path uses the same ECS cluster and renders the approved +2 vCPU / 4 GiB worker task size with `WORKER_CONCURRENCY=10`. It requires these +additional GitHub Actions secrets: + +- `AWS_ECS_PROD_EXECUTION_ROLE_ARN` +- `AWS_ECS_PROD_API_TASK_ROLE_ARN` +- `AWS_ECS_PROD_WORKER_TASK_ROLE_ARN` +- `AWS_ECS_PROD_SECRETS_ARN` +- `PRODUCTION_MIGRATION_DATABASE_URL` (direct migration-only database URL) + +Before publishing the production release, an operator must create and verify +the production runtime secret, IAM roles, API and worker ECS services, API +target group/listener rule, production ACM certificate attachment, and +`/ecs/knowhere-api-prod` and `/ecs/knowhere-worker-prod` log groups. The release +workflow validates these resources, registers immutable image-digest task +definitions, runs the production migration first, and then updates the ECS +services. It does not create or delete AWS resources. + ## Manual staging availability `.github/workflows/manage-staging.yml` exposes three manually dispatched diff --git a/deploy/ecs/render_task_definitions.py b/deploy/ecs/render_task_definitions.py index 4312533c..458cb8a7 100644 --- a/deploy/ecs/render_task_definitions.py +++ b/deploy/ecs/render_task_definitions.py @@ -1,4 +1,4 @@ -"""Render staging ECS task-definition templates without storing secrets.""" +"""Render ECS task-definition templates without storing secrets.""" from __future__ import annotations @@ -17,7 +17,23 @@ "EXECUTION_ROLE_ARN", "API_TASK_ROLE_ARN", "WORKER_TASK_ROLE_ARN", - "STAGING_SECRETS_ARN", + "SECRETS_ARN", + "DEPLOYMENT_ENVIRONMENT", + "RUNTIME_ENVIRONMENT", + "APP_ENV", + "DB_SSL_MODE", + "API_DB_POOL_SIZE", + "API_DB_MAX_OVERFLOW", + "WORKER_DB_SYNC_POOL_SIZE", + "WORKER_DB_SYNC_MAX_OVERFLOW", + "S3_BUCKET_NAME", + "INTERNAL_DASHBOARD_ENDPOINT", + "FRONTEND_URL", + "API_WEBHOOK_ENDPOINT", + "SNS_TOPIC_ARN", + "QSTASH_CALLBACK_BASE_URL", + "WORKER_CPU", + "WORKER_MEMORY", ) FORBIDDEN_ENVIRONMENT_NAMES: Final[frozenset[str]] = frozenset( {"S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"} @@ -86,6 +102,12 @@ def parse_arguments() -> argparse.Namespace: """Parse renderer CLI arguments.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument( + "--environment", + choices=("staging", "production"), + required=True, + help="Deployment environment represented by the rendered definitions.", + ) return parser.parse_args() @@ -99,18 +121,25 @@ def load_variables() -> dict[str, str]: def main() -> None: - """Render both staging task definitions.""" + """Render both API and worker task definitions.""" arguments = parse_arguments() variables = load_variables() + expected_environment = "staging" if arguments.environment == "staging" else "prod" + if variables["DEPLOYMENT_ENVIRONMENT"] != expected_environment: + raise ValueError( + "DEPLOYMENT_ENVIRONMENT must match the selected environment: " + f"expected {expected_environment}, got {variables['DEPLOYMENT_ENVIRONMENT']}" + ) template_directory = Path(__file__).parent + output_suffix = variables["DEPLOYMENT_ENVIRONMENT"] render_template( template_directory / "task-definition-api.staging.json", - arguments.output_dir / "knowhere-api-staging.json", + arguments.output_dir / f"knowhere-api-{output_suffix}.json", variables, ) render_template( template_directory / "task-definition-worker.staging.json", - arguments.output_dir / "knowhere-worker-staging.json", + arguments.output_dir / f"knowhere-worker-{output_suffix}.json", variables, ) diff --git a/deploy/ecs/task-definition-api.staging.json b/deploy/ecs/task-definition-api.staging.json index 71025dd3..0e1013c9 100644 --- a/deploy/ecs/task-definition-api.staging.json +++ b/deploy/ecs/task-definition-api.staging.json @@ -1,5 +1,5 @@ { - "family": "knowhere-api-staging", + "family": "knowhere-api-${DEPLOYMENT_ENVIRONMENT}", "taskRoleArn": "${API_TASK_ROLE_ARN}", "executionRoleArn": "${EXECUTION_ROLE_ARN}", "networkMode": "awsvpc", @@ -24,15 +24,15 @@ } ], "environment": [ - {"name": "ENVIRONMENT", "value": "staging"}, - {"name": "APP_ENV", "value": "staging"}, - {"name": "DB_SSL_MODE", "value": "require"}, - {"name": "DB_POOL_SIZE", "value": "5"}, - {"name": "DB_MAX_OVERFLOW", "value": "5"}, + {"name": "ENVIRONMENT", "value": "${RUNTIME_ENVIRONMENT}"}, + {"name": "APP_ENV", "value": "${APP_ENV}"}, + {"name": "DB_SSL_MODE", "value": "${DB_SSL_MODE}"}, + {"name": "DB_POOL_SIZE", "value": "${API_DB_POOL_SIZE}"}, + {"name": "DB_MAX_OVERFLOW", "value": "${API_DB_MAX_OVERFLOW}"}, {"name": "TMP_PATH", "value": "/tmp/aismart_bid"}, {"name": "S3_TYPE", "value": "s3"}, - {"name": "S3_BUCKET_NAME", "value": "knowhere-storage-staging"}, - {"name": "S3_RESULTS_BUCKET", "value": "knowhere-storage-staging"}, + {"name": "S3_BUCKET_NAME", "value": "${S3_BUCKET_NAME}"}, + {"name": "S3_RESULTS_BUCKET", "value": "${S3_BUCKET_NAME}"}, {"name": "S3_TEMP_PATH", "value": "/tmp"}, {"name": "S3_REGION", "value": "us-east-1"}, {"name": "S3_USE_SSL", "value": "true"}, @@ -48,32 +48,34 @@ {"name": "RATE_LIMIT_ENABLED", "value": "true"}, {"name": "TELEMETRY_ENABLED", "value": "false"}, {"name": "API_STANDALONE_MODE_ENABLED", "value": "true"}, - {"name": "INTERNAL_DASHBOARD_ENDPOINT", "value": "https://staging.knowhereto.ai"}, - {"name": "FRONTEND_URL", "value": "https://staging.knowhereto.ai"}, + {"name": "INTERNAL_DASHBOARD_ENDPOINT", "value": "${INTERNAL_DASHBOARD_ENDPOINT}"}, + {"name": "FRONTEND_URL", "value": "${FRONTEND_URL}"}, {"name": "BILLING_ENABLED", "value": "true"}, - {"name": "QSTASH_CALLBACK_BASE_URL", "value": "https://api-staging.knowhereto.ai/api/v1"}, + {"name": "QSTASH_CALLBACK_BASE_URL", "value": "${QSTASH_CALLBACK_BASE_URL}"}, {"name": "AWS_REGION", "value": "us-east-1"}, - {"name": "AWS_ACCOUNT_ID", "value": "107424103509"} + {"name": "AWS_ACCOUNT_ID", "value": "107424103509"}, + {"name": "API_WEBHOOK_ENDPOINT", "value": "${API_WEBHOOK_ENDPOINT}"}, + {"name": "SNS_TOPIC_ARN", "value": "${SNS_TOPIC_ARN}"} ], "secrets": [ - {"name": "DATABASE_URL", "valueFrom": "${STAGING_SECRETS_ARN}:DATABASE_URL::"}, - {"name": "REDIS_HOST", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_HOST::"}, - {"name": "REDIS_PORT", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_PORT::"}, - {"name": "REDIS_PASSWORD", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_PASSWORD::"}, - {"name": "CELERY_REDIS_URL", "valueFrom": "${STAGING_SECRETS_ARN}:CELERY_REDIS_URL::"}, - {"name": "SECRET_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:SECRET_KEY::"}, - {"name": "DS_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:DS_KEY::"}, - {"name": "ALI_API_KEYS", "valueFrom": "${STAGING_SECRETS_ARN}:ALI_API_KEYS::"}, - {"name": "ARK_API_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:ARK_API_KEY::"}, - {"name": "GPT_API_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:GPT_API_KEY::"}, - {"name": "MINERU_API_KEYS", "valueFrom": "${STAGING_SECRETS_ARN}:MINERU_API_KEYS::"}, - {"name": "STRIPE_SECRET_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:STRIPE_SECRET_KEY::"}, - {"name": "STRIPE_WEBHOOK_SECRET", "valueFrom": "${STAGING_SECRETS_ARN}:STRIPE_WEBHOOK_SECRET::"}, - {"name": "WEBHOOK_MASTER_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:WEBHOOK_MASTER_KEY::"}, - {"name": "LOGFIRE_TOKEN", "valueFrom": "${STAGING_SECRETS_ARN}:LOGFIRE_TOKEN::"}, - {"name": "QSTASH_TOKEN", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_TOKEN::"}, - {"name": "QSTASH_CURRENT_SIGNING_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_CURRENT_SIGNING_KEY::"}, - {"name": "QSTASH_NEXT_SIGNING_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_NEXT_SIGNING_KEY::"} + {"name": "DATABASE_URL", "valueFrom": "${SECRETS_ARN}:DATABASE_URL::"}, + {"name": "REDIS_HOST", "valueFrom": "${SECRETS_ARN}:REDIS_HOST::"}, + {"name": "REDIS_PORT", "valueFrom": "${SECRETS_ARN}:REDIS_PORT::"}, + {"name": "REDIS_PASSWORD", "valueFrom": "${SECRETS_ARN}:REDIS_PASSWORD::"}, + {"name": "CELERY_REDIS_URL", "valueFrom": "${SECRETS_ARN}:CELERY_REDIS_URL::"}, + {"name": "SECRET_KEY", "valueFrom": "${SECRETS_ARN}:SECRET_KEY::"}, + {"name": "DS_KEY", "valueFrom": "${SECRETS_ARN}:DS_KEY::"}, + {"name": "ALI_API_KEYS", "valueFrom": "${SECRETS_ARN}:ALI_API_KEYS::"}, + {"name": "ARK_API_KEY", "valueFrom": "${SECRETS_ARN}:ARK_API_KEY::"}, + {"name": "GPT_API_KEY", "valueFrom": "${SECRETS_ARN}:GPT_API_KEY::"}, + {"name": "MINERU_API_KEYS", "valueFrom": "${SECRETS_ARN}:MINERU_API_KEYS::"}, + {"name": "STRIPE_SECRET_KEY", "valueFrom": "${SECRETS_ARN}:STRIPE_SECRET_KEY::"}, + {"name": "STRIPE_WEBHOOK_SECRET", "valueFrom": "${SECRETS_ARN}:STRIPE_WEBHOOK_SECRET::"}, + {"name": "WEBHOOK_MASTER_KEY", "valueFrom": "${SECRETS_ARN}:WEBHOOK_MASTER_KEY::"}, + {"name": "LOGFIRE_TOKEN", "valueFrom": "${SECRETS_ARN}:LOGFIRE_TOKEN::"}, + {"name": "QSTASH_TOKEN", "valueFrom": "${SECRETS_ARN}:QSTASH_TOKEN::"}, + {"name": "QSTASH_CURRENT_SIGNING_KEY", "valueFrom": "${SECRETS_ARN}:QSTASH_CURRENT_SIGNING_KEY::"}, + {"name": "QSTASH_NEXT_SIGNING_KEY", "valueFrom": "${SECRETS_ARN}:QSTASH_NEXT_SIGNING_KEY::"} ], "healthCheck": { "command": ["CMD-SHELL", "curl -f http://localhost:5005/health || exit 1"], @@ -85,7 +87,7 @@ "logConfiguration": { "logDriver": "awslogs", "options": { - "awslogs-group": "/ecs/knowhere-api-staging", + "awslogs-group": "/ecs/knowhere-api-${DEPLOYMENT_ENVIRONMENT}", "awslogs-region": "us-east-1", "awslogs-stream-prefix": "api" } @@ -94,7 +96,7 @@ ], "tags": [ {"key": "Project", "value": "knowhere"}, - {"key": "Environment", "value": "staging"}, + {"key": "Environment", "value": "${DEPLOYMENT_ENVIRONMENT}"}, {"key": "Service", "value": "api"}, {"key": "ManagedBy", "value": "knowhere-api-infra"} ] diff --git a/deploy/ecs/task-definition-worker.staging.json b/deploy/ecs/task-definition-worker.staging.json index f0c2a709..202339d1 100644 --- a/deploy/ecs/task-definition-worker.staging.json +++ b/deploy/ecs/task-definition-worker.staging.json @@ -1,11 +1,11 @@ { - "family": "knowhere-worker-staging", + "family": "knowhere-worker-${DEPLOYMENT_ENVIRONMENT}", "taskRoleArn": "${WORKER_TASK_ROLE_ARN}", "executionRoleArn": "${EXECUTION_ROLE_ARN}", "networkMode": "awsvpc", "requiresCompatibilities": ["FARGATE"], - "cpu": "2048", - "memory": "4096", + "cpu": "${WORKER_CPU}", + "memory": "${WORKER_MEMORY}", "ephemeralStorage": {"sizeInGiB": 21}, "runtimePlatform": { "cpuArchitecture": "X86_64", @@ -18,16 +18,16 @@ "essential": true, "stopTimeout": 120, "environment": [ - {"name": "ENVIRONMENT", "value": "staging"}, - {"name": "APP_ENV", "value": "staging"}, - {"name": "DB_SSL_MODE", "value": "require"}, - {"name": "DB_SYNC_POOL_SIZE", "value": "2"}, - {"name": "DB_SYNC_MAX_OVERFLOW", "value": "2"}, + {"name": "ENVIRONMENT", "value": "${RUNTIME_ENVIRONMENT}"}, + {"name": "APP_ENV", "value": "${APP_ENV}"}, + {"name": "DB_SSL_MODE", "value": "${DB_SSL_MODE}"}, + {"name": "DB_SYNC_POOL_SIZE", "value": "${WORKER_DB_SYNC_POOL_SIZE}"}, + {"name": "DB_SYNC_MAX_OVERFLOW", "value": "${WORKER_DB_SYNC_MAX_OVERFLOW}"}, {"name": "WORKER_CONCURRENCY", "value": "10"}, {"name": "TMP_PATH", "value": "/tmp/aismart_bid"}, {"name": "S3_TYPE", "value": "s3"}, - {"name": "S3_BUCKET_NAME", "value": "knowhere-storage-staging"}, - {"name": "S3_RESULTS_BUCKET", "value": "knowhere-storage-staging"}, + {"name": "S3_BUCKET_NAME", "value": "${S3_BUCKET_NAME}"}, + {"name": "S3_RESULTS_BUCKET", "value": "${S3_BUCKET_NAME}"}, {"name": "S3_TEMP_PATH", "value": "/tmp"}, {"name": "S3_REGION", "value": "us-east-1"}, {"name": "S3_USE_SSL", "value": "true"}, @@ -45,10 +45,10 @@ {"name": "AWS_ACCOUNT_ID", "value": "107424103509"}, {"name": "RATE_LIMIT_ENABLED", "value": "false"}, {"name": "TELEMETRY_ENABLED", "value": "false"}, - {"name": "INTERNAL_DASHBOARD_ENDPOINT", "value": "https://staging.knowhereto.ai"}, - {"name": "API_WEBHOOK_ENDPOINT", "value": "https://api-staging.knowhereto.ai/v1/internal/s3-events"}, - {"name": "SNS_TOPIC_ARN", "value": "arn:aws:sns:us-east-1:107424103509:knowhere-staging-s3-events"}, - {"name": "QSTASH_CALLBACK_BASE_URL", "value": "https://api-staging.knowhereto.ai/api/v1"}, + {"name": "INTERNAL_DASHBOARD_ENDPOINT", "value": "${INTERNAL_DASHBOARD_ENDPOINT}"}, + {"name": "API_WEBHOOK_ENDPOINT", "value": "${API_WEBHOOK_ENDPOINT}"}, + {"name": "SNS_TOPIC_ARN", "value": "${SNS_TOPIC_ARN}"}, + {"name": "QSTASH_CALLBACK_BASE_URL", "value": "${QSTASH_CALLBACK_BASE_URL}"}, {"name": "HF_HOME", "value": "/mnt/models/huggingface"}, {"name": "TRANSFORMERS_CACHE", "value": "/mnt/models/huggingface"}, {"name": "BILLING_ENABLED", "value": "true"}, @@ -64,24 +64,24 @@ {"name": "ALL_DF_COLS", "value": "content,path,type,length,keywords,summary,know_id,tokens,connectto,addtime,page_nums"} ], "secrets": [ - {"name": "DATABASE_URL", "valueFrom": "${STAGING_SECRETS_ARN}:DATABASE_URL::"}, - {"name": "REDIS_HOST", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_HOST::"}, - {"name": "REDIS_PORT", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_PORT::"}, - {"name": "REDIS_PASSWORD", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_PASSWORD::"}, - {"name": "CELERY_REDIS_URL", "valueFrom": "${STAGING_SECRETS_ARN}:CELERY_REDIS_URL::"}, - {"name": "CELERY_REDIS_PASSWORD", "valueFrom": "${STAGING_SECRETS_ARN}:CELERY_REDIS_PASSWORD::"}, - {"name": "SECRET_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:SECRET_KEY::"}, - {"name": "WEBHOOK_MASTER_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:WEBHOOK_MASTER_KEY::"}, - {"name": "DS_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:DS_KEY::"}, - {"name": "ALI_API_KEYS", "valueFrom": "${STAGING_SECRETS_ARN}:ALI_API_KEYS::"}, - {"name": "ARK_API_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:ARK_API_KEY::"}, - {"name": "GPT_API_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:GPT_API_KEY::"}, - {"name": "MINERU_API_KEYS", "valueFrom": "${STAGING_SECRETS_ARN}:MINERU_API_KEYS::"}, - {"name": "ILOVEAPI_KEYS", "valueFrom": "${STAGING_SECRETS_ARN}:ILOVEAPI_KEYS::"}, - {"name": "LOGFIRE_TOKEN", "valueFrom": "${STAGING_SECRETS_ARN}:LOGFIRE_TOKEN::"}, - {"name": "QSTASH_TOKEN", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_TOKEN::"}, - {"name": "QSTASH_CURRENT_SIGNING_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_CURRENT_SIGNING_KEY::"}, - {"name": "QSTASH_NEXT_SIGNING_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_NEXT_SIGNING_KEY::"} + {"name": "DATABASE_URL", "valueFrom": "${SECRETS_ARN}:DATABASE_URL::"}, + {"name": "REDIS_HOST", "valueFrom": "${SECRETS_ARN}:REDIS_HOST::"}, + {"name": "REDIS_PORT", "valueFrom": "${SECRETS_ARN}:REDIS_PORT::"}, + {"name": "REDIS_PASSWORD", "valueFrom": "${SECRETS_ARN}:REDIS_PASSWORD::"}, + {"name": "CELERY_REDIS_URL", "valueFrom": "${SECRETS_ARN}:CELERY_REDIS_URL::"}, + {"name": "CELERY_REDIS_PASSWORD", "valueFrom": "${SECRETS_ARN}:CELERY_REDIS_PASSWORD::"}, + {"name": "SECRET_KEY", "valueFrom": "${SECRETS_ARN}:SECRET_KEY::"}, + {"name": "WEBHOOK_MASTER_KEY", "valueFrom": "${SECRETS_ARN}:WEBHOOK_MASTER_KEY::"}, + {"name": "DS_KEY", "valueFrom": "${SECRETS_ARN}:DS_KEY::"}, + {"name": "ALI_API_KEYS", "valueFrom": "${SECRETS_ARN}:ALI_API_KEYS::"}, + {"name": "ARK_API_KEY", "valueFrom": "${SECRETS_ARN}:ARK_API_KEY::"}, + {"name": "GPT_API_KEY", "valueFrom": "${SECRETS_ARN}:GPT_API_KEY::"}, + {"name": "MINERU_API_KEYS", "valueFrom": "${SECRETS_ARN}:MINERU_API_KEYS::"}, + {"name": "ILOVEAPI_KEYS", "valueFrom": "${SECRETS_ARN}:ILOVEAPI_KEYS::"}, + {"name": "LOGFIRE_TOKEN", "valueFrom": "${SECRETS_ARN}:LOGFIRE_TOKEN::"}, + {"name": "QSTASH_TOKEN", "valueFrom": "${SECRETS_ARN}:QSTASH_TOKEN::"}, + {"name": "QSTASH_CURRENT_SIGNING_KEY", "valueFrom": "${SECRETS_ARN}:QSTASH_CURRENT_SIGNING_KEY::"}, + {"name": "QSTASH_NEXT_SIGNING_KEY", "valueFrom": "${SECRETS_ARN}:QSTASH_NEXT_SIGNING_KEY::"} ], "healthCheck": { "command": ["CMD-SHELL", "python -c \"from shared.services.worker_health import assert_worker_healthy; assert_worker_healthy()\""], @@ -93,7 +93,7 @@ "logConfiguration": { "logDriver": "awslogs", "options": { - "awslogs-group": "/ecs/knowhere-worker-staging", + "awslogs-group": "/ecs/knowhere-worker-${DEPLOYMENT_ENVIRONMENT}", "awslogs-region": "us-east-1", "awslogs-stream-prefix": "worker" } @@ -102,7 +102,7 @@ ], "tags": [ {"key": "Project", "value": "knowhere"}, - {"key": "Environment", "value": "staging"}, + {"key": "Environment", "value": "${DEPLOYMENT_ENVIRONMENT}"}, {"key": "Service", "value": "worker"}, {"key": "ManagedBy", "value": "knowhere-api-infra"} ] diff --git a/deploy/ecs/test_render_task_definitions.py b/deploy/ecs/test_render_task_definitions.py index 72b13457..b9c1f569 100644 --- a/deploy/ecs/test_render_task_definitions.py +++ b/deploy/ecs/test_render_task_definitions.py @@ -18,7 +18,23 @@ "EXECUTION_ROLE_ARN": "execution-role", "API_TASK_ROLE_ARN": "api-role", "WORKER_TASK_ROLE_ARN": "worker-role", - "STAGING_SECRETS_ARN": "secrets-arn", + "SECRETS_ARN": "secrets-arn", + "DEPLOYMENT_ENVIRONMENT": "staging", + "RUNTIME_ENVIRONMENT": "staging", + "APP_ENV": "staging", + "DB_SSL_MODE": "require", + "API_DB_POOL_SIZE": "5", + "API_DB_MAX_OVERFLOW": "5", + "WORKER_DB_SYNC_POOL_SIZE": "2", + "WORKER_DB_SYNC_MAX_OVERFLOW": "2", + "S3_BUCKET_NAME": "knowhere-storage-staging", + "INTERNAL_DASHBOARD_ENDPOINT": "https://staging.knowhereto.ai", + "FRONTEND_URL": "https://staging.knowhereto.ai", + "API_WEBHOOK_ENDPOINT": "https://api-staging.knowhereto.ai/v1/internal/s3-events", + "SNS_TOPIC_ARN": "arn:aws:sns:us-east-1:107424103509:knowhere-staging-s3-events", + "QSTASH_CALLBACK_BASE_URL": "https://api-staging.knowhereto.ai/api/v1", + "WORKER_CPU": "2048", + "WORKER_MEMORY": "4096", } SHARED_STAGING_ENVIRONMENT: dict[str, str] = { @@ -95,15 +111,19 @@ def test_staging_templates_render_without_long_lived_s3_keys( ], ) def test_staging_templates_preserve_expected_staging_configuration( + tmp_path: Path, template_name: str, container_name: str, expected_environment: dict[str, str], ) -> None: """Fargate preserves the verified staging settings captured on 2026-08-13.""" - definition_path: Path = TEMPLATE_DIRECTORY / template_name - definition: dict[str, object] = json.loads( - definition_path.read_text(encoding="utf-8") + output_path: Path = tmp_path / template_name + render_template( + TEMPLATE_DIRECTORY / template_name, + output_path, + RENDER_VARIABLES, ) + definition: dict[str, object] = json.loads(output_path.read_text(encoding="utf-8")) container_definitions: list[dict[str, object]] = definition[ "containerDefinitions" ] @@ -119,12 +139,15 @@ def test_staging_templates_preserve_expected_staging_configuration( assert environment_values[name] == value -def test_staging_worker_preserves_evidence_selected_capacity() -> None: +def test_staging_worker_preserves_evidence_selected_capacity(tmp_path: Path) -> None: """Worker capacity matches the staging load evidence recorded in issue 22.""" - definition_path: Path = TEMPLATE_DIRECTORY / "task-definition-worker.staging.json" - definition: dict[str, object] = json.loads( - definition_path.read_text(encoding="utf-8") + output_path: Path = tmp_path / "task-definition-worker.staging.json" + render_template( + TEMPLATE_DIRECTORY / "task-definition-worker.staging.json", + output_path, + RENDER_VARIABLES, ) + definition: dict[str, object] = json.loads(output_path.read_text(encoding="utf-8")) # The measured production-envelope replay rejected 1-vCPU tasks and passed # on two fixed 2-vCPU tasks, so CD must not restore the rejected task size.