From b474ad38820166dc353ea4fb59be18fa9feeb07d Mon Sep 17 00:00:00 2001 From: Jonathan Alvarez Delgado Date: Thu, 5 Mar 2026 02:08:03 +0100 Subject: [PATCH 1/3] chore(ops): prepare ECS stage scale-up and resource lifecycle template. Validate deployment --- .gitignore | 6 +- infra/DEPLOYMENT_VALIDATION.md | 127 +++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 infra/DEPLOYMENT_VALIDATION.md diff --git a/.gitignore b/.gitignore index a4e63d224b26..277f4d202a8e 100644 --- a/.gitignore +++ b/.gitignore @@ -52,4 +52,8 @@ venv* infra/pulumi/pulumi-*.txt infra/pulumi/preview-*.txt infra/pulumi/analysis.md -infra/pulumi/infrastructure-inventory.md \ No newline at end of file +infra/pulumi/infrastructure-inventory.md + +# Local ops artefacts (runbooks, ad hoc templates, design docs) +infra/adhoc/ +infra/docs/ diff --git a/infra/DEPLOYMENT_VALIDATION.md b/infra/DEPLOYMENT_VALIDATION.md new file mode 100644 index 000000000000..f0cee69d5fc9 --- /dev/null +++ b/infra/DEPLOYMENT_VALIDATION.md @@ -0,0 +1,127 @@ +# ECS Fargate Stage Deployment Validation + +> **Date**: 2026-03-07 +> **Stack**: `thunderbird-addons/stage` +> **Region**: `us-west-2` + +--- + +## Infrastructure deployment + +``` +pulumi up --stack thunderbird/thunderbird-addons/stage + +Resources: + + 129 created + +- 2 replaced + 131 changes. 26 unchanged + +Duration: 5m 54s +Exit code: 0 +``` + +### Resource breakdown + +| Category | Resources | Notes | +|----------|-----------|-------| +| VPC + networking | VPC, 3 public subnets, 3 private subnets, NAT gateway, IGW, route tables, VPC peering | New VPC peered to existing default VPC | +| VPC endpoints | ECR (api + dkr), SSM, CloudWatch Logs, Secrets Manager, S3 gateway | Private connectivity for Fargate tasks | +| ECS clusters + services | web, worker, versioncheck | All at `desired_count: 0` | +| ALBs + target groups | web, versioncheck | Listeners on 80/443 | +| Security groups | ALB SGs, container SGs, VPC endpoint SG, Redis SG | SG-to-SG ingress (ALB -> container) | +| IAM | Execution roles, task roles, OIDC role, scoped policies | Least-privilege, secrets access scoped | +| ECR | Repository (imported from existing) | Tag mutability updated | +| ElastiCache | Redis replication group | Private subnets only | +| EventBridge | 16 scheduled tasks | All `DISABLED` by default | +| Autoscaling | 3 target-tracking policies | All `suspended`, `min_capacity: 0` | +| CloudWatch | Log groups, KMS keys | Per-cluster logging | + +### Post-deploy state verification + +``` +ECS Services: + web: desired=0 running=0 pending=0 status=ACTIVE + worker: desired=0 running=0 pending=0 status=ACTIVE + versioncheck: desired=0 running=0 pending=0 status=ACTIVE + +Autoscaling: + All 3 services: min=0, DynamicScalingIn=suspended, + DynamicScalingOut=suspended, ScheduledScaling=suspended + +EventBridge Schedules: + All 16: DISABLED +``` + +--- + +## Read-only MySQL user + +A dedicated read-only MySQL user was created for safe bootstrap validation: + +- **Scope**: `SELECT` on the application database only +- **Host restriction**: Connections accepted only from the ECS VPC CIDR +- **Secrets Manager**: Credentials stored as a separate secret (`_ro` suffix) +- **App integration**: `BOOTSTRAP_SAFE=true` environment variable selects the RO + credentials at startup; no code changes required + +--- + +## RO healthcheck (one-off ECS task) + +A one-off Fargate task was launched in the private subnets to validate end-to-end +connectivity from the new VPC to all shared backend services. + +**Task configuration**: +- Image: current `stage-latest` from ECR +- Settings: `BOOTSTRAP_SAFE=true` (RO database credentials) +- Network: private subnets, worker security group, no public IP + +**Results**: + +``` +====================================================================== +ATN Read-Only Health Check (ECS Deployment Validation) +====================================================================== + [OK] Django settings import + DJANGO_SETTINGS_MODULE=settings_local_stage + [OK] MySQL database (read-only ORM query) + Connected, 241480 addons (56ms) + [OK] Cache backend + Backend: django.core.cache.backends.memcached.MemcachedCache (0ms) + [OK] Celery broker (RabbitMQ) + Connected (19ms) + [OK] Elasticsearch / OpenSearch + Reachable, version: 5.6.17 (46ms) +---------------------------------------------------------------------- + Results: 5 passed, 0 failed +====================================================================== +``` + +All five checks passed, confirming: + +1. Django settings load correctly in the ECS environment +2. The RO MySQL user can query the application database from the new VPC +3. Memcached is reachable across the VPC peering connection +4. RabbitMQ (Celery broker) is reachable across the VPC peering connection +5. Elasticsearch 5.6 is reachable across the VPC peering connection + +--- + +## Safety layers active + +| Layer | Purpose | State | +|-------|---------|-------| +| `desired_count: 0` | No tasks run unless explicitly scaled | Active | +| Autoscaling suspended | Prevents automatic scale-out | Active | +| EventBridge `DISABLED` | No cron jobs fire | Active | +| `BOOTSTRAP_SAFE=true` | App uses RO database credentials | Active | + +--- + +## Next steps + +1. Scale `versioncheck` to 1 (read-heavy, safest service) +2. Scale `web` to 1 +3. Scale `worker` (coordinate with legacy EC2 worker shutdown) +4. Enable EventBridge schedules incrementally +5. Flip `BOOTSTRAP_SAFE` to `false` for RW operations (separate deliberate step) From 004027c5ee2e3786210b1159fb8ace8217e659af Mon Sep 17 00:00:00 2001 From: Jonathan Alvarez Delgado Date: Sat, 7 Mar 2026 19:49:47 +0100 Subject: [PATCH 2/3] fix(pulumi): enable external Fargate tasks IP, add NETAPP_STORAGE_ROOT, update docs --- infra/pulumi/README.md | 124 +++++++++++++++++---------------- infra/pulumi/__main__.py | 8 ++- infra/pulumi/config.stage.yaml | 6 ++ 3 files changed, 77 insertions(+), 61 deletions(-) diff --git a/infra/pulumi/README.md b/infra/pulumi/README.md index ccd0c801d522..8a47e0210052 100644 --- a/infra/pulumi/README.md +++ b/infra/pulumi/README.md @@ -1,6 +1,6 @@ # Thunderbird Add-ons Infra (Pulumi) -ECS Fargate infrastructure for addons-server +ECS Fargate infrastructure for addons-server (stage environment) ## Prerequisites @@ -16,34 +16,58 @@ source .venv/bin/activate pip install -r requirements.txt pulumi login # browser-based authn flow Pulumi Cloud -# Select the staging stack (name may vary depending on org setup) pulumi stack select thunderbird/thunderbird-addons/stage ``` -## Preview Changes +## Preview and Deploy ```bash -pulumi preview +# Preview (RO -- no AWS changes) +pulumi preview --diff + +# Deploy (RW -- creates/updates AWS resources) +pulumi up ``` +## Architecture + +| Component | Implementation | +|-----------|---------------| +| Web | Fargate service, ALB (HTTPS) | +| Worker | Fargate service (internal, no ALB) | +| Versioncheck | Fargate service, ALB (HTTPS) | +| Cron | 16 EventBridge-scheduled ECS tasks | +| Cache | ElastiCache Redis (private subnets) | +| Networking | New VPC peered to existing default VPC | + +## Safety Layers + +Services deploy cold by default. Each layer is independently verifiable + +| Layer | Config key | Default | +|-------|-----------|---------| +| Desired count | `desired_count` | `0` | +| Autoscaling | `suspend` | `true` | +| EventBridge schedules | `state` | `DISABLED` | +| DB credentials | `BOOTSTRAP_SAFE` env var | `true` (RO user) | + ## CI/CD -GitHub Actions workflow (`.github/workflows/build-and-push.yml`) handles image builds. +### Build and Push (`build-and-push.yml`) - **Pull requests**: Build validation only (no AWS auth) - **Push to stage**: Build + push to ECR via OIDC +- **Manual trigger**: `workflow_dispatch` (for re-builds without a code push) ### Enabling ECR Publishing -1. Ensure AWS OIDC provider exists for `token.actions.githubusercontent.com` -2. IAM role is created by Pulumi with trust policy scoped to `refs/heads/stage` +1. AWS OIDC provider for `token.actions.githubusercontent.com` (already exists) +2. IAM role created by Pulumi with trust policy scoped to `refs/heads/stage` 3. Set repository variable: `AWS_ROLE_ARN` (from Pulumi output `gha_ecr_publish_role_arn`) ## Scheduled Tasks -Scheduled tasks mirror the existing cron workload from the legacy environment and are executed as ECS tasks via EventBridge Scheduler. - -16 cron jobs run via EventBridge Scheduler: +16 cron jobs run via EventBridge Scheduler (all `DISABLED` by default): | Task | Schedule | Command | |------|----------|---------| @@ -66,7 +90,8 @@ Scheduled tasks mirror the existing cron workload from the legacy environment an ## Image Tagging -- `atn-stage-addons-server:stage-latest` - current stage build +- `stage-latest` -- current stage build +- `sha-{commit}` -- per-commit builds - ECR lifecycle: keep 50 tagged images, expire untagged after 7 days ## Secrets @@ -74,39 +99,28 @@ Scheduled tasks mirror the existing cron workload from the legacy environment an No secrets are stored in the repository. Application expects Secrets Manager paths under `atn/stage/*`: -- Database credentials +- Database credentials (RW and RO variants) - Django secret key -- External service API keys +- External service configuration See `settings_local_stage.py` for full mapping. ## Post-Deployment Verification -All commands are read-only - -### ECR Repository - -```bash -aws ecr describe-images \ - --repository-name atn-stage-addons-server \ - --region us-west-2 \ - --query 'imageDetails[*].[imageTags,imagePushedAt]' \ - --output table -``` +All commands below are read-only ### ECS Services ```bash -# List services -aws ecs list-services --cluster atn-stage-web-cluster --region us-west-2 -aws ecs list-services --cluster atn-stage-worker-cluster --region us-west-2 - -# Check service status -aws ecs describe-services \ - --cluster atn-stage-web-cluster \ - --services atn-stage-web \ - --region us-west-2 \ - --query 'services[*].[serviceName,runningCount,desiredCount,status]' +for svc in web worker versioncheck; do + echo "=== $svc ===" + aws ecs describe-services \ + --cluster "thunderbird-addons-stage-${svc}" \ + --services "thunderbird-addons-stage-${svc}" \ + --region us-west-2 \ + --query 'services[0].[desiredCount,runningCount,status]' \ + --output text +done ``` ### Scheduled Tasks @@ -119,38 +133,28 @@ aws scheduler list-schedules \ --output table ``` -### CloudWatch Logs - -```bash -# Recent web logs -aws logs tail /ecs/thunderbird-addons-stage-web --since 5m --region us-west-2 - -# Recent cron logs -aws logs tail /ecs/thunderbird-addons-stage-cron --since 5m --region us-west-2 -``` +### RO Healthcheck -### ALB Health Check +The `ro_healthcheck` management command validates connectivity to all backends +from within the ECS VPC. Run as a one-off Fargate task with `BOOTSTRAP_SAFE=true`. ```bash -# Get ALB DNS (after deployment) -pulumi stack output --json | jq -r '.web_alb_dns' - -# Test health endpoint -curl -I https://{alb-dns}/services/monitor +aws ecs run-task \ + --cluster thunderbird-addons-stage-worker \ + --task-definition thunderbird-addons-stage-ro-healthcheck \ + --launch-type FARGATE \ + --network-configuration "..." \ + --region us-west-2 ``` ## Resources Created -- New VPC with public/private subnets across 3 AZs (connectivity to existing RDS may require VPC peering - confirm with Andrei) +- VPC with public/private subnets across 3 AZs peered to existing default VPC - ECR repository with lifecycle policy -- ECS clusters (web, worker) -- Fargate services (web, worker, versioncheck) -- ElastiCache Redis cluster +- 3 ECS Fargate services (web, worker, versioncheck) with ALBs where applicable +- ElastiCache Redis replication group - 16 EventBridge scheduled tasks -- ALB with HTTPS listener -- IAM roles (task execution, task, scheduler, OIDC) -- CloudWatch log groups - -## Workflow - -All infrastructure changes are proposed via pull requests and reviewed before deployment. Direct `pulumi up` execution is restricted to approved paths. +- IAM roles (task execution, task, scheduler, OIDC for CI) +- CloudWatch log groups with KMS encryption +- VPC endpoints (ECR, SSM, Logs, Secrets Manager, S3) +- Application autoscaling targets (suspended by default) diff --git a/infra/pulumi/__main__.py b/infra/pulumi/__main__.py index a8d89c3dd5ec..599545a9bc64 100755 --- a/infra/pulumi/__main__.py +++ b/infra/pulumi/__main__.py @@ -528,9 +528,15 @@ def main(): fargate_services = {} for service_name, service_config in fargate_configs.items(): - # Inject subnet IDs based on whether service is internal or external is_internal = service_config.get("internal", True) + # Internet-facing ALBs require public subnets and tb_pulumi uses a single + # subnet list for both ALB and tasks, so external services must land in + # public subnets. To compensate this we'd force assign_public_ip=True so + # tasks can reach ECR/internet via IGW (private subnet tasks use NAT) + # TODO: consider tb_pulumi proposal to support separate ALB/task subnets subnets = private_subnets if is_internal else public_subnets + if not is_internal: + service_config["assign_public_ip"] = True if subnets: # Get security groups for this service diff --git a/infra/pulumi/config.stage.yaml b/infra/pulumi/config.stage.yaml index 74e1b2ac8f47..c85f6db80530 100644 --- a/infra/pulumi/config.stage.yaml +++ b/infra/pulumi/config.stage.yaml @@ -195,6 +195,8 @@ resources: value: settings_local_stage - name: BOOTSTRAP_SAFE value: 'true' + - name: NETAPP_STORAGE_ROOT + value: /tmp/storage - name: UWSGI_PROCESSES value: '4' - name: UWSGI_THREADS @@ -235,6 +237,8 @@ resources: value: settings_local_stage - name: BOOTSTRAP_SAFE value: 'true' + - name: NETAPP_STORAGE_ROOT + value: /tmp/storage - name: CELERY_CONCURRENCY value: '4' - name: CELERY_QUEUES @@ -295,6 +299,8 @@ resources: value: settings_local_stage - name: BOOTSTRAP_SAFE value: 'true' + - name: NETAPP_STORAGE_ROOT + value: /tmp/storage - name: UWSGI_PROCESSES value: '4' - name: UWSGI_THREADS From b9379725bb790be7896361878c26ae259c816de3 Mon Sep 17 00:00:00 2001 From: Jonathan Alvarez Delgado Date: Sat, 7 Mar 2026 21:17:17 +0100 Subject: [PATCH 3/3] fix(stage): allow ALB health checks without weakening ALLOWED_HOSTS --- settings_local_stage.py | 4 ++++ src/olympia/amo/middleware_healthcheck.py | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 src/olympia/amo/middleware_healthcheck.py diff --git a/settings_local_stage.py b/settings_local_stage.py index 9c7e6952ce63..a68ec2b6fa48 100644 --- a/settings_local_stage.py +++ b/settings_local_stage.py @@ -320,6 +320,10 @@ def get_secret(secret_name, region_name="us-west-2"): '.mozaws.net', ] +MIDDLEWARE = ( + 'olympia.amo.middleware_healthcheck.ALBHealthCheckMiddleware', +) + MIDDLEWARE + FLIGTAR = 'addons+fligtar-rip@thunderbird.net' THEMES_EMAIL = 'addons+theme-reviews@thunderbird.net' ABUSE_EMAIL = 'addons+abuse@thunderbird.net' diff --git a/src/olympia/amo/middleware_healthcheck.py b/src/olympia/amo/middleware_healthcheck.py new file mode 100644 index 000000000000..a4c99d2864a7 --- /dev/null +++ b/src/olympia/amo/middleware_healthcheck.py @@ -0,0 +1,23 @@ +from django.http import JsonResponse + + +class ALBHealthCheckMiddleware: + """Respond to ALB health checks before Django host validation runs + + AWS ALB would send health checks with the LB node IP as the Host header + which Django rejects via ALLOWED_HOSTS. This middleware intercepts the + health check path and returns 200 before CommonMiddleware validates + the Host header + + Must be placed at the top of MIDDLEWARE to run before host validation + """ + + HEALTH_CHECK_PATH = "/services/monitor.json" + + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + if request.path == self.HEALTH_CHECK_PATH and request.method == "GET": + return JsonResponse({"status": "ok"}) + return self.get_response(request)