From e82dcafa7d5910dc73ca47b4cd31d18b33e745de Mon Sep 17 00:00:00 2001 From: Jonathan Alvarez Delgado Date: Fri, 27 Feb 2026 23:28:00 +0100 Subject: [PATCH 1/2] safety(pulumi): deploy cold (desired_count 0), suspend autoscaling, disable schedules --- .gitignore | 2 +- infra/pulumi/__main__.py | 6 ++++-- infra/pulumi/config.stage.yaml | 32 +++++++++++++++++++------------- settings_local_stage.py | 14 ++++++++++++-- 4 files changed, 36 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index f69500f2fdf0..a4e63d224b26 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,6 @@ venv* # Pulumi local artefacts (outputs, notes, analysis) infra/pulumi/pulumi-*.txt -infra/pulumi/preview-output-*.txt +infra/pulumi/preview-*.txt infra/pulumi/analysis.md infra/pulumi/infrastructure-inventory.md \ No newline at end of file diff --git a/infra/pulumi/__main__.py b/infra/pulumi/__main__.py index 6cbf83b6a06b..de739f6c8c05 100755 --- a/infra/pulumi/__main__.py +++ b/infra/pulumi/__main__.py @@ -780,7 +780,9 @@ def main(): "help", ], # Default; again overridden per schedule "environment": [ - {"name": "DJANGO_SETTINGS_MODULE", "value": "settings"} + {"name": "DJANGO_SETTINGS_MODULE", "value": "settings_local_stage"}, + {"name": "BOOTSTRAP_SAFE", "value": "true"}, + {"name": "NETAPP_STORAGE_ROOT", "value": "/tmp/storage"}, ], "logConfiguration": { "logDriver": "awslogs", @@ -936,7 +938,7 @@ def main(): {"containerOverrides": [{"name": "cron", "command": command}]} ), ), - state="ENABLED", + state=task_config.get("state", "DISABLED"), opts=pulumi.ResourceOptions( parent=schedule_group, depends_on=[cron_task_definition, scheduler_role], diff --git a/infra/pulumi/config.stage.yaml b/infra/pulumi/config.stage.yaml index 8a04967f89aa..74e1b2ac8f47 100644 --- a/infra/pulumi/config.stage.yaml +++ b/infra/pulumi/config.stage.yaml @@ -20,17 +20,17 @@ resources: addons-server: name: atn-stage-addons-server image_tag_mutability: MUTABLE - force_delete: true # Stage only: allows pulumi destroy with images present + force_delete: false # Repo now holds CI-built images; protect from accidental destroy scan_on_push: true encryption_type: AES256 - # Lifecycle policy keep last 50 tagged images (here any tag), expire untagged after 7 days - # This catches SHA tags, stage-latest, and any future tag patterns + # Lifecycle policy keep last 50 tagged images (stage-/sha- prefixes), expire untagged after 7 days + # Matches tags prefixed with stage- (e.g., stage-latest) and sha- (commit SHAs) lifecycle_policy: | { "rules": [ { "rulePriority": 1, - "description": "Keep last 50 tagged images (any tag)", + "description": "Keep last 50 tagged images (stage-/sha- prefixes)", "selection": { "tagStatus": "tagged", "tagPrefixList": ["stage-", "sha-"], @@ -150,10 +150,7 @@ resources: # Image: 768512802988.dkr.ecr.us-west-2.amazonaws.com/atn-stage-addons-server:stage-latest tb:fargate:FargateClusterWithLogging: web: - # desired_count intentionally omitted: autoscaling owns the count. - # tb_pulumi sets ignore_changes on desired_count when not specified, - # preventing Pulumi from fighting the autoscaler. min_capacity in - # the autoscaling config acts as the effective baseline + desired_count: 0 # Start cold; scale up manually after validation assign_public_ip: false internal: false # Public-facing ALB enable_container_insights: true @@ -196,6 +193,8 @@ resources: environment: - name: DJANGO_SETTINGS_MODULE value: settings_local_stage + - name: BOOTSTRAP_SAFE + value: 'true' - name: UWSGI_PROCESSES value: '4' - name: UWSGI_THREADS @@ -211,7 +210,7 @@ resources: # t3a.large has 8GB RAM - needed for addons-linter memory requirements # Multiple queue groups for different workloads worker: - # desired_count omitted: autoscaling owns the count (see web comment above) + desired_count: 0 # Start cold; scale up manually after validation assign_public_ip: false internal: true build_load_balancer: false # Workers don't need ALB @@ -234,6 +233,8 @@ resources: environment: - name: DJANGO_SETTINGS_MODULE value: settings_local_stage + - name: BOOTSTRAP_SAFE + value: 'true' - name: CELERY_CONCURRENCY value: '4' - name: CELERY_QUEUES @@ -249,7 +250,7 @@ resources: # Separate ALB endpoint for version checking API (versioncheck.addons.thunderbird.net) # Lightweight service - c7a.medium equivalent versioncheck: - # desired_count omitted: autoscaling owns the count (see web comment) + desired_count: 0 # Start cold; scale up manually after validation assign_public_ip: false internal: false enable_container_insights: true @@ -292,6 +293,8 @@ resources: environment: - name: DJANGO_SETTINGS_MODULE value: settings_local_stage + - name: BOOTSTRAP_SAFE + value: 'true' - name: UWSGI_PROCESSES value: '4' - name: UWSGI_THREADS @@ -326,23 +329,26 @@ resources: web: cpu_threshold: 70 ram_threshold: 70 - min_capacity: 2 + min_capacity: 0 # Start at 0; scale up manually after validation max_capacity: 8 cooldown: 300 + suspend: true # Suspended until services are validated worker: cpu_threshold: 70 ram_threshold: 80 # Higher: addons-linter memory spikes are normal - min_capacity: 2 + min_capacity: 0 max_capacity: 6 cooldown: 300 + suspend: true versioncheck: cpu_threshold: 70 ram_threshold: 70 - min_capacity: 1 + min_capacity: 0 max_capacity: 4 cooldown: 300 + suspend: true # ============================================================================= # ElastiCache - Memcached (intended to replace current Memcached setup) diff --git a/settings_local_stage.py b/settings_local_stage.py index 80ea678a7bad..9c7e6952ce63 100644 --- a/settings_local_stage.py +++ b/settings_local_stage.py @@ -34,9 +34,19 @@ def get_secret(secret_name, region_name="us-west-2"): raise Exception(f"Failed to retrieve secret {secret_name}: {e}") +# ----------------------------------------------------------------------------- +# Bootstrap safety toggle +# ----------------------------------------------------------------------------- +# When BOOTSTRAP_SAFE is true we deliberately use RO database credentials +# (if present) so that even if something accidentally starts, MySQL itself +# enforces read-only access +BOOTSTRAP_SAFE = env.bool("BOOTSTRAP_SAFE", default=False) +MYSQL_SECRET_NAME = "atn/stage/mysql_ro" if BOOTSTRAP_SAFE else "atn/stage/mysql" + + # Retrieve secrets from AWS Secrets Manager _email_url_secret = get_secret('atn/stage/email_url') -_mysql_secret = get_secret('atn/stage/mysql') +_mysql_secret = get_secret(MYSQL_SECRET_NAME) _inbound_email_secret = get_secret('atn/stage/inbound_email') _django_secret = get_secret('atn/stage/django_secret_key') _celery_broker_secret = get_secret('atn/stage/celery_broker') @@ -288,7 +298,7 @@ def get_secret(secret_name, region_name="us-west-2"): ES_DEFAULT_NUM_SHARDS = 10 -READ_ONLY = env.bool('READ_ONLY', default=False) +READ_ONLY = env.bool("READ_ONLY", default=BOOTSTRAP_SAFE) # TODO: Github user ? GITHUB_API_USER = '' From 5a596e043590d7d7066279653b8e2a7e7e24262c Mon Sep 17 00:00:00 2001 From: Jonathan Alvarez Delgado Date: Sat, 28 Feb 2026 22:52:51 +0100 Subject: [PATCH 2/2] style: ruff format __main__.py --- infra/pulumi/__main__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/infra/pulumi/__main__.py b/infra/pulumi/__main__.py index de739f6c8c05..a8d89c3dd5ec 100755 --- a/infra/pulumi/__main__.py +++ b/infra/pulumi/__main__.py @@ -780,7 +780,10 @@ def main(): "help", ], # Default; again overridden per schedule "environment": [ - {"name": "DJANGO_SETTINGS_MODULE", "value": "settings_local_stage"}, + { + "name": "DJANGO_SETTINGS_MODULE", + "value": "settings_local_stage", + }, {"name": "BOOTSTRAP_SAFE", "value": "true"}, {"name": "NETAPP_STORAGE_ROOT", "value": "/tmp/storage"}, ],