From c71627d13612f1067fe42d2ee62e7101d9af9bab Mon Sep 17 00:00:00 2001 From: Tom Arne Date: Fri, 18 Sep 2026 12:00:47 +0200 Subject: [PATCH 1/6] Wait for ECS Express deployment rollout before reporting success --- .../deploy-ecs-express-service/action.yml | 64 ++++++++++++++++--- 1 file changed, 55 insertions(+), 9 deletions(-) diff --git a/.github/actions/deployment/preview/deploy-ecs-express-service/action.yml b/.github/actions/deployment/preview/deploy-ecs-express-service/action.yml index fdc0e1b8..ece8a26e 100644 --- a/.github/actions/deployment/preview/deploy-ecs-express-service/action.yml +++ b/.github/actions/deployment/preview/deploy-ecs-express-service/action.yml @@ -86,14 +86,30 @@ runs: echo "Service $SERVICE_NAME already exists with ARN: $SERVICE_ARN" echo "service_exists=true" >> $GITHUB_OUTPUT echo "service_arn=$SERVICE_ARN" >> $GITHUB_OUTPUT + HAS_VPC=$(echo "$DESCRIBE_OUTPUT" | jq -r 'if (.service.networkConfiguration // null) != null then "true" else "false" end' 2>/dev/null || echo "false") + echo "has_vpc_config=$HAS_VPC" >> $GITHUB_OUTPUT elif echo "$DESCRIBE_OUTPUT" | grep -q -i "ServiceNotFoundException\|not found"; then echo "Service $SERVICE_NAME does not exist. Will create a new one." echo "service_exists=false" >> $GITHUB_OUTPUT + echo "has_vpc_config=false" >> $GITHUB_OUTPUT else echo "Failed to describe service: $DESCRIBE_OUTPUT" exit 1 fi + - name: Warn if VPC config needs to change + id: delete-for-vpc-change + if: | + steps.check-service.outputs.service_exists == 'true' && ( + (inputs.use-shared-vpc == 'true' && steps.check-service.outputs.has_vpc_config == 'false') || + (inputs.use-shared-vpc != 'true' && steps.check-service.outputs.has_vpc_config == 'true') + ) + shell: bash + run: | + echo "::warning::The preview service exists with a different VPC configuration than requested." + echo "::warning::AWS does not support changing the VPC of an existing ECS Express Gateway service." + echo "::warning::The service will be updated on its current network. After the next preview cleanup deletes it, it will be recreated with the correct VPC config on the next deployment." + - name: Prepare Environment Variables Configuration id: prepare-env-vars shell: bash @@ -236,11 +252,6 @@ runs: SERVICE_ARN="${{ steps.check-service.outputs.service_arn }}" echo "Updating Preview: ${{ steps.set-service-variables.outputs.service_name }} ($SERVICE_ARN)..." - VPC_ARGS=() - if [[ "${{ inputs.use-shared-vpc }}" == "true" ]]; then - VPC_ARGS+=(--network-configuration "securityGroups=[${{ steps.vpc-config.outputs.security_group_id }}],subnets=[${{ steps.vpc-config.outputs.subnet_ids }}]") - fi - aws ecs update-express-gateway-service \ --service-arn "$SERVICE_ARN" \ --execution-role-arn "$executionRoleArn" \ @@ -248,8 +259,7 @@ runs: --primary-container "$primary_container" \ --health-check-path "$HEALTH_CHECK_ENDPOINT" \ --scaling-target '{"minTaskCount": 1, "maxTaskCount": 1}' \ - --region "${{ inputs.aws-region }}" \ - "${VPC_ARGS[@]}" + --region "${{ inputs.aws-region }}" echo "Update started! $SERVICE_ARN" echo "service_arn=$SERVICE_ARN" >> $GITHUB_OUTPUT @@ -260,15 +270,21 @@ runs: run: | SERVICE_ARN="${{ steps.create-service.outputs.service_arn || steps.update-service.outputs.service_arn }}" + # The top-level service status only reflects whether the gateway/load balancer + # infrastructure is provisioned, not whether a task is running and healthy behind + # it. Wait for that infra first, then wait for the actual deployment rollout + # (which does track task + target-group health) to finish, so we don't report + # success while the service is still returning 502s. for i in {1..60}; do SERVICE_JSON=$(aws ecs describe-express-gateway-service --service-arn "$SERVICE_ARN" --region ${{ inputs.aws-region }}) STATUS=$(echo "$SERVICE_JSON" | jq -r '.service.status.statusCode') STATUS_REASON=$(echo "$SERVICE_JSON" | jq -r '.service.status.statusReason // ""') + DEPLOYMENT_ARN=$(echo "$SERVICE_JSON" | jq -r '.service.currentDeployment // ""') SERVICE_URL=$(echo "$SERVICE_JSON" | jq -r \ '([.service.activeConfigurations[0].ingressPaths[]? | select(.accessType=="PUBLIC") | .endpoint][0] // "") | sub("^https?://"; "") | sub("/$"; "")') echo "Current status: $STATUS ($STATUS_REASON), URL: ${SERVICE_URL:-}" - if [[ "$STATUS" == "ACTIVE" ]] && [[ -n "$SERVICE_URL" ]]; then - echo "Service is active." + if [[ "$STATUS" == "ACTIVE" ]] && [[ -n "$SERVICE_URL" ]] && [[ -n "$DEPLOYMENT_ARN" ]]; then + echo "Service infrastructure is active." break fi sleep 10 @@ -284,4 +300,34 @@ runs: exit 1 fi + if [[ -z "$DEPLOYMENT_ARN" ]]; then + echo "Service is ACTIVE but no current deployment ARN is available." + exit 1 + fi + + echo "Waiting for deployment $DEPLOYMENT_ARN to finish rolling out (task started and passing health checks)..." + for i in {1..60}; do + DEPLOYMENT_JSON=$(aws ecs describe-service-deployments --service-deployment-arns "$DEPLOYMENT_ARN" --region ${{ inputs.aws-region }}) + DEPLOYMENT_STATUS=$(echo "$DEPLOYMENT_JSON" | jq -r '.serviceDeployments[0].status // ""') + DEPLOYMENT_STATUS_REASON=$(echo "$DEPLOYMENT_JSON" | jq -r '.serviceDeployments[0].statusReason // ""') + echo "Current deployment status: $DEPLOYMENT_STATUS ($DEPLOYMENT_STATUS_REASON)" + + if [[ "$DEPLOYMENT_STATUS" == "SUCCESSFUL" ]]; then + echo "Deployment is healthy and serving traffic." + break + fi + + if [[ "$DEPLOYMENT_STATUS" =~ ^(STOPPED|STOP_REQUESTED|ROLLBACK_SUCCESSFUL|ROLLBACK_FAILED)$ ]]; then + echo "Deployment failed: $DEPLOYMENT_STATUS ($DEPLOYMENT_STATUS_REASON)" + exit 1 + fi + + sleep 10 + done + + if [[ "$DEPLOYMENT_STATUS" != "SUCCESSFUL" ]]; then + echo "Deployment did not reach SUCCESSFUL status in time." + exit 1 + fi + echo "service_url=${SERVICE_URL}" >> $GITHUB_OUTPUT \ No newline at end of file From d213d8a6df714fc0547f4ceda61eb4231ea52b08 Mon Sep 17 00:00:00 2001 From: Tom Arne Date: Fri, 18 Sep 2026 12:09:21 +0200 Subject: [PATCH 2/6] Pin ECS Express deployment actions to current branch for health-wait fix testing --- .github/workflows/deployment.preview.on-comment.yml | 3 ++- .github/workflows/deployment.preview.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deployment.preview.on-comment.yml b/.github/workflows/deployment.preview.on-comment.yml index 2d9d9ffe..97349b33 100644 --- a/.github/workflows/deployment.preview.on-comment.yml +++ b/.github/workflows/deployment.preview.on-comment.yml @@ -71,7 +71,8 @@ jobs: preview: needs: trigger - uses: nsbno/platform-actions/.github/workflows/deployment.preview.yml@v2 + # TEMP: pinned to this branch to test the ECS Express health-wait fix; revert to @v2 before merging + uses: nsbno/platform-actions/.github/workflows/deployment.preview.yml@fix-preview-url-comment-before-ready with: health-endpoint: ${{ inputs.health-endpoint }} use-ecs-express-mode: ${{ inputs.use-ecs-express-mode }} diff --git a/.github/workflows/deployment.preview.yml b/.github/workflows/deployment.preview.yml index 16794c75..a4ace6e8 100644 --- a/.github/workflows/deployment.preview.yml +++ b/.github/workflows/deployment.preview.yml @@ -170,7 +170,8 @@ jobs: - name: Deploy Preview (ECS) if: ${{ inputs.use-ecs-express-mode == true && (inputs.only-on-preview-comment == false || steps.check-comments.outputs.continue == 'true') }} id: deploy-ecs - uses: nsbno/platform-actions/.github/actions/deployment/preview/deploy-ecs-express-service@v2 + # TEMP: pinned to this branch to test the ECS Express health-wait fix; revert to @v2 before merging + uses: nsbno/platform-actions/.github/actions/deployment/preview/deploy-ecs-express-service@fix-preview-url-comment-before-ready with: health-endpoint: ${{ inputs.health-endpoint }} service-name: ${{ steps.deployment-info.outputs.ecs-service-name }} From c3d6dcf29dd8f0d7c249f113fd0e5d49718459f0 Mon Sep 17 00:00:00 2001 From: Tom Arne Date: Tue, 22 Sep 2026 16:22:29 +0200 Subject: [PATCH 3/6] Enhance deployment readiness checks for ECS Express by including lifecycle stage in status evaluation --- .../deploy-ecs-express-service/action.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/actions/deployment/preview/deploy-ecs-express-service/action.yml b/.github/actions/deployment/preview/deploy-ecs-express-service/action.yml index ece8a26e..2830e103 100644 --- a/.github/actions/deployment/preview/deploy-ecs-express-service/action.yml +++ b/.github/actions/deployment/preview/deploy-ecs-express-service/action.yml @@ -306,18 +306,29 @@ runs: fi echo "Waiting for deployment $DEPLOYMENT_ARN to finish rolling out (task started and passing health checks)..." + # Blue/green-style strategies (LINEAR, CANARY, BLUE_GREEN) only reach status + # SUCCESSFUL after the full post-shift bake time and old-revision cleanup, but + # production traffic is already 100% on the new revision once the lifecycle + # stage passes PRODUCTION_TRAFFIC_SHIFT. Treat that as ready too, so preview + # deploys aren't stuck waiting out a multi-minute bake window for no reason. for i in {1..60}; do DEPLOYMENT_JSON=$(aws ecs describe-service-deployments --service-deployment-arns "$DEPLOYMENT_ARN" --region ${{ inputs.aws-region }}) DEPLOYMENT_STATUS=$(echo "$DEPLOYMENT_JSON" | jq -r '.serviceDeployments[0].status // ""') DEPLOYMENT_STATUS_REASON=$(echo "$DEPLOYMENT_JSON" | jq -r '.serviceDeployments[0].statusReason // ""') - echo "Current deployment status: $DEPLOYMENT_STATUS ($DEPLOYMENT_STATUS_REASON)" + LIFECYCLE_STAGE=$(echo "$DEPLOYMENT_JSON" | jq -r '.serviceDeployments[0].lifecycleStage // ""') + echo "Current deployment status: $DEPLOYMENT_STATUS ($DEPLOYMENT_STATUS_REASON), lifecycle stage: ${LIFECYCLE_STAGE:-}" if [[ "$DEPLOYMENT_STATUS" == "SUCCESSFUL" ]]; then echo "Deployment is healthy and serving traffic." break fi - if [[ "$DEPLOYMENT_STATUS" =~ ^(STOPPED|STOP_REQUESTED|ROLLBACK_SUCCESSFUL|ROLLBACK_FAILED)$ ]]; then + if [[ "$LIFECYCLE_STAGE" =~ ^(POST_PRODUCTION_TRAFFIC_SHIFT|BAKE_TIME|CLEAN_UP)$ ]]; then + echo "Production traffic has fully shifted to the new revision." + break + fi + + if [[ "$DEPLOYMENT_STATUS" =~ ^(STOPPED|STOP_REQUESTED|ROLLBACK_REQUESTED|ROLLBACK_IN_PROGRESS|ROLLBACK_SUCCESSFUL|ROLLBACK_FAILED)$ ]]; then echo "Deployment failed: $DEPLOYMENT_STATUS ($DEPLOYMENT_STATUS_REASON)" exit 1 fi @@ -325,8 +336,8 @@ runs: sleep 10 done - if [[ "$DEPLOYMENT_STATUS" != "SUCCESSFUL" ]]; then - echo "Deployment did not reach SUCCESSFUL status in time." + if [[ "$DEPLOYMENT_STATUS" != "SUCCESSFUL" ]] && [[ ! "$LIFECYCLE_STAGE" =~ ^(POST_PRODUCTION_TRAFFIC_SHIFT|BAKE_TIME|CLEAN_UP)$ ]]; then + echo "Deployment did not become ready in time." exit 1 fi From e9c64097dea51fda6e4692db23dae0e37771577d Mon Sep 17 00:00:00 2001 From: Tom Arne Date: Tue, 22 Sep 2026 17:18:07 +0200 Subject: [PATCH 4/6] Remove comments --- .../preview/deploy-ecs-express-service/action.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/actions/deployment/preview/deploy-ecs-express-service/action.yml b/.github/actions/deployment/preview/deploy-ecs-express-service/action.yml index 2830e103..5f0107f6 100644 --- a/.github/actions/deployment/preview/deploy-ecs-express-service/action.yml +++ b/.github/actions/deployment/preview/deploy-ecs-express-service/action.yml @@ -270,11 +270,6 @@ runs: run: | SERVICE_ARN="${{ steps.create-service.outputs.service_arn || steps.update-service.outputs.service_arn }}" - # The top-level service status only reflects whether the gateway/load balancer - # infrastructure is provisioned, not whether a task is running and healthy behind - # it. Wait for that infra first, then wait for the actual deployment rollout - # (which does track task + target-group health) to finish, so we don't report - # success while the service is still returning 502s. for i in {1..60}; do SERVICE_JSON=$(aws ecs describe-express-gateway-service --service-arn "$SERVICE_ARN" --region ${{ inputs.aws-region }}) STATUS=$(echo "$SERVICE_JSON" | jq -r '.service.status.statusCode') @@ -306,11 +301,6 @@ runs: fi echo "Waiting for deployment $DEPLOYMENT_ARN to finish rolling out (task started and passing health checks)..." - # Blue/green-style strategies (LINEAR, CANARY, BLUE_GREEN) only reach status - # SUCCESSFUL after the full post-shift bake time and old-revision cleanup, but - # production traffic is already 100% on the new revision once the lifecycle - # stage passes PRODUCTION_TRAFFIC_SHIFT. Treat that as ready too, so preview - # deploys aren't stuck waiting out a multi-minute bake window for no reason. for i in {1..60}; do DEPLOYMENT_JSON=$(aws ecs describe-service-deployments --service-deployment-arns "$DEPLOYMENT_ARN" --region ${{ inputs.aws-region }}) DEPLOYMENT_STATUS=$(echo "$DEPLOYMENT_JSON" | jq -r '.serviceDeployments[0].status // ""') From abb6a30af8cb999ea2b9d4cb681168ad5e0a347b Mon Sep 17 00:00:00 2001 From: Tom Arne Date: Tue, 22 Sep 2026 17:20:36 +0200 Subject: [PATCH 5/6] Remove temporary branch pinning for ECS Express deployment actions --- .github/workflows/deployment.preview.on-comment.yml | 3 +-- .github/workflows/deployment.preview.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deployment.preview.on-comment.yml b/.github/workflows/deployment.preview.on-comment.yml index 97349b33..2d9d9ffe 100644 --- a/.github/workflows/deployment.preview.on-comment.yml +++ b/.github/workflows/deployment.preview.on-comment.yml @@ -71,8 +71,7 @@ jobs: preview: needs: trigger - # TEMP: pinned to this branch to test the ECS Express health-wait fix; revert to @v2 before merging - uses: nsbno/platform-actions/.github/workflows/deployment.preview.yml@fix-preview-url-comment-before-ready + uses: nsbno/platform-actions/.github/workflows/deployment.preview.yml@v2 with: health-endpoint: ${{ inputs.health-endpoint }} use-ecs-express-mode: ${{ inputs.use-ecs-express-mode }} diff --git a/.github/workflows/deployment.preview.yml b/.github/workflows/deployment.preview.yml index a4ace6e8..16794c75 100644 --- a/.github/workflows/deployment.preview.yml +++ b/.github/workflows/deployment.preview.yml @@ -170,8 +170,7 @@ jobs: - name: Deploy Preview (ECS) if: ${{ inputs.use-ecs-express-mode == true && (inputs.only-on-preview-comment == false || steps.check-comments.outputs.continue == 'true') }} id: deploy-ecs - # TEMP: pinned to this branch to test the ECS Express health-wait fix; revert to @v2 before merging - uses: nsbno/platform-actions/.github/actions/deployment/preview/deploy-ecs-express-service@fix-preview-url-comment-before-ready + uses: nsbno/platform-actions/.github/actions/deployment/preview/deploy-ecs-express-service@v2 with: health-endpoint: ${{ inputs.health-endpoint }} service-name: ${{ steps.deployment-info.outputs.ecs-service-name }} From 720fa8252745acc1eeb23cfeaefb29d7ca162be7 Mon Sep 17 00:00:00 2001 From: Tom Arne Date: Tue, 22 Sep 2026 17:27:15 +0200 Subject: [PATCH 6/6] Revert "Remove temporary branch pinning for ECS Express deployment actions" This reverts commit abb6a30af8cb999ea2b9d4cb681168ad5e0a347b. --- .github/workflows/deployment.preview.on-comment.yml | 3 ++- .github/workflows/deployment.preview.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deployment.preview.on-comment.yml b/.github/workflows/deployment.preview.on-comment.yml index 2d9d9ffe..97349b33 100644 --- a/.github/workflows/deployment.preview.on-comment.yml +++ b/.github/workflows/deployment.preview.on-comment.yml @@ -71,7 +71,8 @@ jobs: preview: needs: trigger - uses: nsbno/platform-actions/.github/workflows/deployment.preview.yml@v2 + # TEMP: pinned to this branch to test the ECS Express health-wait fix; revert to @v2 before merging + uses: nsbno/platform-actions/.github/workflows/deployment.preview.yml@fix-preview-url-comment-before-ready with: health-endpoint: ${{ inputs.health-endpoint }} use-ecs-express-mode: ${{ inputs.use-ecs-express-mode }} diff --git a/.github/workflows/deployment.preview.yml b/.github/workflows/deployment.preview.yml index 16794c75..a4ace6e8 100644 --- a/.github/workflows/deployment.preview.yml +++ b/.github/workflows/deployment.preview.yml @@ -170,7 +170,8 @@ jobs: - name: Deploy Preview (ECS) if: ${{ inputs.use-ecs-express-mode == true && (inputs.only-on-preview-comment == false || steps.check-comments.outputs.continue == 'true') }} id: deploy-ecs - uses: nsbno/platform-actions/.github/actions/deployment/preview/deploy-ecs-express-service@v2 + # TEMP: pinned to this branch to test the ECS Express health-wait fix; revert to @v2 before merging + uses: nsbno/platform-actions/.github/actions/deployment/preview/deploy-ecs-express-service@fix-preview-url-comment-before-ready with: health-endpoint: ${{ inputs.health-endpoint }} service-name: ${{ steps.deployment-info.outputs.ecs-service-name }}