diff --git a/.github/workflows/manage-staging.yml b/.github/workflows/manage-staging.yml index 2614c077..31432856 100644 --- a/.github/workflows/manage-staging.yml +++ b/.github/workflows/manage-staging.yml @@ -153,6 +153,27 @@ jobs: return 1 } + wait_for_public_api_health() { + local api_health_deadline="$((SECONDS + 300))" + + # services-stable can return as soon as the API task is RUNNING, + # before the load balancer has registered a healthy target. Treat + # transient 502/503 responses as cold-start progress, but keep a + # hard timeout so a broken target never reports a successful start. + while (( SECONDS < api_health_deadline )); do + if curl --fail --silent --show-error --max-time 10 \ + https://api-staging.knowhereto.ai/health >/dev/null; then + return 0 + fi + + echo "Waiting for the public staging API health endpoint" + sleep 10 + done + + echo "Timed out waiting for the public staging API health endpoint" >&2 + return 1 + } + read_services() { aws --profile knowhere ecs describe-services \ --cluster knowhere-fargate \ @@ -174,8 +195,7 @@ jobs: update_service knowhere-api-staging 1 wait_for_service knowhere-api-staging - curl --fail --silent --show-error --max-time 30 \ - https://api-staging.knowhereto.ai/health >/dev/null + wait_for_public_api_health startup_seconds="$(($(date +%s) - start_started_epoch))" ;; stop) diff --git a/deploy/ecs/test_manage_staging_workflow.py b/deploy/ecs/test_manage_staging_workflow.py index 9e9db55a..c50f0a7a 100644 --- a/deploy/ecs/test_manage_staging_workflow.py +++ b/deploy/ecs/test_manage_staging_workflow.py @@ -60,7 +60,10 @@ def test_start_restores_healthy_workers_before_api() -> None: start_block.index("update_service knowhere-api-staging 1") ) assert start_block.index("update_service knowhere-api-staging 1") < ( - start_block.index("https://api-staging.knowhereto.ai/health") + start_block.index("wait_for_service knowhere-api-staging") + ) + assert start_block.index("wait_for_service knowhere-api-staging") < ( + start_block.index("wait_for_public_api_health") ) @@ -79,6 +82,23 @@ def test_worker_health_gate_polls_through_container_start_period() -> None: assert 'echo "Timed out waiting for two healthy worker tasks"' in health_gate +def test_public_api_health_gate_retries_transient_alb_errors() -> None: + """A newly running API task may not yet have a healthy public ALB route.""" + workflow: str = _read_workflow() + health_gate: str = workflow.split( + " wait_for_public_api_health() {", maxsplit=1 + )[1].split("\n }", maxsplit=1)[0] + + assert 'api_health_deadline="$((SECONDS + 300))"' in health_gate + assert "while (( SECONDS < api_health_deadline )); do" in health_gate + assert "https://api-staging.knowhereto.ai/health" in health_gate + assert "return 0" in health_gate + assert "sleep 10" in health_gate + assert 'echo "Timed out waiting for the public staging API health endpoint"' in ( + health_gate + ) + + def test_stop_closes_api_then_drains_before_workers() -> None: """The stop operation preserves the accepted API-first 30-minute drain.""" workflow: str = _read_workflow() @@ -110,7 +130,13 @@ def test_status_uses_the_jobs_ledger_for_backlog() -> None: def test_start_reports_readiness_time() -> None: """Operators can compare cold-start readiness with the schedule lead time.""" workflow: str = _read_workflow() + start_block: str = workflow.split(" start)", maxsplit=1)[1].split( + " ;;", maxsplit=1 + )[0] assert 'start_started_epoch="$(date +%s)"' in workflow + assert start_block.index("wait_for_public_api_health") < start_block.index( + 'startup_seconds="$(($(date +%s) - start_started_epoch))"' + ) assert 'startup_seconds="$(($(date +%s) - start_started_epoch))"' in workflow assert "startupSeconds" in workflow