Wait for public staging API readiness

.github/workflows/manage-staging.yml

@@ -153,6 +153,27 @@ jobs:
             return 1
           }
 
+          wait_for_public_api_health() {
+            local api_health_deadline="$((SECONDS + 300))"
+
+            # services-stable can return as soon as the API task is RUNNING,
+            # before the load balancer has registered a healthy target. Treat
+            # transient 502/503 responses as cold-start progress, but keep a
+            # hard timeout so a broken target never reports a successful start.
+            while (( SECONDS < api_health_deadline )); do
+              if curl --fail --silent --show-error --max-time 10 \
+                https://api-staging.knowhereto.ai/health >/dev/null; then
+                return 0
+              fi
+
+              echo "Waiting for the public staging API health endpoint"
+              sleep 10
+            done
+
+            echo "Timed out waiting for the public staging API health endpoint" >&2
+            return 1
+          }
+
           read_services() {
             aws --profile knowhere ecs describe-services \
               --cluster knowhere-fargate \
@@ -174,8 +195,7 @@ jobs:
 
               update_service knowhere-api-staging 1
               wait_for_service knowhere-api-staging
-              curl --fail --silent --show-error --max-time 30 \
-                https://api-staging.knowhereto.ai/health >/dev/null
+              wait_for_public_api_health
               startup_seconds="$(($(date +%s) - start_started_epoch))"
               ;;
             stop)

deploy/ecs/test_manage_staging_workflow.py

@@ -60,7 +60,10 @@ def test_start_restores_healthy_workers_before_api() -> None:
         start_block.index("update_service knowhere-api-staging 1")
     )
     assert start_block.index("update_service knowhere-api-staging 1") < (
-        start_block.index("https://api-staging.knowhereto.ai/health")
+        start_block.index("wait_for_service knowhere-api-staging")
+    )
+    assert start_block.index("wait_for_service knowhere-api-staging") < (
+        start_block.index("wait_for_public_api_health")
     )
 
 
@@ -79,6 +82,23 @@ def test_worker_health_gate_polls_through_container_start_period() -> None:
     assert 'echo "Timed out waiting for two healthy worker tasks"' in health_gate
 
 
+def test_public_api_health_gate_retries_transient_alb_errors() -> None:
+    """A newly running API task may not yet have a healthy public ALB route."""
+    workflow: str = _read_workflow()
+    health_gate: str = workflow.split(
+        "          wait_for_public_api_health() {", maxsplit=1
+    )[1].split("\n          }", maxsplit=1)[0]
+
+    assert 'api_health_deadline="$((SECONDS + 300))"' in health_gate
+    assert "while (( SECONDS < api_health_deadline )); do" in health_gate
+    assert "https://api-staging.knowhereto.ai/health" in health_gate
+    assert "return 0" in health_gate
+    assert "sleep 10" in health_gate
+    assert 'echo "Timed out waiting for the public staging API health endpoint"' in (
+        health_gate
+    )
+
+
 def test_stop_closes_api_then_drains_before_workers() -> None:
     """The stop operation preserves the accepted API-first 30-minute drain."""
     workflow: str = _read_workflow()
@@ -110,7 +130,13 @@ def test_status_uses_the_jobs_ledger_for_backlog() -> None:
 def test_start_reports_readiness_time() -> None:
     """Operators can compare cold-start readiness with the schedule lead time."""
     workflow: str = _read_workflow()
+    start_block: str = workflow.split("            start)", maxsplit=1)[1].split(
+        "              ;;", maxsplit=1
+    )[0]
 
     assert 'start_started_epoch="$(date +%s)"' in workflow
+    assert start_block.index("wait_for_public_api_health") < start_block.index(
+        'startup_seconds="$(($(date +%s) - start_started_epoch))"'
+    )
     assert 'startup_seconds="$(($(date +%s) - start_started_epoch))"' in workflow
     assert "startupSeconds" in workflow