From 69c8c9ecd36cb864fd5116d13296c1b4327b7647 Mon Sep 17 00:00:00 2001
From: hellices <hellices@naver.com>
Date: Sat, 15 Aug 2026 14:22:30 +0900
Subject: [PATCH 1/2] fix(sre-lab): harden guided scenario operations

Close iterative review findings across alert polling, recovery state, Storage RBAC propagation, capture retries, cleanup evidence, and operator guidance. Add regression coverage for transient Azure failures and incomplete manual setup.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 monitor/sre-agent-event-lab/README.md         |   2 +-
 .../guides/01-agent-setup.md                  |  72 +++-
 .../guides/02-scenario-s1.md                  |   2 +-
 .../guides/04-scenario-s3.md                  |  21 +-
 .../sre-agent-event-lab/guides/05-results.md  |   2 +
 .../scripts/capture-scenario.sh               |  19 +-
 .../scripts/cleanup-external.sh               |  29 +-
 monitor/sre-agent-event-lab/scripts/common.sh |   6 +-
 monitor/sre-agent-event-lab/scripts/doctor.sh |  15 +-
 .../sre-agent-event-lab/scripts/lab_state.py  |  18 +
 .../scripts/query-evidence.sh                 |  14 +-
 .../scripts/run-scenario.sh                   | 123 ++++++-
 .../scripts/tests/lab_script_harness.py       | 105 +++++-
 .../scripts/tests/test_cleanup_external.py    |  20 ++
 .../scripts/tests/test_common.py              |   7 +
 .../scripts/tests/test_doctor.py              |  48 +++
 .../scripts/tests/test_lab_guides.py          |  36 ++
 .../scripts/tests/test_lab_scripts.py         | 319 ++++++++++++++++++
 .../scripts/tests/test_lab_state.py           |  11 +
 19 files changed, 825 insertions(+), 44 deletions(-)

diff --git a/monitor/sre-agent-event-lab/README.md b/monitor/sre-agent-event-lab/README.md
index ca35e257..bb0266ec 100644
--- a/monitor/sre-agent-event-lab/README.md
+++ b/monitor/sre-agent-event-lab/README.md
@@ -149,7 +149,7 @@ azd down --purge
 - predown hook `scripts/cleanup-external.sh --yes`: `evidence/agent-setup.json`에 기록된 구독 범위 Monitoring Contributor 할당만 제거합니다. 기록된 principal·역할·범위가 실제 할당과 모두 일치할 때만 삭제하고, 하나라도 어긋나면 아무것도 지우지 않습니다.
 - postdown hook `scripts/cleanup-external.sh --reset-image-env --yes`: 기록된 `SRE_CONTAINER_IMAGE`와 `SRE_IMAGE_TAG`를 비웁니다. 삭제가 실제로 성공한 뒤에만 실행되어야 하므로 predown이 아니라 postdown입니다.
 
-중요: predown hook은 `azd down`이 삭제 **확인** 프롬프트를 띄우기 **전에** 실행됩니다. 그 프롬프트에서 **취소**해도 이미 제거된 Monitoring Contributor 할당은 돌아오지 않습니다. 리소스 그룹은 남지만 Agent의 구독 범위 권한은 사라진 상태이므로, 계속 쓰려면 역할 할당을 다시 만들고 `./scripts/lab.sh acknowledge agent-setup`으로 `evidence/agent-setup.json`을 새 할당 ID로 갱신해야 합니다.
+중요: predown hook은 `azd down`이 삭제 **확인** 프롬프트를 띄우기 **전에** 실행됩니다. 그 프롬프트에서 **취소**해도 이미 제거된 Monitoring Contributor 할당은 돌아오지 않습니다. 리소스 그룹은 남지만 Agent의 구독 범위 권한은 사라진 상태이므로, 계속 쓰려면 역할 할당을 다시 만들고 `evidence/agent-setup.json`을 새 할당 ID로 직접 갱신한 뒤 `./scripts/lab.sh acknowledge agent-setup`을 실행해야 합니다.
 
 hook이 실패해 손으로 다시 실행할 때는 아래를 직접 호출합니다. `--yes` 없이는 계획만 출력합니다.
 
diff --git a/monitor/sre-agent-event-lab/guides/01-agent-setup.md b/monitor/sre-agent-event-lab/guides/01-agent-setup.md
index bbef6fca..66670718 100644
--- a/monitor/sre-agent-event-lab/guides/01-agent-setup.md
+++ b/monitor/sre-agent-event-lab/guides/01-agent-setup.md
@@ -51,6 +51,52 @@ Agent가 실습 리소스를 읽고 구독의 경고를 훑을 수 있어야 합
 
 Monitoring Contributor는 [Azure Monitor 스캐너](https://learn.microsoft.com/azure/sre-agent/azure-monitor-alerts)가 요구하는 유일한 구독 범위 권한입니다. 할당 ID는 아래에서 기록해 두고 정리 단계에서 자동으로 제거합니다.
 
+포털에서 확인한 두 관리 ID의 object ID를 넣고 역할을 만듭니다. Reader는 azd가 삭제할 리소스 그룹 범위이고, Monitoring Contributor 두 건만 구독 범위이므로 ID를 변수에 보관합니다.
+
+```bash
+SUBSCRIPTION_ID="$(azd env get-value AZURE_SUBSCRIPTION_ID)"
+RESOURCE_GROUP="$(azd env get-value AZURE_RESOURCE_GROUP)"
+SUBSCRIPTION_SCOPE="/subscriptions/${SUBSCRIPTION_ID}"
+RESOURCE_GROUP_SCOPE="${SUBSCRIPTION_SCOPE}/resourceGroups/${RESOURCE_GROUP}"
+AGENT_PRINCIPAL_ID="<Agent 시스템 할당 ID의 objectId>"
+AGENT_UAMI_PRINCIPAL_ID="<Agent 사용자 할당 ID의 objectId>"
+SRE_AGENT_ENDPOINT="https://<agent>.<region>.azuresre.ai"
+
+az role assignment create \
+  --assignee-object-id "${AGENT_PRINCIPAL_ID}" \
+  --assignee-principal-type ServicePrincipal \
+  --role "Reader" \
+  --scope "${RESOURCE_GROUP_SCOPE}" \
+  --output none
+az role assignment create \
+  --assignee-object-id "${AGENT_UAMI_PRINCIPAL_ID}" \
+  --assignee-principal-type ServicePrincipal \
+  --role "Reader" \
+  --scope "${RESOURCE_GROUP_SCOPE}" \
+  --output none
+
+MONITORING_CONTRIBUTOR_ASSIGNMENT_ID="$(az role assignment create \
+  --assignee-object-id "${AGENT_PRINCIPAL_ID}" \
+  --assignee-principal-type ServicePrincipal \
+  --role "Monitoring Contributor" \
+  --scope "${SUBSCRIPTION_SCOPE}" \
+  --query id -o tsv 2>/dev/null || az role assignment list \
+    --assignee-object-id "${AGENT_PRINCIPAL_ID}" \
+    --role "Monitoring Contributor" \
+    --scope "${SUBSCRIPTION_SCOPE}" \
+    --query "[0].id" -o tsv)"
+UAMI_MONITORING_CONTRIBUTOR_ASSIGNMENT_ID="$(az role assignment create \
+  --assignee-object-id "${AGENT_UAMI_PRINCIPAL_ID}" \
+  --assignee-principal-type ServicePrincipal \
+  --role "Monitoring Contributor" \
+  --scope "${SUBSCRIPTION_SCOPE}" \
+  --query id -o tsv 2>/dev/null || az role assignment list \
+    --assignee-object-id "${AGENT_UAMI_PRINCIPAL_ID}" \
+    --role "Monitoring Contributor" \
+    --scope "${SUBSCRIPTION_SCOPE}" \
+    --query "[0].id" -o tsv)"
+```
+
 ## Builder > Incident platform 연결
 
 1. 왼쪽에서 **Builder > Incident platform**을 엽니다.
@@ -98,17 +144,31 @@ azd env set SRE_KNOWLEDGE_PATH "runbooks/incident-response.md"
 ## 근거 파일 만들기
 
 `evidence/agent-setup.json`은 Git에서 제외되며, 정리 단계가 되돌릴 대상을 이 파일에서만 읽습니다. 값은 모두 식별자와 엔드포인트 URL입니다.
+아래 블록은 역할을 만든 위 블록과 같은 셸에서 실행해야 앞에서 설정한 변수와 할당 ID를 그대로 사용합니다.
 
 ```bash
-cat > evidence/agent-setup.json <<'JSON'
+mkdir -p evidence
+cat > evidence/agent-setup.json <<JSON
 {
-  "agent_principal_id": "<Agent 시스템 할당 ID의 objectId>",
-  "agent_user_assigned_principal_id": "<Agent 사용자 할당 ID의 objectId>",
-  "agent_endpoint": "https://<agent>.<region>.azuresre.ai",
-  "monitoring_contributor_assignment_id": "/subscriptions/<id>/providers/Microsoft.Authorization/roleAssignments/<guid>",
-  "uami_monitoring_contributor_assignment_id": "/subscriptions/<id>/providers/Microsoft.Authorization/roleAssignments/<guid>"
+  "agent_principal_id": "${AGENT_PRINCIPAL_ID}",
+  "agent_user_assigned_principal_id": "${AGENT_UAMI_PRINCIPAL_ID}",
+  "agent_endpoint": "${SRE_AGENT_ENDPOINT}",
+  "monitoring_contributor_assignment_id": "${MONITORING_CONTRIBUTOR_ASSIGNMENT_ID}",
+  "uami_monitoring_contributor_assignment_id": "${UAMI_MONITORING_CONTRIBUTOR_ASSIGNMENT_ID}"
 }
 JSON
+
+jq -e \
+  --arg scope "${SUBSCRIPTION_SCOPE}/providers/Microsoft.Authorization/roleAssignments/" \
+  '(.monitoring_contributor_assignment_id | startswith($scope))
+   and (.uami_monitoring_contributor_assignment_id | startswith($scope))
+   and (.agent_principal_id | length > 0)
+   and (.agent_user_assigned_principal_id | length > 0)
+   and (.agent_endpoint | test("^https://[^<>]+$"))' \
+   evidence/agent-setup.json >/dev/null || {
+     echo "agent-setup.json is incomplete; re-check the principals, endpoint, and assignment IDs above." >&2
+     false
+   }
 ```
 
 ## 점검과 승인
diff --git a/monitor/sre-agent-event-lab/guides/02-scenario-s1.md b/monitor/sre-agent-event-lab/guides/02-scenario-s1.md
index 86ca55ff..8ccd1e2a 100644
--- a/monitor/sre-agent-event-lab/guides/02-scenario-s1.md
+++ b/monitor/sre-agent-event-lab/guides/02-scenario-s1.md
@@ -78,7 +78,7 @@ cd monitor/sre-agent-event-lab
 1. Container App의 활성 revision이 다시 정상입니다.
 2. 이 실행이 발생시킨 경고가 Azure Monitor에서 `Resolved`가 됩니다.
 
-경고 해제는 최대 15분, 워크로드 정상화는 최대 10분까지 기다립니다. 둘 중 하나라도 시간 안에 확인되지 않으면 실행은 실패로 기록되고 S2는 계속 막힙니다. 실패한 실행은 원인을 고친 뒤 `./scripts/lab.sh run s1`을 다시 실행하면 새 시도로 이어집니다. 다시 실행하는 순간 이전 시도의 `s1_recovered`와 `s1_captured` 기록은 장애를 주입하기 전에 지워지므로, 새 시도가 복구되고 `capture`까지 끝날 때까지 S2는 다시 막힙니다. 이미 성공한 시나리오를 한 번 더 돌릴 때도 같습니다.
+경고 해제는 최대 25분, 워크로드 정상화는 최대 10분까지 기다립니다. 1분 주기의 stateful log alert는 실패 요청이 5분 조회 창에서 빠진 뒤에도 조건이 10분간 불충족이어야 `Resolved`가 되므로 여유 시간을 포함합니다. 둘 중 하나라도 시간 안에 확인되지 않으면 실행은 실패로 기록되고 S2는 계속 막힙니다. 실패한 실행은 원인을 고친 뒤 `./scripts/lab.sh run s1`을 다시 실행하면 새 시도로 이어집니다. 다시 실행하는 순간 이전 시도의 `s1_recovered`와 `s1_captured` 기록은 장애를 주입하기 전에 지워지므로, 새 시도가 복구되고 `capture`까지 끝날 때까지 S2는 다시 막힙니다. 이미 성공한 시나리오를 한 번 더 돌릴 때도 같습니다.
 
 되돌리기 자체가 실패하면(예: `az containerapp update` 거부, 새 revision이 준비되지 않음) 스크립트는 `CRITICAL:` 두 줄을 출력하고 0이 아닌 코드로 끝냅니다. 주입한 장애가 그대로 남아 있다는 뜻이므로, 다음 시나리오를 실행하기 전에 `FAILURE_MODE=none`을 수동으로 되돌리고 revision이 정상인지 확인하세요.
 
diff --git a/monitor/sre-agent-event-lab/guides/04-scenario-s3.md b/monitor/sre-agent-event-lab/guides/04-scenario-s3.md
index 851e1f54..95cbed3e 100644
--- a/monitor/sre-agent-event-lab/guides/04-scenario-s3.md
+++ b/monitor/sre-agent-event-lab/guides/04-scenario-s3.md
@@ -22,13 +22,16 @@ cd monitor/sre-agent-event-lab
 | 순서 | 변화 |
 |---|---|
 | 1 | 워크로드 관리 ID의 `Storage Blob Data Reader` 할당이 Blob 컨테이너 범위에서 삭제됩니다 |
-| 2 | `/api/documents`에 요청 60건(동시 4)이 들어가고 모두 HTTP 503을 받습니다 |
-| 3 | Application Insights workspace 테이블 `AppDependencies`에 Storage 대상 `ResultCode == "403"`이 쌓입니다 |
-| 4 | 5분 창의 403 의존성 실패가 5건을 넘으면 `alert-sre-lab-s3-storage-rbac`(Sev2)이 발생합니다 |
-| 5 | 복구로 같은 이름·같은 범위의 역할 할당이 다시 만들어집니다 |
+| 2 | 데이터 평면에 권한 삭제가 전파되어 단건 probe가 HTTP 503을 받을 때까지 최대 5분 기다립니다 |
+| 3 | `/api/documents`에 요청 60건(동시 4)이 들어가고 모두 HTTP 503을 받습니다 |
+| 4 | Application Insights workspace 테이블 `AppDependencies`에 Storage 대상 `ResultCode == "403"`이 쌓입니다 |
+| 5 | 5분 창의 403 의존성 실패가 5건을 넘으면 `alert-sre-lab-s3-storage-rbac`(Sev2)이 발생합니다 |
+| 6 | 복구로 같은 이름·같은 범위의 역할 할당이 다시 만들어집니다 |
 
 삭제와 복구는 출력에 기록된 Blob 컨테이너 범위의 단일 역할에만 적용됩니다. 구독이나 리소스 그룹 범위의 다른 할당은 건드리지 않습니다.
 
+단건 probe가 제한 시간 안에 503을 확인하지 못하면 `Storage RBAC deletion did not produce HTTP 503 within 300s`를 출력하고 본 부하를 시작하지 않은 채 역할을 복구합니다.
+
 ## SRE Agent에서 확인할 항목
 
 - 앱이 돌려준 503과 실제 원인인 Storage 403을 구분하는지
@@ -58,6 +61,16 @@ cd monitor/sre-agent-event-lab
 
 역할 복구 자체가 실패하면 스크립트는 `CRITICAL:` 두 줄을 출력하고 0이 아닌 코드로 끝냅니다. 워크로드에 Blob 권한이 없는 상태가 그대로 남으므로, 같은 이름·같은 범위의 `Storage Blob Data Reader` 할당을 수동으로 다시 만든 뒤 다음 단계로 넘어가세요.
 
+```bash
+az role assignment create \
+  --name "$(azd env get-value AZURE_BLOB_ROLE_ASSIGNMENT_NAME)" \
+  --assignee-object-id "$(azd env get-value AZURE_CONTAINER_APP_PRINCIPAL_ID)" \
+  --assignee-principal-type ServicePrincipal \
+  --role "Storage Blob Data Reader" \
+  --scope "$(azd env get-value AZURE_STORAGE_CONTAINER_SCOPE)" \
+  --output none
+```
+
 ## 다음 단계
 
 수집한 근거를 채점합니다: [05-results.md](05-results.md)
diff --git a/monitor/sre-agent-event-lab/guides/05-results.md b/monitor/sre-agent-event-lab/guides/05-results.md
index b5aa7eda..743dae51 100644
--- a/monitor/sre-agent-event-lab/guides/05-results.md
+++ b/monitor/sre-agent-event-lab/guides/05-results.md
@@ -35,6 +35,8 @@ cd monitor/sre-agent-event-lab
 
 시나리오의 `run_status`가 `recovered`가 아니면 채점기는 capture 내용과 관계없이 모든 항목을 `FAIL`·0점으로 기록합니다.
 
+한 번 `conclusion`으로 끝난 capture는 같은 실행에서 다시 수집할 수 없습니다. 두 번째 결과가 필요하면 해당 시나리오를 다시 실행해 새 attempt와 evidence 디렉터리를 만든 뒤 capture합니다.
+
 `impact_scope`는 alert 규칙 자체의 scope를 그대로 옮겨 적는 것으로는 채워지지
 않습니다. 이 랩의 모든 alert는 Log Analytics workspace scope입니다
 (`infra/alerts.bicep`의 `scopes`/`targetResourceTypes:
diff --git a/monitor/sre-agent-event-lab/scripts/capture-scenario.sh b/monitor/sre-agent-event-lab/scripts/capture-scenario.sh
index 3a9cb3ca..29c6faad 100755
--- a/monitor/sre-agent-event-lab/scripts/capture-scenario.sh
+++ b/monitor/sre-agent-event-lab/scripts/capture-scenario.sh
@@ -34,8 +34,17 @@ readonly EVIDENCE_DIR
 readonly TIMELINE_FILE="${EVIDENCE_DIR}/timeline.json"
 readonly NORMALIZED_FILE="${EVIDENCE_DIR}/normalized-timeline.json"
 
+CURRENT_CAPTURE_STATE="$(lab_state capture-status "${SCENARIO}")"
+if [[ "${CURRENT_CAPTURE_STATE}" == "conclusion" ]]; then
+  echo "${SCENARIO} already has a conclusion for this run." >&2
+  echo "To collect another capture, start a new attempt first: ./scripts/lab.sh run ${SCENARIO}" >&2
+  echo "To regenerate only the rendered assets, run: ${PYTHON} ${SCRIPT_DIR}/render_capture.py ${NORMALIZED_FILE} ${ASSET_DIR} --scenario ${SCENARIO}" >&2
+  exit 1
+fi
+
 if [[ ! -f "${AGENT_SETUP_FILE}" ]]; then
   echo "Missing Agent setup evidence: ${AGENT_SETUP_FILE}" >&2
+  echo "Create evidence/agent-setup.json as shown in guides/01-agent-setup.md." >&2
   exit 1
 fi
 if [[ ! -f "${TIMELINE_FILE}" ]]; then
@@ -55,8 +64,14 @@ fi
 
 AGENT_ENDPOINT="$(jq -r '.agent_endpoint // empty' "${AGENT_SETUP_FILE}")"
 ALERT_ID="$(jq -r '.alert_id // empty' "${TIMELINE_FILE}")"
-if [[ -z "${AGENT_ENDPOINT}" || -z "${ALERT_ID}" ]]; then
-  echo "Agent endpoint or alert ID is missing from evidence." >&2
+if [[ -z "${AGENT_ENDPOINT}" || "${AGENT_ENDPOINT}" != https://* || "${AGENT_ENDPOINT}" == *"<"* || "${AGENT_ENDPOINT}" == *">"* ]]; then
+  echo "agent_endpoint is missing or is not a valid HTTPS endpoint in ${AGENT_SETUP_FILE}." >&2
+  echo "Update evidence/agent-setup.json as shown in guides/01-agent-setup.md." >&2
+  exit 1
+fi
+if [[ -z "${ALERT_ID}" ]]; then
+  echo "alert_id is missing from ${TIMELINE_FILE}." >&2
+  echo "Create a new scenario timeline by running: ./scripts/lab.sh run ${SCENARIO}" >&2
   exit 1
 fi
 readonly AGENT_ENDPOINT ALERT_ID
diff --git a/monitor/sre-agent-event-lab/scripts/cleanup-external.sh b/monitor/sre-agent-event-lab/scripts/cleanup-external.sh
index 23b80cb1..8e5ff645 100755
--- a/monitor/sre-agent-event-lab/scripts/cleanup-external.sh
+++ b/monitor/sre-agent-event-lab/scripts/cleanup-external.sh
@@ -92,10 +92,6 @@ require_command() {
   }
 }
 
-lowercase() {
-  printf '%s' "$1" | tr '[:upper:]' '[:lower:]'
-}
-
 if [[ "${MODE}" == "image-env" ]]; then
   require_command azd
   if [[ "${CONFIRMED}" -ne 1 ]]; then
@@ -147,20 +143,24 @@ fi
 if ! RECORDED_ASSIGNMENTS="$(jq -r '
   [
     {
+      assignment_key: "monitoring_contributor_assignment_id",
       assignment: (.monitoring_contributor_assignment_id // ""),
+      principal_key: "agent_principal_id",
       principal: (.agent_principal_id // "")
     },
     {
+      assignment_key: "uami_monitoring_contributor_assignment_id",
       assignment: (.uami_monitoring_contributor_assignment_id // ""),
+      principal_key: "agent_user_assigned_principal_id",
       principal: (.agent_user_assigned_principal_id // "")
     }
   ]
   | .[]
-  | [.assignment, .principal]
-  | @tsv
+  | [.assignment_key, .assignment, .principal_key, .principal]
+  | join("|")
 ' "${CLEANUP_SETUP_FILE}" 2>/dev/null)"; then
   echo "Agent setup evidence is not valid JSON: ${CLEANUP_SETUP_FILE}" >&2
-  echo "Recreate it: lab.sh acknowledge agent-setup" >&2
+  echo "Rewrite evidence/agent-setup.json as shown in guides/01-agent-setup.md, then run: lab.sh acknowledge agent-setup" >&2
   exit 1
 fi
 readonly RECORDED_ASSIGNMENTS
@@ -252,13 +252,20 @@ verify_recorded_assignment() {
 # a duplicate. Bash 3.2 (macOS) aborts under `set -u` when an empty array
 # is expanded, which is why this stays a string instead of an array.
 VERIFIED_ASSIGNMENT_IDS=$'\n'
-while IFS=$'\t' read -r assignment_id expected_principal_id; do
-  if [[ -z "${assignment_id}" ]]; then
+while IFS='|' read -r assignment_key assignment_id principal_key expected_principal_id; do
+  if [[ -z "${assignment_id}" && -z "${expected_principal_id}" ]]; then
     continue
   fi
+  if [[ -z "${assignment_id}" ]]; then
+    echo "${assignment_key} is empty for principal ${expected_principal_id:-unknown}." >&2
+    echo "Find the live ID: az role assignment list --assignee-object-id ${expected_principal_id:-<principalId>} --role \"Monitoring Contributor\" --scope ${SUBSCRIPTION_SCOPE} --query \"[0].id\" -o tsv" >&2
+    echo "If no assignment is returned and teardown should continue, clear both ${assignment_key} and ${principal_key} in the evidence file." >&2
+    echo "Update evidence/agent-setup.json as shown in guides/01-agent-setup.md, then run: lab.sh acknowledge agent-setup" >&2
+    exit 1
+  fi
   if [[ -z "${expected_principal_id}" ]]; then
-    echo "Incomplete Agent setup evidence: ${assignment_id} was recorded without its Agent principal ID." >&2
-    echo "Recreate it: lab.sh acknowledge agent-setup" >&2
+    echo "Incomplete Agent setup evidence: ${assignment_id} was recorded without ${principal_key}." >&2
+    echo "Update evidence/agent-setup.json as shown in guides/01-agent-setup.md, then run: lab.sh acknowledge agent-setup" >&2
     exit 1
   fi
   case "${VERIFIED_ASSIGNMENT_IDS}" in
diff --git a/monitor/sre-agent-event-lab/scripts/common.sh b/monitor/sre-agent-event-lab/scripts/common.sh
index e5d8b618..4f68bb22 100755
--- a/monitor/sre-agent-event-lab/scripts/common.sh
+++ b/monitor/sre-agent-event-lab/scripts/common.sh
@@ -18,6 +18,10 @@ require_commands() {
   done
 }
 
+lowercase() {
+  printf '%s' "$1" | tr '[:upper:]' '[:lower:]'
+}
+
 # azd_value NAME -- the current azd environment's value for NAME, or empty
 # when azd has no such value (e.g. before the environment was provisioned).
 #
@@ -410,7 +414,7 @@ alert_monitor_condition() {
 # an open incident, and both incidents' evidence would be unreadable.
 wait_for_alert_resolved() {
   local alert_id="$1"
-  local timeout_seconds="${2:-900}"
+  local timeout_seconds="${2:-1500}"
   local interval_seconds="${3:-20}"
   local deadline=$(( SECONDS + timeout_seconds ))
   local condition=""
diff --git a/monitor/sre-agent-event-lab/scripts/doctor.sh b/monitor/sre-agent-event-lab/scripts/doctor.sh
index ea52a46d..c6fd482a 100755
--- a/monitor/sre-agent-event-lab/scripts/doctor.sh
+++ b/monitor/sre-agent-event-lab/scripts/doctor.sh
@@ -275,16 +275,27 @@ fi
 if [[ "${AZURE_SAFE}" -eq 1 ]]; then
   RESOURCE_GROUP_SCOPE="/subscriptions/${SUBSCRIPTION_ID}/resourceGroups/${RESOURCE_GROUP}"
   if [[ ! -f "${AGENT_SETUP_FILE}" ]]; then
-    report "Reader role assignment" FAIL "Agent setup evidence missing: ${AGENT_SETUP_FILE}. Run: lab.sh acknowledge agent-setup after recording the Agent identities."
+    report "Reader role assignment" FAIL "Agent setup evidence missing: ${AGENT_SETUP_FILE}. Create evidence/agent-setup.json as shown in guides/01-agent-setup.md, then run: lab.sh acknowledge agent-setup."
   elif ! AGENT_SETUP_JSON="$(jq '.' "${AGENT_SETUP_FILE}" 2>/dev/null)"; then
     # A hand-edited or truncated evidence file is one FAIL row, not a raw
     # `jq` abort: under `set -e` an unguarded parse would kill the run and
     # swallow every remaining check, including the MANUAL rows an operator
     # still needs.
-    report "Reader role assignment" FAIL "Agent setup evidence is not valid JSON: ${AGENT_SETUP_FILE}. Recreate it: lab.sh acknowledge agent-setup"
+    report "Reader role assignment" FAIL "Agent setup evidence is not valid JSON: ${AGENT_SETUP_FILE}. Rewrite evidence/agent-setup.json as shown in guides/01-agent-setup.md, then run: lab.sh acknowledge agent-setup."
   else
     agent_principal_id="$(jq -r '.agent_principal_id // empty' <<<"${AGENT_SETUP_JSON}" 2>/dev/null || true)"
     agent_uami_principal_id="$(jq -r '.agent_user_assigned_principal_id // empty' <<<"${AGENT_SETUP_JSON}" 2>/dev/null || true)"
+    monitoring_assignment_id="$(jq -r '.monitoring_contributor_assignment_id // empty' <<<"${AGENT_SETUP_JSON}" 2>/dev/null || true)"
+    uami_monitoring_assignment_id="$(jq -r '.uami_monitoring_contributor_assignment_id // empty' <<<"${AGENT_SETUP_JSON}" 2>/dev/null || true)"
+    EXPECTED_ASSIGNMENT_PREFIX="/subscriptions/${SUBSCRIPTION_ID}/providers/Microsoft.Authorization/roleAssignments/"
+    MISSING_CLEANUP_KEYS=()
+    [[ "$(lowercase "${monitoring_assignment_id}")" == "$(lowercase "${EXPECTED_ASSIGNMENT_PREFIX}")"* ]] || MISSING_CLEANUP_KEYS+=("monitoring_contributor_assignment_id")
+    [[ "$(lowercase "${uami_monitoring_assignment_id}")" == "$(lowercase "${EXPECTED_ASSIGNMENT_PREFIX}")"* ]] || MISSING_CLEANUP_KEYS+=("uami_monitoring_contributor_assignment_id")
+    if [[ "${#MISSING_CLEANUP_KEYS[@]}" -gt 0 ]]; then
+      report "Monitoring Contributor cleanup evidence" FAIL "Missing or invalid: ${MISSING_CLEANUP_KEYS[*]}. Find each live ID with: az role assignment list --assignee-object-id <principalId> --role \"Monitoring Contributor\" --scope /subscriptions/${SUBSCRIPTION_ID} --query \"[0].id\" -o tsv; then update ${AGENT_SETUP_FILE}."
+    else
+      report "Monitoring Contributor cleanup evidence" PASS "Both subscription-scoped role assignment IDs are recorded."
+    fi
     if [[ -z "${agent_principal_id}" || -z "${agent_uami_principal_id}" ]]; then
       report "Reader role assignment" FAIL "Agent setup evidence is missing agent_principal_id/agent_user_assigned_principal_id: ${AGENT_SETUP_FILE}."
     else
diff --git a/monitor/sre-agent-event-lab/scripts/lab_state.py b/monitor/sre-agent-event-lab/scripts/lab_state.py
index d7b50d3a..9f2ceada 100755
--- a/monitor/sre-agent-event-lab/scripts/lab_state.py
+++ b/monitor/sre-agent-event-lab/scripts/lab_state.py
@@ -609,6 +609,16 @@ def record_capture(
             )
         entry = self._scenario(scenario)
         run_status = entry.get("run_status")
+        previous_capture_status = entry.get("capture_status")
+        if (
+            previous_capture_status == SUCCESSFUL_CAPTURE
+            and capture_status != SUCCESSFUL_CAPTURE
+        ):
+            raise InvalidTransition(
+                "{0} already has a conclusion for its current run. Re-run the "
+                "scenario before collecting another capture; the successful "
+                "result will not be replaced by {1}.".format(scenario, capture_status)
+            )
         if capture_status == SUCCESSFUL_CAPTURE and run_status != RUN_RECOVERED:
             raise InvalidTransition(
                 "Cannot record a {0} for {1}: its run is {2}, not {3}. Only a "
@@ -758,6 +768,11 @@ def parse_args(argv: Optional[Sequence[str]] = None) -> argparse.Namespace:
     capture.add_argument("--status", choices=CAPTURE_STATES)
     capture.add_argument("--evidence-dir")
 
+    capture_status = commands.add_parser(
+        "capture-status", help="print a scenario's recorded capture status"
+    )
+    capture_status.add_argument("scenario", choices=SCENARIOS)
+
     evidence = commands.add_parser("evidence-dir", help="print a scenario's evidence dir")
     evidence.add_argument("scenario", choices=SCENARIOS)
 
@@ -809,6 +824,9 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
             state.record_capture(args.scenario, status, args.evidence_dir)
             print(status)
             return 0
+        if args.command == "capture-status":
+            print(state.capture_status(args.scenario) or "")
+            return 0
         if args.command == "evidence-dir":
             directory = state.evidence_dir(args.scenario)
             if not directory:
diff --git a/monitor/sre-agent-event-lab/scripts/query-evidence.sh b/monitor/sre-agent-event-lab/scripts/query-evidence.sh
index af6471da..038f7894 100755
--- a/monitor/sre-agent-event-lab/scripts/query-evidence.sh
+++ b/monitor/sre-agent-event-lab/scripts/query-evidence.sh
@@ -17,7 +17,6 @@ readonly END_UTC="$4"
 require_lab_config
 verify_subscription
 verify_lab_resource_group
-mkdir -p "${EVIDENCE_DIR}"
 
 APP_NAME="$(deployment_output containerAppName)"
 WORKSPACE_CUSTOMER_ID="$(deployment_output workspaceCustomerId)"
@@ -27,6 +26,19 @@ TELEMETRY_SERVICE_NAME="$(deployment_output telemetryServiceName)"
 readonly APP_NAME WORKSPACE_CUSTOMER_ID WORKLOAD_PRINCIPAL_ID
 readonly STORAGE_CONTAINER_SCOPE TELEMETRY_SERVICE_NAME
 
+MISSING_OUTPUTS=""
+[[ -n "${APP_NAME}" ]] || MISSING_OUTPUTS="${MISSING_OUTPUTS} containerAppName"
+[[ -n "${WORKSPACE_CUSTOMER_ID}" ]] || MISSING_OUTPUTS="${MISSING_OUTPUTS} workspaceCustomerId"
+[[ -n "${WORKLOAD_PRINCIPAL_ID}" ]] || MISSING_OUTPUTS="${MISSING_OUTPUTS} containerAppPrincipalId"
+[[ -n "${STORAGE_CONTAINER_SCOPE}" ]] || MISSING_OUTPUTS="${MISSING_OUTPUTS} storageContainerScope"
+[[ -n "${TELEMETRY_SERVICE_NAME}" ]] || MISSING_OUTPUTS="${MISSING_OUTPUTS} telemetryServiceName"
+if [[ -n "${MISSING_OUTPUTS}" ]]; then
+  echo "Missing deployment outputs:${MISSING_OUTPUTS}. Run: azd provision" >&2
+  exit 1
+fi
+
+mkdir -p "${EVIDENCE_DIR}"
+
 query_workspace() {
   local query="$1"
   local output_file="$2"
diff --git a/monitor/sre-agent-event-lab/scripts/run-scenario.sh b/monitor/sre-agent-event-lab/scripts/run-scenario.sh
index 6d959f32..fe2c4901 100755
--- a/monitor/sre-agent-event-lab/scripts/run-scenario.sh
+++ b/monitor/sre-agent-event-lab/scripts/run-scenario.sh
@@ -23,13 +23,15 @@ verify_lab_resource_group
 lab_state require-run "${SCENARIO}"
 
 # Overridable only for tests; production runs use the defaults.
-readonly ALERT_RESOLVE_TIMEOUT_SECONDS="${LAB_ALERT_RESOLVE_TIMEOUT_SECONDS:-900}"
+readonly ALERT_RESOLVE_TIMEOUT_SECONDS="${LAB_ALERT_RESOLVE_TIMEOUT_SECONDS:-1500}"
 readonly ALERT_RESOLVE_POLL_INTERVAL_SECONDS="${LAB_ALERT_RESOLVE_POLL_INTERVAL_SECONDS:-20}"
 readonly RECOVERY_HEALTH_TIMEOUT_SECONDS="${LAB_RECOVERY_HEALTH_TIMEOUT_SECONDS:-600}"
 readonly ALERT_FIRE_TIMEOUT_SECONDS="${LAB_ALERT_FIRE_TIMEOUT_SECONDS:-720}"
 readonly ALERT_FIRE_POLL_INTERVAL_SECONDS="${LAB_ALERT_FIRE_POLL_INTERVAL_SECONDS:-20}"
 readonly REVISION_READY_TIMEOUT_SECONDS="${LAB_REVISION_READY_TIMEOUT_SECONDS:-600}"
 readonly REVISION_READY_POLL_INTERVAL_SECONDS="${LAB_REVISION_READY_POLL_INTERVAL_SECONDS:-10}"
+readonly S3_PROPAGATION_TIMEOUT_SECONDS="${LAB_S3_PROPAGATION_TIMEOUT_SECONDS:-300}"
+readonly S3_PROPAGATION_POLL_INTERVAL_SECONDS="${LAB_S3_PROPAGATION_POLL_INTERVAL_SECONDS:-10}"
 
 APP_NAME="$(deployment_output containerAppName)"
 APP_FQDN="$(deployment_output containerAppFqdn)"
@@ -39,6 +41,19 @@ BLOB_ROLE_ASSIGNMENT_NAME="$(deployment_output blobRoleAssignmentName)"
 readonly APP_NAME APP_FQDN WORKLOAD_PRINCIPAL_ID STORAGE_CONTAINER_SCOPE
 readonly BLOB_ROLE_ASSIGNMENT_NAME
 
+MISSING_OUTPUTS=""
+[[ -n "${APP_NAME}" ]] || MISSING_OUTPUTS="${MISSING_OUTPUTS} containerAppName"
+[[ -n "${APP_FQDN}" ]] || MISSING_OUTPUTS="${MISSING_OUTPUTS} containerAppFqdn"
+if [[ "${SCENARIO}" == "s3" ]]; then
+  [[ -n "${WORKLOAD_PRINCIPAL_ID}" ]] || MISSING_OUTPUTS="${MISSING_OUTPUTS} containerAppPrincipalId"
+  [[ -n "${STORAGE_CONTAINER_SCOPE}" ]] || MISSING_OUTPUTS="${MISSING_OUTPUTS} storageContainerScope"
+  [[ -n "${BLOB_ROLE_ASSIGNMENT_NAME}" ]] || MISSING_OUTPUTS="${MISSING_OUTPUTS} blobRoleAssignmentName"
+fi
+if [[ -n "${MISSING_OUTPUTS}" ]]; then
+  echo "Missing deployment outputs:${MISSING_OUTPUTS}. Run: azd provision" >&2
+  exit 1
+fi
+
 EVIDENCE_DIR="$(evidence_dir_path "${SCENARIO}")"
 readonly EVIDENCE_DIR
 
@@ -57,8 +72,15 @@ readonly EVIDENCE_DIR
 # created afterwards, so a refusal leaves nothing behind in `evidence/`.
 lab_state begin-run "${SCENARIO}" "${EVIDENCE_DIR}"
 mkdir -p "${EVIDENCE_DIR}"
+readonly ALERT_LIST_RESPONSE_FILE="${EVIDENCE_DIR}/alert-list-response.json"
+readonly ALERT_LIST_ERROR_FILE="${EVIDENCE_DIR}/alert-list-error.log"
+readonly ALERT_LIST_ATTEMPT_ERROR_FILE="${EVIDENCE_DIR}/.alert-list-error.tmp"
 
 RECOVERED=0
+OUTCOME_RECORDED=0
+ALERT_LIST_FAILURES=0
+ALERT_LIST_VALID_RESPONSES=0
+ALERT_LIST_POLLS=0
 INJECTED_AT=""
 REVISION_READY_AT=""
 ROLE_DELETED_AT=""
@@ -66,6 +88,35 @@ ALERT_RULE_NAME=""
 ALERT_ID=""
 ALERT_FIRED_AT=""
 
+record_failed_run() {
+  local reason="$1"
+  if ! lab_state mark-failed "${SCENARIO}" "${EVIDENCE_DIR}" --reason "${reason}"; then
+    return 1
+  fi
+  OUTCOME_RECORDED=1
+}
+
+wait_for_s3_fault() {
+  local started="${SECONDS}"
+  local probe_file="${EVIDENCE_DIR}/rbac-probe.json"
+
+  while (( SECONDS - started < S3_PROPAGATION_TIMEOUT_SECONDS )); do
+    if python3 "${SCRIPT_DIR}/loadgen.py" \
+      "https://${APP_FQDN}/api/documents" \
+      --requests 1 \
+      --concurrency 1 \
+      --expect-status 503 \
+      --output "${probe_file}"; then
+      return 0
+    fi
+    echo "Waiting for the Storage RBAC deletion to reach the data plane..." >&2
+    sleep "${S3_PROPAGATION_POLL_INTERVAL_SECONDS}"
+  done
+
+  echo "Storage RBAC deletion did not produce HTTP 503 within ${S3_PROPAGATION_TIMEOUT_SECONDS}s." >&2
+  return 1
+}
+
 # restore_container_app_env SETTING -- reverts one injected Container App
 # setting and waits for the revision that carries it to become active.
 #
@@ -153,10 +204,24 @@ recover() {
 
 recover_on_exit() {
   local original_status="$?"
+  local recovery_status=0
+  local failure_reason
   trap - EXIT
   if ! recover; then
+    recovery_status=1
     echo "CRITICAL: scenario recovery failed for ${SCENARIO}." >&2
     echo "CRITICAL: the injected fault is still active. Revert it by hand before running any other scenario." >&2
+  fi
+  if [[ "${OUTCOME_RECORDED}" -eq 0 ]]; then
+    failure_reason="run aborted with status ${original_status}"
+    if [[ "${recovery_status}" -ne 0 ]]; then
+      failure_reason="${failure_reason}; automatic recovery failed"
+    fi
+    if ! record_failed_run "${failure_reason}"; then
+      echo "CRITICAL: could not record the aborted ${SCENARIO} run as failed." >&2
+    fi
+  fi
+  if [[ "${recovery_status}" -ne 0 ]]; then
     exit 1
   fi
   exit "${original_status}"
@@ -216,6 +281,11 @@ case "${SCENARIO}" in
     INJECTED_AT="$(utc_now)"
     az role assignment delete --ids "${ROLE_ASSIGNMENT_ID}"
     ROLE_DELETED_AT="$(utc_now)"
+    if ! wait_for_s3_fault; then
+      S3_PROPAGATION_FAILURE="Storage RBAC deletion did not produce HTTP 503 within ${S3_PROPAGATION_TIMEOUT_SECONDS}s."
+      record_failed_run "${S3_PROPAGATION_FAILURE}"
+      exit 1
+    fi
     python3 "${SCRIPT_DIR}/loadgen.py" \
       "https://${APP_FQDN}/api/documents" \
       --requests 60 \
@@ -228,13 +298,41 @@ readonly ALERT_RULE_NAME
 
 started="${SECONDS}"
 while (( SECONDS - started < ALERT_FIRE_TIMEOUT_SECONDS )); do
-  alerts_json="$(az rest \
+  ALERT_LIST_POLLS=$((ALERT_LIST_POLLS + 1))
+  if ! az rest \
     --method get \
-    --url "https://management.azure.com/subscriptions/${SUBSCRIPTION_ID}/providers/Microsoft.AlertsManagement/alerts?api-version=2019-03-01&targetResourceGroup=${RESOURCE_GROUP}&monitorCondition=Fired")"
-  ALERT_ID="$(jq -r \
+    --url "https://management.azure.com/subscriptions/${SUBSCRIPTION_ID}/providers/Microsoft.AlertsManagement/alerts?api-version=2019-03-01&targetResourceGroup=${RESOURCE_GROUP}&monitorCondition=Fired" \
+    >"${ALERT_LIST_RESPONSE_FILE}" 2>"${ALERT_LIST_ATTEMPT_ERROR_FILE}"; then
+    ALERT_LIST_FAILURES=$((ALERT_LIST_FAILURES + 1))
+    mv "${ALERT_LIST_ATTEMPT_ERROR_FILE}" "${ALERT_LIST_ERROR_FILE}"
+    echo "Azure Alerts list request failed; the last error is in ${ALERT_LIST_ERROR_FILE}. Retrying within the ${ALERT_FIRE_TIMEOUT_SECONDS}s budget." >&2
+    sleep "${ALERT_FIRE_POLL_INTERVAL_SECONDS}"
+    continue
+  fi
+  rm -f "${ALERT_LIST_ATTEMPT_ERROR_FILE}"
+  if [[ ! -s "${ALERT_LIST_RESPONSE_FILE}" ]]; then
+    ALERT_LIST_FAILURES=$((ALERT_LIST_FAILURES + 1))
+    ALERT_ID=""
+    echo "Azure Alerts list returned an empty response." >"${ALERT_LIST_ERROR_FILE}"
+    echo "Azure Alerts list returned an empty response; details are in ${ALERT_LIST_ERROR_FILE}. Retrying within the ${ALERT_FIRE_TIMEOUT_SECONDS}s budget." >&2
+    sleep "${ALERT_FIRE_POLL_INTERVAL_SECONDS}"
+    continue
+  fi
+  alerts_json="$(cat "${ALERT_LIST_RESPONSE_FILE}")"
+  candidate_alert_id=""
+  if ! candidate_alert_id="$(jq -r \
     --arg rule "${ALERT_RULE_NAME}" \
     'first(.value[] | select(.properties.essentials.alertRule | endswith($rule)) | .id) // ""' \
-    <<<"${alerts_json}")"
+    <<<"${alerts_json}" 2>/dev/null)"; then
+    ALERT_LIST_FAILURES=$((ALERT_LIST_FAILURES + 1))
+    ALERT_ID=""
+    printf 'Azure Alerts list returned invalid JSON. Response: %s\n' "${alerts_json}" >"${ALERT_LIST_ERROR_FILE}"
+    echo "Azure Alerts list returned invalid JSON; details are in ${ALERT_LIST_ERROR_FILE}. Retrying within the ${ALERT_FIRE_TIMEOUT_SECONDS}s budget." >&2
+    sleep "${ALERT_FIRE_POLL_INTERVAL_SECONDS}"
+    continue
+  fi
+  ALERT_LIST_VALID_RESPONSES=$((ALERT_LIST_VALID_RESPONSES + 1))
+  ALERT_ID="${candidate_alert_id}"
   if [[ -n "${ALERT_ID}" ]]; then
     ALERT_FIRED_AT="$(jq -r \
       --arg id "${ALERT_ID}" \
@@ -252,17 +350,23 @@ done
 # attempt as anything but failed. The EXIT trap (still armed here) still
 # reverts whatever was injected above.
 if [[ -z "${ALERT_ID}" ]]; then
-  ALERT_NEVER_FIRED_REASON="alert ${ALERT_RULE_NAME} did not fire within ${ALERT_FIRE_TIMEOUT_SECONDS}s."
-  lab_state mark-failed "${SCENARIO}" "${EVIDENCE_DIR}" --reason "${ALERT_NEVER_FIRED_REASON}"
+  if [[ "${ALERT_LIST_VALID_RESPONSES}" -eq 0 && "${ALERT_LIST_FAILURES}" -gt 0 ]]; then
+    ALERT_NEVER_FIRED_REASON="could not query Azure Alerts after ${ALERT_LIST_FAILURES} failed attempts within ${ALERT_FIRE_TIMEOUT_SECONDS}s; last error: ${ALERT_LIST_ERROR_FILE}"
+  else
+    ALERT_NEVER_FIRED_REASON="alert ${ALERT_RULE_NAME} did not fire within ${ALERT_FIRE_TIMEOUT_SECONDS}s."
+    if [[ "${ALERT_LIST_FAILURES}" -gt 0 ]]; then
+      ALERT_NEVER_FIRED_REASON="${ALERT_NEVER_FIRED_REASON} ${ALERT_LIST_FAILURES} of ${ALERT_LIST_POLLS} polls failed; last error: ${ALERT_LIST_ERROR_FILE}"
+    fi
+  fi
   echo "${ALERT_NEVER_FIRED_REASON}" >&2
   echo "Evidence directory: ${EVIDENCE_DIR}" >&2
+  record_failed_run "${ALERT_NEVER_FIRED_REASON}"
   exit 1
 fi
 
 recover
 RECOVERED_AT="$(utc_now)"
 readonly RECOVERED_AT
-trap - EXIT
 
 # Recovery is only real when the workload is healthy again *and* Azure
 # Monitor closed the alert this run fired. Both are waited for before any
@@ -313,12 +417,13 @@ jq -n \
   }' | tee "${EVIDENCE_DIR}/timeline.json"
 
 if [[ "${RECOVERY_CONFIRMED}" -ne 1 ]]; then
-  lab_state mark-failed "${SCENARIO}" "${EVIDENCE_DIR}" --reason "${RECOVERY_FAILURE}"
   echo "Scenario ${SCENARIO} is recorded as failed: ${RECOVERY_FAILURE}." >&2
   echo "Evidence directory: ${EVIDENCE_DIR}" >&2
+  record_failed_run "${RECOVERY_FAILURE}"
   exit 1
 fi
 
 lab_state mark-recovered "${SCENARIO}" "${EVIDENCE_DIR}"
+OUTCOME_RECORDED=1
 
 printf 'Evidence directory: %s\n' "${EVIDENCE_DIR}"
diff --git a/monitor/sre-agent-event-lab/scripts/tests/lab_script_harness.py b/monitor/sre-agent-event-lab/scripts/tests/lab_script_harness.py
index dd5ce816..6a7565ea 100644
--- a/monitor/sre-agent-event-lab/scripts/tests/lab_script_harness.py
+++ b/monitor/sre-agent-event-lab/scripts/tests/lab_script_harness.py
@@ -25,6 +25,7 @@
 # images (`loadgen.py`, `capture_agent.py`, `render_capture.py`) and hand
 # every other script to the real interpreter running the suite.
 REAL_PYTHON = sys.executable
+REAL_JQ = shutil.which("jq") or "/usr/bin/jq"
 
 SUBSCRIPTION_ID = "11111111-2222-3333-4444-555555555555"
 RESOURCE_GROUP = "rg-sre-lab-exec"
@@ -226,7 +227,21 @@ def _az_stub_source(log_path, state_dir):
       printf '{{"properties": {{"principalId": "%s", "roleDefinitionId": "/subscriptions/{SUBSCRIPTION_ID}/providers/Microsoft.Authorization/roleDefinitions/{MONITORING_CONTRIBUTOR_ROLE_ID}", "scope": "/subscriptions/{SUBSCRIPTION_ID}"}}}}\\n' \\
         "${{principal_id}}"
     elif [[ "$*" == *"monitorCondition=Fired"* ]]; then
-      if [[ -f "${{state}}/alert_never_fires" ]]; then
+      if [[ -f "${{state}}/alert_failure_after_success" ]] && [[ -f "${{state}}/alert_list_seen_success" ]]; then
+        printf 'ERROR: (Forbidden) alert read permission expired.\n' >&2
+        exit 1
+      elif [[ -f "${{state}}/alert_failure_after_success" ]]; then
+        : > "${{state}}/alert_list_seen_success"
+        printf '{{"value": []}}\\n'
+      elif [[ -f "${{state}}/alert_list_failures" ]] && (( $(cat "${{state}}/alert_list_failures") > 0 )); then
+        printf '%s\n' "$(( $(cat "${{state}}/alert_list_failures") - 1 ))" > "${{state}}/alert_list_failures"
+        printf 'ERROR: (TooManyRequests) transient alert list failure.\n' >&2
+        exit 1
+      elif [[ -f "${{state}}/alert_list_invalid_json" ]]; then
+        printf '%s trailing-garbage\n' '{ALERTS_JSON}'
+      elif [[ -f "${{state}}/alert_list_empty_body" ]]; then
+        :
+      elif [[ -f "${{state}}/alert_never_fires" ]]; then
         printf '{{"value": []}}\\n'
       else
         printf '%s\\n' '{ALERTS_JSON}'
@@ -266,7 +281,7 @@ def _begin_run_probe(probe_path):
 
 
 def _lab_python_stub_source(
-    log_path, capture_timeline, pillow_importable=True, probe_path=None
+    log_path, state_dir, capture_timeline, pillow_importable=True, probe_path=None
 ):
     """A fake `${LAB_ROOT}/app/.venv/bin/python`.
 
@@ -286,11 +301,18 @@ def _lab_python_stub_source(
     probe = _begin_run_probe(probe_path) if probe_path else ""
     return f"""#!/usr/bin/env bash
 printf '%s\\n' "$*" >> "{log_path}"
+state="{state_dir}"
 {probe}
 if [[ "${{1:-}}" == "-c" && "${{2:-}}" == *PIL* ]]; then
   exit {pil_exit}
 fi
 case "${{1:-}}" in
+  *lab_state.py)
+    if [[ "$*" == *" mark-failed "* && -f "${{state}}/mark_failed_fails" ]]; then
+      exit 3
+    fi
+    exec "{REAL_PYTHON}" "$@"
+    ;;
   *capture_agent.py)
     shift
     output_dir=""
@@ -307,6 +329,9 @@ def _lab_python_stub_source(
     normalized="${{1:-}}"
     asset_dir="${{2:-}}"
     [[ -f "${{normalized}}" ]] || exit 1
+    if [[ -f "${{state}}/render_fails" ]]; then
+      exit 5
+    fi
     mkdir -p "${{asset_dir}}"
     printf 'GIF89a' > "${{asset_dir}}/investigation.gif"
     printf 'timeline\\n' > "${{asset_dir}}/timeline.mmd"
@@ -321,19 +346,48 @@ def _lab_python_stub_source(
 """
 
 
-def _python3_stub_source(log_path, probe_path=None):
+def _python3_stub_source(log_path, state_dir, probe_path=None):
     """A fake `python3`: `loadgen.py` is faked, everything else is real."""
     probe = _begin_run_probe(probe_path) if probe_path else ""
     return f"""#!/usr/bin/env bash
 printf '%s\\n' "$*" >> "{log_path}"
+state="{state_dir}"
 {probe}
 case "${{1:-}}" in
-  *loadgen.py) exit 0 ;;
+  *lab_state.py)
+    if [[ "$*" == *" mark-failed "* && -f "${{state}}/mark_failed_fails" ]]; then
+      exit 3
+    fi
+    exec "{REAL_PYTHON}" "$@"
+    ;;
+  *loadgen.py)
+    if [[ -f "${{state}}/loadgen_fails" ]]; then
+      exit 2
+    fi
+    if [[ "$*" == *"/api/documents"* && "$*" == *"--requests 1"* ]] \
+      && [[ -f "${{state}}/s3_probe_failures" ]] \
+      && (( $(cat "${{state}}/s3_probe_failures") > 0 )); then
+      printf '%s\n' "$(( $(cat "${{state}}/s3_probe_failures") - 1 ))" > "${{state}}/s3_probe_failures"
+      exit 2
+    fi
+    exit 0
+    ;;
   *) exec "{REAL_PYTHON}" "$@" ;;
 esac
 """
 
 
+def _jq_stub_source(timeline_jq_fails):
+    if not timeline_jq_fails:
+        return None
+    return f"""#!/usr/bin/env bash
+if [[ "${{1:-}}" == "-n" ]]; then
+  exit 4
+fi
+exec "{REAL_JQ}" "$@"
+"""
+
+
 def make_lab(
     tmp_path,
     azd_values=None,
@@ -347,6 +401,15 @@ def make_lab(
     recovery_revision_stalls=False,
     role_create_fails=False,
     injection_update_fails=False,
+    alert_list_failures=0,
+    alert_list_invalid_json=False,
+    loadgen_fails=False,
+    s3_probe_failures=0,
+    mark_failed_fails=False,
+    timeline_jq_fails=False,
+    alert_failure_after_success=False,
+    render_fails=False,
+    alert_list_empty_body=False,
 ):
     """A throwaway copy of the lab plus fake CLIs; returns a run context."""
     lab = tmp_path / "lab"
@@ -376,6 +439,22 @@ def make_lab(
         (state_dir / "role_create_fails").write_text("1\n")
     if injection_update_fails:
         (state_dir / "injection_update_fails").write_text("1\n")
+    if alert_list_failures:
+        (state_dir / "alert_list_failures").write_text(f"{alert_list_failures}\n")
+    if alert_list_invalid_json:
+        (state_dir / "alert_list_invalid_json").write_text("1\n")
+    if loadgen_fails:
+        (state_dir / "loadgen_fails").write_text("1\n")
+    if s3_probe_failures:
+        (state_dir / "s3_probe_failures").write_text(f"{s3_probe_failures}\n")
+    if mark_failed_fails:
+        (state_dir / "mark_failed_fails").write_text("1\n")
+    if alert_failure_after_success:
+        (state_dir / "alert_failure_after_success").write_text("1\n")
+    if render_fails:
+        (state_dir / "render_fails").write_text("1\n")
+    if alert_list_empty_body:
+        (state_dir / "alert_list_empty_body").write_text("1\n")
 
     az_log = tmp_path / "az-calls.log"
     azd_log = tmp_path / "azd-calls.log"
@@ -391,8 +470,12 @@ def make_lab(
         azd_log,
     )
     write_executable(
-        bin_dir / "python3", _python3_stub_source(python_log, begin_run_probe)
+        bin_dir / "python3",
+        _python3_stub_source(python_log, state_dir, begin_run_probe),
     )
+    jq_stub = _jq_stub_source(timeline_jq_fails)
+    if jq_stub:
+        write_executable(bin_dir / "jq", jq_stub)
 
     venv_bin = lab / "app" / ".venv" / "bin"
     if venv_present:
@@ -400,7 +483,11 @@ def make_lab(
         write_executable(
             venv_bin / "python",
             _lab_python_stub_source(
-                lab_python_log, capture_timeline, pillow_importable, begin_run_probe
+                lab_python_log,
+                state_dir,
+                capture_timeline,
+                pillow_importable,
+                begin_run_probe,
             ),
         )
 
@@ -413,6 +500,7 @@ def make_lab(
         workdir,
         az_log,
         azd_log,
+        python_log,
         lab_python_log,
         state_dir,
         begin_run_probe,
@@ -427,6 +515,7 @@ def __init__(
         workdir,
         az_log,
         azd_log,
+        python_log,
         lab_python_log,
         state_dir,
         begin_run_probe,
@@ -436,6 +525,7 @@ def __init__(
         self.workdir = workdir
         self.az_log = az_log
         self.azd_log = azd_log
+        self.python_log = python_log
         self.lab_python_log = lab_python_log
         self.state_dir = state_dir
         self.begin_run_probe = begin_run_probe
@@ -472,6 +562,9 @@ def az_calls(self):
     def azd_calls(self):
         return self.azd_log.read_text() if self.azd_log.exists() else ""
 
+    def python_calls(self):
+        return self.python_log.read_text() if self.python_log.exists() else ""
+
     def write_agent_setup(self, principal_ids=("principal-one", "principal-two")):
         assignment_ids = [
             f"/subscriptions/{SUBSCRIPTION_ID}/providers"
diff --git a/monitor/sre-agent-event-lab/scripts/tests/test_cleanup_external.py b/monitor/sre-agent-event-lab/scripts/tests/test_cleanup_external.py
index 495b46f8..4f4fa08b 100644
--- a/monitor/sre-agent-event-lab/scripts/tests/test_cleanup_external.py
+++ b/monitor/sre-agent-event-lab/scripts/tests/test_cleanup_external.py
@@ -387,6 +387,21 @@ def test_cleanup_succeeds_when_the_evidence_records_no_assignment(tmp_path):
     assert "no subscription role assignment" in run.stdout
 
 
+def test_cleanup_refuses_an_empty_assignment_recorded_with_a_principal(tmp_path):
+    run = run_cleanup(
+        tmp_path,
+        ["--yes"],
+        evidence=agent_setup(monitoring_assignment_id=""),
+    )
+
+    assert run.returncode != 0
+    assert "monitoring_contributor_assignment_id is empty" in run.stderr
+    assert AGENT_PRINCIPAL_ID in run.stderr
+    assert "az role assignment list" in run.stderr
+    assert "agent_principal_id" in run.stderr
+    assert "role assignment delete" not in run.az_calls
+
+
 def test_cleanup_refuses_an_assignment_recorded_without_its_principal(tmp_path):
     """Without the principal the record cannot be verified, and an
     unverifiable deletion is exactly what this hook must never do."""
@@ -403,6 +418,8 @@ def test_cleanup_refuses_malformed_evidence(tmp_path):
     run = run_cleanup(tmp_path, ["--yes"], raw_evidence="{not json at all")
 
     assert run.returncode != 0
+    assert "guides/01-agent-setup.md" in run.stderr
+    assert "agent-setup.json" in run.stderr
     assert "role assignment delete" not in run.az_calls
 
 
@@ -584,3 +601,6 @@ def test_readme_documents_the_teardown_hooks_and_manual_recovery():
         "the README must name the recovery step (re-running Agent setup/role "
         "assignment) an operator needs after canceling"
     )
+    assert "agent-setup.json" in section
+    assert "새 할당 ID" in section
+    assert "직접 갱신한 뒤" in section
diff --git a/monitor/sre-agent-event-lab/scripts/tests/test_common.py b/monitor/sre-agent-event-lab/scripts/tests/test_common.py
index 020fa76f..15ede579 100644
--- a/monitor/sre-agent-event-lab/scripts/tests/test_common.py
+++ b/monitor/sre-agent-event-lab/scripts/tests/test_common.py
@@ -300,6 +300,13 @@ def test_run_scenario_records_recovery_only_after_health_and_alert_checks():
     assert "lab_state mark-failed" in script
 
 
+def test_run_scenario_default_alert_resolution_budget_covers_stateful_log_alerts():
+    script = RUN_SCENARIO_SH.read_text()
+
+    assert 'LAB_ALERT_RESOLVE_TIMEOUT_SECONDS:-1500' in script
+    assert 'local timeout_seconds="${2:-1500}"' in COMMON_SH.read_text()
+
+
 def test_capture_scenario_records_the_terminal_state_from_the_timeline():
     """The capture status is derived from the normalized timeline, so a
     missing thread/investigation/conclusion is recorded as itself and can
diff --git a/monitor/sre-agent-event-lab/scripts/tests/test_doctor.py b/monitor/sre-agent-event-lab/scripts/tests/test_doctor.py
index 3ae38eea..2da687bd 100644
--- a/monitor/sre-agent-event-lab/scripts/tests/test_doctor.py
+++ b/monitor/sre-agent-event-lab/scripts/tests/test_doctor.py
@@ -8,6 +8,8 @@
 the script never queries Azure once its own safety gate (commands, login,
 azd configuration, subscription equality, resource-group tags) has failed.
 """
+import json
+
 import pytest
 
 from doctor_harness import (
@@ -242,9 +244,53 @@ def test_doctor_fails_when_agent_setup_evidence_is_missing(fake_az):
 
     assert result.returncode == 1
     assert "Reader role assignment\tFAIL" in result.stdout
+    assert "agent-setup.json" in result.stdout
+    assert "guides/01-agent-setup.md" in result.stdout
     assert "lab.sh acknowledge agent-setup" in result.stdout
 
 
+def test_doctor_fails_when_subscription_assignment_ids_are_empty(fake_az):
+    fake_az.agent_setup_body = json.dumps(
+        {
+            "agent_endpoint": "https://sre-agent.example.com/api/incidents",
+            "monitoring_contributor_assignment_id": "",
+            "agent_principal_id": AGENT_PRINCIPAL_ID,
+            "uami_monitoring_contributor_assignment_id": (
+                f"{SUBSCRIPTION_SCOPE}/providers/Microsoft.Authorization/"
+                "roleAssignments/principal-two"
+            ),
+            "agent_user_assigned_principal_id": AGENT_UAMI_PRINCIPAL_ID,
+        }
+    )
+
+    result = run_doctor(fake_az)
+
+    assert result.returncode == 1
+    assert "Monitoring Contributor cleanup evidence\tFAIL" in result.stdout
+    assert "monitoring_contributor_assignment_id" in result.stdout
+    assert "az role assignment list" in result.stdout
+
+
+def test_doctor_accepts_case_insensitive_arm_assignment_ids(fake_az):
+    assignment_prefix = (
+        f"{SUBSCRIPTION_SCOPE}/providers/microsoft.authorization/roleassignments/"
+    )
+    fake_az.agent_setup_body = json.dumps(
+        {
+            "agent_endpoint": "https://sre-agent.example.com/api/incidents",
+            "monitoring_contributor_assignment_id": assignment_prefix + "principal-one",
+            "agent_principal_id": AGENT_PRINCIPAL_ID,
+            "uami_monitoring_contributor_assignment_id": assignment_prefix + "principal-two",
+            "agent_user_assigned_principal_id": AGENT_UAMI_PRINCIPAL_ID,
+        }
+    )
+
+    result = run_doctor(fake_az)
+
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert "Monitoring Contributor cleanup evidence\tPASS" in result.stdout
+
+
 def test_doctor_fails_when_one_recorded_identity_lacks_reader(fake_az):
     fake_az.reader_role_assigned[AGENT_UAMI_PRINCIPAL_ID] = False
 
@@ -483,6 +529,8 @@ def test_doctor_fails_gracefully_on_malformed_agent_setup_evidence(fake_az):
     assert "Reader role assignment\tFAIL" in result.stdout
     detail = _detail_of(result, "Reader role assignment")
     assert "valid JSON" in detail
+    assert "agent-setup.json" in detail
+    assert "guides/01-agent-setup.md" in detail
     assert "lab.sh acknowledge agent-setup" in detail
 
 
diff --git a/monitor/sre-agent-event-lab/scripts/tests/test_lab_guides.py b/monitor/sre-agent-event-lab/scripts/tests/test_lab_guides.py
index 870d05b8..369f8000 100644
--- a/monitor/sre-agent-event-lab/scripts/tests/test_lab_guides.py
+++ b/monitor/sre-agent-event-lab/scripts/tests/test_lab_guides.py
@@ -431,6 +431,42 @@ def test_scenario_guides_say_a_rerun_retires_the_previous_attempt():
         assert re.search(r"(지워|지웁|초기화|사라)", joined), (name, joined)
 
 
+def test_s1_guide_documents_the_full_stateful_alert_resolution_budget():
+    text = (GUIDES / "02-scenario-s1.md").read_text()
+
+    assert "25분" in text
+
+
+def test_agent_setup_guide_creates_roles_and_captures_assignment_ids():
+    text = (GUIDES / "01-agent-setup.md").read_text()
+
+    assert text.count("az role assignment create") >= 4
+    assert "--assignee-object-id" in text
+    assert '"Reader"' in text
+    assert '"Monitoring Contributor"' in text
+    assert "--query id" in text
+    assert "MONITORING_CONTRIBUTOR_ASSIGNMENT_ID" in text
+    assert "UAMI_MONITORING_CONTRIBUTOR_ASSIGNMENT_ID" in text
+    assert "az role assignment list" in text
+    assert "jq -e" in text
+    assert "같은 셸" in text
+    assert "agent-setup.json is incomplete" in text
+    assert "exit 1" not in text
+    assert re.search(r"\n\s+false\n", text)
+    assert 'test("^https://[^<>]+$")' in text
+
+
+def test_s3_guide_documents_propagation_wait_and_manual_restore_command():
+    text = (GUIDES / "04-scenario-s3.md").read_text()
+
+    assert "최대 5분" in text
+    assert "did not produce HTTP 503 within 300s" in text
+    assert "AZURE_BLOB_ROLE_ASSIGNMENT_NAME" in text
+    assert "AZURE_CONTAINER_APP_PRINCIPAL_ID" in text
+    assert "AZURE_STORAGE_CONTAINER_SCOPE" in text
+    assert "az role assignment create" in text
+
+
 def test_validation_results_keeps_the_one_minute_static_run_and_explains_it():
     """The recorded S1/S2/S3 run used a one-minute static threshold. That
     result stays plausible because the later live failure was the legacy
diff --git a/monitor/sre-agent-event-lab/scripts/tests/test_lab_scripts.py b/monitor/sre-agent-event-lab/scripts/tests/test_lab_scripts.py
index ce67929c..6f37803d 100644
--- a/monitor/sre-agent-event-lab/scripts/tests/test_lab_scripts.py
+++ b/monitor/sre-agent-event-lab/scripts/tests/test_lab_scripts.py
@@ -13,6 +13,7 @@
 import pytest
 
 from lab_script_harness import (
+    AZD_VALUES,
     ENV_NAME,
     MISSING_CONCLUSION_TIMELINE,
     RESOURCE_GROUP,
@@ -31,6 +32,8 @@
     "LAB_RECOVERY_HEALTH_TIMEOUT_SECONDS": "5",
     "LAB_REVISION_READY_TIMEOUT_SECONDS": "5",
     "LAB_REVISION_READY_POLL_INTERVAL_SECONDS": "1",
+    "LAB_S3_PROPAGATION_TIMEOUT_SECONDS": "5",
+    "LAB_S3_PROPAGATION_POLL_INTERVAL_SECONDS": "1",
 }
 
 NO_ALERT_WAITS = dict(
@@ -253,6 +256,134 @@ def test_run_scenario_marks_failed_when_the_alert_never_fires(tmp_path):
     assert scenario_state["evidence_dir"] == str(evidence_dir)
 
 
+def test_run_scenario_retries_a_transient_alert_list_failure(tmp_path):
+    lab_run = make_lab(tmp_path, alert_list_failures=1)
+    lab_run.seed_state()
+
+    result = lab_run.run("run-scenario.sh", ["s1"], env=BOUNDED_WAITS)
+
+    assert result.returncode == 0, result.stdout + result.stderr
+    alert_reads = [
+        line
+        for line in lab_run.az_calls().splitlines()
+        if "monitorCondition=Fired" in line
+    ]
+    assert len(alert_reads) >= 2
+    assert lab_run.scenario_state("s1")["run_status"] == "recovered"
+
+
+def test_run_scenario_records_an_unexpected_abort_as_failed(tmp_path):
+    lab_run = make_lab(tmp_path, loadgen_fails=True)
+    lab_run.seed_state()
+
+    result = lab_run.run("run-scenario.sh", ["s1"], env=BOUNDED_WAITS)
+
+    assert result.returncode != 0
+    assert "FAILURE_MODE=none" in lab_run.az_calls()
+    scenario = lab_run.scenario_state("s1")
+    assert scenario["run_status"] == "failed"
+    assert "aborted" in scenario["failure_reason"]
+
+
+def test_run_scenario_reports_when_aborted_state_cannot_be_recorded(tmp_path):
+    lab_run = make_lab(tmp_path, loadgen_fails=True, mark_failed_fails=True)
+    lab_run.seed_state()
+
+    result = lab_run.run("run-scenario.sh", ["s1"], env=BOUNDED_WAITS)
+
+    assert result.returncode != 0
+    assert "CRITICAL: could not record the aborted s1 run as failed" in result.stderr
+    assert lab_run.scenario_state("s1")["run_status"] == "running"
+
+
+def test_run_scenario_records_a_post_recovery_timeline_failure(tmp_path):
+    lab_run = make_lab(tmp_path, timeline_jq_fails=True)
+    lab_run.seed_state()
+
+    result = lab_run.run("run-scenario.sh", ["s1"], env=BOUNDED_WAITS)
+
+    assert result.returncode != 0
+    scenario = lab_run.scenario_state("s1")
+    assert scenario["run_status"] == "failed"
+    assert "aborted" in scenario["failure_reason"]
+
+
+def test_run_scenario_discards_partial_jq_output_from_invalid_alert_json(tmp_path):
+    lab_run = make_lab(tmp_path, alert_list_invalid_json=True)
+    lab_run.seed_state()
+
+    result = lab_run.run("run-scenario.sh", ["s1"], env=NO_ALERT_WAITS)
+
+    assert result.returncode != 0
+    scenario = lab_run.scenario_state("s1")
+    assert scenario["run_status"] == "failed"
+    assert "could not query Azure Alerts" in scenario["failure_reason"]
+
+
+def test_run_scenario_records_persistent_alert_api_errors_separately(tmp_path):
+    lab_run = make_lab(tmp_path, alert_list_failures=100)
+    lab_run.seed_state()
+
+    result = lab_run.run("run-scenario.sh", ["s1"], env=NO_ALERT_WAITS)
+
+    assert result.returncode != 0
+    scenario = lab_run.scenario_state("s1")
+    assert "could not query Azure Alerts" in scenario["failure_reason"]
+    assert "did not fire" not in scenario["failure_reason"]
+    error_file = Path(scenario["evidence_dir"]) / "alert-list-error.log"
+    assert error_file.is_file()
+    assert "TooManyRequests" in error_file.read_text()
+
+
+def test_run_scenario_reports_alert_errors_after_an_earlier_valid_poll(tmp_path):
+    lab_run = make_lab(
+        tmp_path,
+        alert_fires=False,
+        alert_failure_after_success=True,
+    )
+    lab_run.seed_state()
+
+    result = lab_run.run("run-scenario.sh", ["s1"], env=NO_ALERT_WAITS)
+
+    assert result.returncode != 0
+    reason = lab_run.scenario_state("s1")["failure_reason"]
+    assert "did not fire" in reason
+    assert "polls failed" in reason
+    assert "alert-list-error.log" in reason
+    error_file = Path(lab_run.scenario_state("s1")["evidence_dir"]) / "alert-list-error.log"
+    assert "Forbidden" in error_file.read_text()
+
+
+def test_run_scenario_rejects_an_empty_successful_alert_response(tmp_path):
+    lab_run = make_lab(tmp_path, alert_list_empty_body=True)
+    lab_run.seed_state()
+
+    result = lab_run.run("run-scenario.sh", ["s1"], env=NO_ALERT_WAITS)
+
+    assert result.returncode != 0
+    reason = lab_run.scenario_state("s1")["failure_reason"]
+    assert "could not query Azure Alerts" in reason
+    assert "empty response" in (
+        Path(lab_run.scenario_state("s1")["evidence_dir"]) / "alert-list-error.log"
+    ).read_text()
+
+
+def test_run_scenario_preserves_reason_when_failure_state_write_fails(tmp_path):
+    lab_run = make_lab(
+        tmp_path,
+        alert_fires=False,
+        mark_failed_fails=True,
+    )
+    lab_run.seed_state()
+
+    result = lab_run.run("run-scenario.sh", ["s1"], env=NO_ALERT_WAITS)
+
+    assert result.returncode != 0
+    assert "did not fire" in result.stderr
+    assert "Evidence directory:" in result.stderr
+    assert "CRITICAL: could not record" in result.stderr
+
+
 def test_run_scenario_s1_reports_a_rejected_recovery_update_as_critical(tmp_path):
     """`recover` runs both directly and from the EXIT trap, and the trap
     calls it as `if ! recover`, which turns `set -e` off for the whole
@@ -346,6 +477,72 @@ def test_run_scenario_s3_recovers_and_records_a_successful_run(tmp_path):
     assert lab_run.scenario_state("s3")["run_status"] == "recovered"
 
 
+def test_run_scenario_s3_waits_for_rbac_revocation_before_the_final_load(tmp_path):
+    lab_run = make_lab(tmp_path, s3_probe_failures=2)
+    lab_run.seed_state(
+        scenarios=dict(
+            captured("s1", tmp_path / "s1"), **captured("s2", tmp_path / "s2")
+        )
+    )
+
+    result = lab_run.run("run-scenario.sh", ["s3"], env=BOUNDED_WAITS)
+
+    assert result.returncode == 0, result.stdout + result.stderr
+    calls = lab_run.python_calls().splitlines()
+    probe_indexes = [
+        index
+        for index, call in enumerate(calls)
+        if "/api/documents" in call and "--requests 1" in call
+    ]
+    final_index = next(
+        index
+        for index, call in enumerate(calls)
+        if "/api/documents" in call and "--requests 60" in call
+    )
+    assert len(probe_indexes) >= 3
+    assert max(probe_indexes) < final_index
+
+
+def test_run_scenario_s3_records_the_propagation_timeout_reason(tmp_path):
+    lab_run = make_lab(tmp_path, s3_probe_failures=1000)
+    lab_run.seed_state(
+        scenarios=dict(
+            captured("s1", tmp_path / "s1"), **captured("s2", tmp_path / "s2")
+        )
+    )
+    waits = dict(BOUNDED_WAITS, LAB_S3_PROPAGATION_TIMEOUT_SECONDS="2")
+
+    result = lab_run.run("run-scenario.sh", ["s3"], env=waits)
+
+    assert result.returncode != 0
+    scenario = lab_run.scenario_state("s3")
+    assert scenario["run_status"] == "failed"
+    assert "did not produce HTTP 503 within 2s" in scenario["failure_reason"]
+
+
+def test_run_scenario_s3_refuses_missing_recovery_outputs_before_deletion(tmp_path):
+    values = dict(AZD_VALUES)
+    values["containerAppPrincipalId"] = ""
+    values["AZURE_STORAGE_CONTAINER_SCOPE"] = ""
+    values["AZURE_BLOB_ROLE_ASSIGNMENT_NAME"] = ""
+    lab_run = make_lab(tmp_path, azd_values=values)
+    lab_run.seed_state(
+        scenarios=dict(
+            captured("s1", tmp_path / "s1"), **captured("s2", tmp_path / "s2")
+        )
+    )
+
+    result = lab_run.run("run-scenario.sh", ["s3"], env=BOUNDED_WAITS)
+
+    assert result.returncode != 0
+    assert "containerAppPrincipalId" in result.stderr
+    assert "storageContainerScope" in result.stderr
+    assert "blobRoleAssignmentName" in result.stderr
+    assert "azd provision" in result.stderr
+    assert "role assignment delete" not in lab_run.az_calls()
+    assert not sorted((lab_run.lab / "evidence").glob("s3-*"))
+
+
 def test_a_failed_run_blocks_the_next_scenario(tmp_path):
     lab_run = make_lab(tmp_path, alert_resolves=False)
     lab_run.seed_state()
@@ -692,6 +889,128 @@ def test_capture_scenario_records_a_missing_conclusion_as_itself(tmp_path):
     assert "s1_captured" in blocked.stderr
 
 
+def test_capture_scenario_refuses_to_replace_a_successful_capture(tmp_path):
+    lab_run = make_lab(tmp_path)
+    lab_run.write_agent_setup()
+    lab_run.seed_state()
+    run_result = lab_run.run("run-scenario.sh", ["s1"], env=BOUNDED_WAITS)
+    assert run_result.returncode == 0, run_result.stderr
+    first = lab_run.run("capture-scenario.sh", ["s1"])
+    assert first.returncode == 0, first.stdout + first.stderr
+    calls_before = lab_run.lab_python_log.read_text().count("capture_agent.py")
+
+    second = lab_run.run("capture-scenario.sh", ["s1"])
+
+    assert second.returncode != 0
+    assert "already has a conclusion" in second.stderr
+    assert "lab.sh run s1" in second.stderr
+    assert lab_run.scenario_state("s1")["capture_status"] == "conclusion"
+    assert lab_run.lab_python_log.read_text().count("capture_agent.py") == calls_before
+
+
+def test_capture_scenario_render_failure_names_the_direct_retry(tmp_path):
+    lab_run = make_lab(tmp_path, render_fails=True)
+    lab_run.write_agent_setup()
+    lab_run.seed_state()
+    run_result = lab_run.run("run-scenario.sh", ["s1"], env=BOUNDED_WAITS)
+    assert run_result.returncode == 0, run_result.stderr
+    first = lab_run.run("capture-scenario.sh", ["s1"])
+    assert first.returncode != 0
+    assert lab_run.scenario_state("s1")["capture_status"] == "conclusion"
+
+    second = lab_run.run("capture-scenario.sh", ["s1"])
+
+    assert second.returncode != 0
+    assert "render_capture.py" in second.stderr
+    assert "normalized-timeline.json" in second.stderr
+
+
+def test_query_evidence_refuses_missing_outputs_before_writing_artifacts(tmp_path):
+    values = dict(AZD_VALUES)
+    values["containerAppPrincipalId"] = ""
+    values["AZURE_STORAGE_CONTAINER_SCOPE"] = ""
+    lab_run = make_lab(tmp_path, azd_values=values)
+    evidence_dir = tmp_path / "evidence-out"
+
+    result = lab_run.run(
+        "query-evidence.sh",
+        ["s1", str(evidence_dir), "2026-08-14T00:00:00Z", "2026-08-14T01:00:00Z"],
+    )
+
+    assert result.returncode != 0
+    assert "containerAppPrincipalId" in result.stderr
+    assert "storageContainerScope" in result.stderr
+    assert "azd provision" in result.stderr
+    assert not evidence_dir.exists()
+
+
+def test_capture_scenario_missing_setup_names_the_file_and_guide(tmp_path):
+    lab_run = make_lab(tmp_path)
+    lab_run.seed_state()
+    run_result = lab_run.run("run-scenario.sh", ["s1"], env=BOUNDED_WAITS)
+    assert run_result.returncode == 0, run_result.stderr
+
+    result = lab_run.run("capture-scenario.sh", ["s1"])
+
+    assert result.returncode != 0
+    assert "evidence/agent-setup.json" in result.stderr
+    assert "guides/01-agent-setup.md" in result.stderr
+
+
+def test_capture_scenario_missing_endpoint_names_the_setup_file(tmp_path):
+    lab_run = make_lab(tmp_path)
+    setup_path = lab_run.write_agent_setup()
+    setup = json.loads(setup_path.read_text())
+    setup["agent_endpoint"] = ""
+    setup_path.write_text(json.dumps(setup))
+    lab_run.seed_state()
+    run_result = lab_run.run("run-scenario.sh", ["s1"], env=BOUNDED_WAITS)
+    assert run_result.returncode == 0, run_result.stderr
+
+    result = lab_run.run("capture-scenario.sh", ["s1"])
+
+    assert result.returncode != 0
+    assert "agent_endpoint" in result.stderr
+    assert "evidence/agent-setup.json" in result.stderr
+    assert "guides/01-agent-setup.md" in result.stderr
+
+
+def test_capture_scenario_rejects_a_placeholder_endpoint(tmp_path):
+    lab_run = make_lab(tmp_path)
+    setup_path = lab_run.write_agent_setup()
+    setup = json.loads(setup_path.read_text())
+    setup["agent_endpoint"] = "https://<agent>.<region>.azuresre.ai"
+    setup_path.write_text(json.dumps(setup))
+    lab_run.seed_state()
+    run_result = lab_run.run("run-scenario.sh", ["s1"], env=BOUNDED_WAITS)
+    assert run_result.returncode == 0, run_result.stderr
+
+    result = lab_run.run("capture-scenario.sh", ["s1"])
+
+    assert result.returncode != 0
+    assert "valid HTTPS" in result.stderr
+    assert "agent-setup.json" in result.stderr
+
+
+def test_capture_scenario_missing_alert_id_names_the_timeline_and_rerun(tmp_path):
+    lab_run = make_lab(tmp_path)
+    lab_run.write_agent_setup()
+    lab_run.seed_state()
+    run_result = lab_run.run("run-scenario.sh", ["s1"], env=BOUNDED_WAITS)
+    assert run_result.returncode == 0, run_result.stderr
+    evidence_dir = Path(lab_run.scenario_state("s1")["evidence_dir"])
+    timeline_path = evidence_dir / "timeline.json"
+    timeline = json.loads(timeline_path.read_text())
+    timeline["alert_id"] = ""
+    timeline_path.write_text(json.dumps(timeline))
+
+    result = lab_run.run("capture-scenario.sh", ["s1"])
+
+    assert result.returncode != 0
+    assert str(timeline_path) in result.stderr
+    assert "lab.sh run s1" in result.stderr
+
+
 def test_capture_scenario_without_a_recorded_run_names_the_command_to_run(tmp_path):
     lab_run = make_lab(tmp_path)
     lab_run.write_agent_setup()
diff --git a/monitor/sre-agent-event-lab/scripts/tests/test_lab_state.py b/monitor/sre-agent-event-lab/scripts/tests/test_lab_state.py
index c4ebd9da..bc5bafc1 100644
--- a/monitor/sre-agent-event-lab/scripts/tests/test_lab_state.py
+++ b/monitor/sre-agent-event-lab/scripts/tests/test_lab_state.py
@@ -831,6 +831,17 @@ def test_a_conclusion_is_recorded_once_the_run_recovered(tmp_path):
     assert state.is_successful_capture("s1")
 
 
+def test_record_capture_refuses_to_downgrade_a_conclusion(tmp_path):
+    state = lab_state.LabState(tmp_path / "state.json")
+    state.mark_recovered("s1", str(tmp_path / "s1"))
+    state.record_capture("s1", "conclusion")
+
+    with pytest.raises(lab_state.InvalidTransition, match="already has a conclusion"):
+        state.record_capture("s1", "thread-not-created")
+
+    assert state.capture_status("s1") == "conclusion"
+
+
 def test_cli_record_capture_refuses_a_conclusion_for_a_failed_run(tmp_path):
     path = tmp_path / "state.json"
     evidence_dir = tmp_path / "s1-20260814T000000Z"

From b61161beb1f1d2794275ab823e6c8a0bd058bc91 Mon Sep 17 00:00:00 2001
From: hellices <hellices@naver.com>
Date: Sat, 15 Aug 2026 14:30:20 +0900
Subject: [PATCH 2/2] fix(sre-lab): retain failure reason across trap retry

Keep the intended scenario failure reason pending until mark-failed succeeds, so an EXIT-trap retry cannot replace actionable alert or propagation diagnostics with a generic abort status.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .../sre-agent-event-lab/scripts/run-scenario.sh |  5 ++++-
 .../scripts/tests/lab_script_harness.py         |  8 ++++++++
 .../scripts/tests/test_lab_scripts.py           | 17 +++++++++++++++++
 3 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/monitor/sre-agent-event-lab/scripts/run-scenario.sh b/monitor/sre-agent-event-lab/scripts/run-scenario.sh
index fe2c4901..808d2e0d 100755
--- a/monitor/sre-agent-event-lab/scripts/run-scenario.sh
+++ b/monitor/sre-agent-event-lab/scripts/run-scenario.sh
@@ -78,6 +78,7 @@ readonly ALERT_LIST_ATTEMPT_ERROR_FILE="${EVIDENCE_DIR}/.alert-list-error.tmp"
 
 RECOVERED=0
 OUTCOME_RECORDED=0
+PENDING_FAILURE_REASON=""
 ALERT_LIST_FAILURES=0
 ALERT_LIST_VALID_RESPONSES=0
 ALERT_LIST_POLLS=0
@@ -90,10 +91,12 @@ ALERT_FIRED_AT=""
 
 record_failed_run() {
   local reason="$1"
+  PENDING_FAILURE_REASON="${reason}"
   if ! lab_state mark-failed "${SCENARIO}" "${EVIDENCE_DIR}" --reason "${reason}"; then
     return 1
   fi
   OUTCOME_RECORDED=1
+  PENDING_FAILURE_REASON=""
 }
 
 wait_for_s3_fault() {
@@ -213,7 +216,7 @@ recover_on_exit() {
     echo "CRITICAL: the injected fault is still active. Revert it by hand before running any other scenario." >&2
   fi
   if [[ "${OUTCOME_RECORDED}" -eq 0 ]]; then
-    failure_reason="run aborted with status ${original_status}"
+    failure_reason="${PENDING_FAILURE_REASON:-run aborted with status ${original_status}}"
     if [[ "${recovery_status}" -ne 0 ]]; then
       failure_reason="${failure_reason}; automatic recovery failed"
     fi
diff --git a/monitor/sre-agent-event-lab/scripts/tests/lab_script_harness.py b/monitor/sre-agent-event-lab/scripts/tests/lab_script_harness.py
index 6a7565ea..bea2a249 100644
--- a/monitor/sre-agent-event-lab/scripts/tests/lab_script_harness.py
+++ b/monitor/sre-agent-event-lab/scripts/tests/lab_script_harness.py
@@ -311,6 +311,11 @@ def _lab_python_stub_source(
     if [[ "$*" == *" mark-failed "* && -f "${{state}}/mark_failed_fails" ]]; then
       exit 3
     fi
+    if [[ "$*" == *" mark-failed "* && -f "${{state}}/mark_failed_failures" ]] \
+      && (( $(cat "${{state}}/mark_failed_failures") > 0 )); then
+      printf '%s\n' "$(( $(cat "${{state}}/mark_failed_failures") - 1 ))" > "${{state}}/mark_failed_failures"
+      exit 3
+    fi
     exec "{REAL_PYTHON}" "$@"
     ;;
   *capture_agent.py)
@@ -410,6 +415,7 @@ def make_lab(
     alert_failure_after_success=False,
     render_fails=False,
     alert_list_empty_body=False,
+    mark_failed_failures=0,
 ):
     """A throwaway copy of the lab plus fake CLIs; returns a run context."""
     lab = tmp_path / "lab"
@@ -449,6 +455,8 @@ def make_lab(
         (state_dir / "s3_probe_failures").write_text(f"{s3_probe_failures}\n")
     if mark_failed_fails:
         (state_dir / "mark_failed_fails").write_text("1\n")
+    if mark_failed_failures:
+        (state_dir / "mark_failed_failures").write_text(f"{mark_failed_failures}\n")
     if alert_failure_after_success:
         (state_dir / "alert_failure_after_success").write_text("1\n")
     if render_fails:
diff --git a/monitor/sre-agent-event-lab/scripts/tests/test_lab_scripts.py b/monitor/sre-agent-event-lab/scripts/tests/test_lab_scripts.py
index 6f37803d..94d508db 100644
--- a/monitor/sre-agent-event-lab/scripts/tests/test_lab_scripts.py
+++ b/monitor/sre-agent-event-lab/scripts/tests/test_lab_scripts.py
@@ -384,6 +384,23 @@ def test_run_scenario_preserves_reason_when_failure_state_write_fails(tmp_path):
     assert "CRITICAL: could not record" in result.stderr
 
 
+def test_run_scenario_retries_the_original_failure_reason_from_the_exit_trap(tmp_path):
+    lab_run = make_lab(
+        tmp_path,
+        alert_fires=False,
+        mark_failed_failures=1,
+    )
+    lab_run.seed_state()
+
+    result = lab_run.run("run-scenario.sh", ["s1"], env=NO_ALERT_WAITS)
+
+    assert result.returncode != 0
+    scenario = lab_run.scenario_state("s1")
+    assert scenario["run_status"] == "failed"
+    assert "did not fire" in scenario["failure_reason"]
+    assert "run aborted" not in scenario["failure_reason"]
+
+
 def test_run_scenario_s1_reports_a_rejected_recovery_update_as_critical(tmp_path):
     """`recover` runs both directly and from the EXIT trap, and the trap
     calls it as `if ! recover`, which turns `set -e` off for the whole